public inbox for [email protected]  

help / color / mirror / Atom feed

[PATCH 2/8] Expand the external data before forming the tuple
50+ messages / 3 participants
[nested] [flat]

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* Re: Docs: Encourage strong server verification with SCRAM
@ 2023-05-26 01:09 Jonathan S. Katz <[email protected]>
  2023-05-26 22:47 ` Re: Docs: Encourage strong server verification with SCRAM Jacob Champion <[email protected]>
  0 siblings, 1 reply; 50+ messages in thread

From: Jonathan S. Katz @ 2023-05-26 01:09 UTC (permalink / raw)
  To: Jacob Champion <[email protected]>; +Cc: Stephen Frost <[email protected]>; Daniel Gustafsson <[email protected]>; PostgreSQL Hackers <[email protected]>; Michael Paquier <[email protected]>

On 5/25/23 3:27 PM, Jacob Champion wrote:
> On Thu, May 25, 2023 at 10:48 AM Jonathan S. Katz <[email protected]> wrote:
>> Overall, +1 to tightening the language around the docs in this area.
>>
>> However, to paraphrase Stephen, I think the language, as currently
>> written, makes the problem sound scarier than it actually is, and I
>> agree that we should just inline it above.
> 
> How does v2 look? I more or less divided the current text into a local
> section and a network section. (I'm still not clear on where in the
> current text you're wanting me to inline a sudden aside on SCRAM; it
> doesn't really seem to fit in any of the existing paragraphs.)

I read through the proposal and like this much better. I missed 
Stephen's point on the "where" to put it in this section; I actually 
don't know if I agree with that (he says while painting the bikeshed), 
given the we spend two paragraphs describing how to prevent spoofing in 
general over the network, vs. just during SCRAM authentication.

I rewrote this to just focus on server spoofing that can occur with 
SCRAM authentication and did some wordsmithing. I was torn on keeping in 
the part of offline analysis of an intercepted hash, given one can do 
this with md5 as well, but perhaps it helps elaborate on the consequences.

Thanks,

Jonathan

diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml
index dbe23db54f..a3f4b258f7 100644
--- a/doc/src/sgml/runtime.sgml
+++ b/doc/src/sgml/runtime.sgml
@@ -2014,6 +2014,18 @@ pg_dumpall -p 5432 | psql -d postgres -p 5433
    CA.
   </para>
 
+  <para>
+   To prevent server spoofing from occurring when using
+   <link linkend="auth-password">scram-sha-256</link> password authentication
+   over a network, you should ensure you are connecting using SSL. Additionally,
+   the SCRAM implementation in <application>libpq</application> cannot protect
+   the entire authentication exchange, but using the
+   <literal>channel_binding=require</literal> connection parameter provides a
+   mitigation against server spoofing. An attacker that uses a rogue server to
+   intercept a SCRAM exchange can use offline analysis to determine the hashed
+   password from the client.
+  </para>
+
   <para>
     To prevent spoofing with GSSAPI, the server must be configured to accept
     only <literal>hostgssenc</literal> connections


Attachments:

  [text/plain] v3-docs-encourage-strong-server-verification-with-SCRAM.patch (1.0K, ../../[email protected]/2-v3-docs-encourage-strong-server-verification-with-SCRAM.patch)
  download | inline diff:
diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml
index dbe23db54f..a3f4b258f7 100644
--- a/doc/src/sgml/runtime.sgml
+++ b/doc/src/sgml/runtime.sgml
@@ -2014,6 +2014,18 @@ pg_dumpall -p 5432 | psql -d postgres -p 5433
    CA.
   </para>
 
+  <para>
+   To prevent server spoofing from occurring when using
+   <link linkend="auth-password">scram-sha-256</link> password authentication
+   over a network, you should ensure you are connecting using SSL. Additionally,
+   the SCRAM implementation in <application>libpq</application> cannot protect
+   the entire authentication exchange, but using the
+   <literal>channel_binding=require</literal> connection parameter provides a
+   mitigation against server spoofing. An attacker that uses a rogue server to
+   intercept a SCRAM exchange can use offline analysis to determine the hashed
+   password from the client.
+  </para>
+
   <para>
     To prevent spoofing with GSSAPI, the server must be configured to accept
     only <literal>hostgssenc</literal> connections


  [application/pgp-signature] OpenPGP_signature (840B, ../../[email protected]/3-OpenPGP_signature)
  download

^ permalink  raw  reply  [nested|flat] 50+ messages in thread

* Re: Docs: Encourage strong server verification with SCRAM
  2023-05-26 01:09 Re: Docs: Encourage strong server verification with SCRAM Jonathan S. Katz <[email protected]>
@ 2023-05-26 22:47 ` Jacob Champion <[email protected]>
  0 siblings, 0 replies; 50+ messages in thread

From: Jacob Champion @ 2023-05-26 22:47 UTC (permalink / raw)
  To: Jonathan S. Katz <[email protected]>; +Cc: Stephen Frost <[email protected]>; Daniel Gustafsson <[email protected]>; PostgreSQL Hackers <[email protected]>; Michael Paquier <[email protected]>

On Thu, May 25, 2023 at 6:10 PM Jonathan S. Katz <[email protected]> wrote:
> I read through the proposal and like this much better.

Great!

> I rewrote this to just focus on server spoofing that can occur with
> SCRAM authentication and did some wordsmithing. I was torn on keeping in
> the part of offline analysis of an intercepted hash, given one can do
> this with md5 as well, but perhaps it helps elaborate on the consequences.

This part:

> +   To prevent server spoofing from occurring when using
> +   <link linkend="auth-password">scram-sha-256</link> password authentication
> +   over a network, you should ensure you are connecting using SSL.

seems to backtrack on the recommendation -- you have to use
sslmode=verify-full, not just SSL, to avoid handing a weak(er) hash to
an untrusted party.

--Jacob






^ permalink  raw  reply  [nested|flat] 50+ messages in thread

end of thread, other threads:[~2023-05-26 22:47 UTC | newest]

Thread overview: 50+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2023-05-26 01:09 Re: Docs: Encourage strong server verification with SCRAM Jonathan S. Katz <[email protected]>
2023-05-26 22:47 ` Re: Docs: Encourage strong server verification with SCRAM Jacob Champion <[email protected]>

This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox