[PATCH 2/8] Expand the external data before forming the tuple

public inbox for [email protected]  
help / color / mirror / Atom feed

[PATCH 2/8] Expand the external data before forming the tuple
48+ messages / 1 participants
[nested] [flat]

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25 Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 48+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 48+ messages in thread

end of thread, other threads:[~2021-03-04 09:25 UTC | newest]

Thread overview: 48+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>

This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox