agora inbox for [email protected]  

help / color / mirror / Atom feed

[PATCH 2/2] Change error when tuple-routing sees NULL in range keys
51+ messages / 4 participants
[nested] [flat]

* [PATCH 2/2] Change error when tuple-routing sees NULL in range keys
@ 2017-05-16 09:44  amit <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: amit @ 2017-05-16 09:44 UTC (permalink / raw)

---
 src/backend/catalog/partition.c      | 21 ++++++++++++++++-----
 src/test/regress/expected/insert.out |  3 ++-
 2 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/src/backend/catalog/partition.c b/src/backend/catalog/partition.c
index dee904d99d..336305a3bb 100644
--- a/src/backend/catalog/partition.c
+++ b/src/backend/catalog/partition.c
@@ -1964,7 +1964,8 @@ get_partition_for_tuple(PartitionDispatch *pd,
 		{
 			*failed_at = parent;
 			*failed_slot = slot;
-			return -1;
+			result = -1;
+			goto error_exit;
 		}
 
 		/*
@@ -1980,12 +1981,21 @@ get_partition_for_tuple(PartitionDispatch *pd,
 
 		if (key->strategy == PARTITION_STRATEGY_RANGE)
 		{
-			/* Disallow nulls in the range partition key of the tuple */
+			/*
+			 * Since we cannot route tuples with NULL partition keys through
+			 * a range-partitioned table, simply return that no partition
+			 * exists
+			 */
 			for (i = 0; i < key->partnatts; i++)
+			{
 				if (isnull[i])
-					ereport(ERROR,
-							(errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED),
-						errmsg("range partition key of row contains null")));
+				{
+					*failed_at = parent;
+					*failed_slot = slot;
+					result = -1;
+					goto error_exit;
+				}
+			}
 		}
 
 		/*
@@ -2048,6 +2058,7 @@ get_partition_for_tuple(PartitionDispatch *pd,
 			parent = pd[-parent->indexes[cur_index]];
 	}
 
+error_exit:
 	ecxt->ecxt_scantuple = ecxt_scantuple_old;
 	return result;
 }
diff --git a/src/test/regress/expected/insert.out b/src/test/regress/expected/insert.out
index 232777bc3d..8b0752a0d2 100644
--- a/src/test/regress/expected/insert.out
+++ b/src/test/regress/expected/insert.out
@@ -247,7 +247,8 @@ insert into range_parted values ('b', 1);
 insert into range_parted values ('b', 10);
 -- fail (partition key (b+0) is null)
 insert into range_parted values ('a');
-ERROR:  range partition key of row contains null
+ERROR:  no partition of relation "range_parted" found for row
+DETAIL:  Partition key of the failing row contains (a, (b + 0)) = (a, null).
 select tableoid::regclass, * from range_parted;
  tableoid | a | b  
 ----------+---+----
-- 
2.11.0


--------------329A8F02F9EB16ACAD79147E
Content-Type: text/plain
Content-Disposition: inline
Content-Transfer-Encoding: 8bit
MIME-Version: 1.0


-- 
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

--------------329A8F02F9EB16ACAD79147E--





^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* [PATCH 2/8] Expand the external data before forming the tuple
@ 2021-03-04 09:25  Dilip Kumar <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Dilip Kumar @ 2021-03-04 09:25 UTC (permalink / raw)

All the callers of HeapTupleGetDatum and HeapTupleHeaderGetDatum, who might
contain the external varlena in their tuple, try to flatten the external
varlena before forming the tuple wherever it is possible.

Dilip Kumar based on idea by Robert Haas
---
 src/backend/executor/execExprInterp.c | 29 +++++++++++++++++++++++++--
 src/backend/utils/adt/jsonfuncs.c     |  8 ++++++--
 src/pl/plpgsql/src/pl_exec.c          |  5 ++++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 20949a5dec..c3754acca4 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -2840,12 +2840,25 @@ ExecEvalRow(ExprState *state, ExprEvalStep *op)
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < op->d.row.tupdesc->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(op->d.row.tupdesc, i);
+
+		if (op->d.row.elemnulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.row.elemvalues[i])))
+			continue;
+
+		op->d.row.elemvalues[i] =
+			PointerGetDatum(PG_DETOAST_DATUM_PACKED(op->d.row.elemvalues[i]));
+	}
+
 	/* build tuple from evaluated field values */
 	tuple = heap_form_tuple(op->d.row.tupdesc,
 							op->d.row.elemvalues,
 							op->d.row.elemnulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
@@ -3085,12 +3098,24 @@ ExecEvalFieldStoreForm(ExprState *state, ExprEvalStep *op, ExprContext *econtext
 {
 	HeapTuple	tuple;
 
+	/* Retrieve externally stored values and decompress. */
+	for (int i = 0; i < (*op->d.fieldstore.argdesc)->natts; i++)
+	{
+		Form_pg_attribute attr = TupleDescAttr(*op->d.fieldstore.argdesc, i);
+
+		if (op->d.fieldstore.nulls[i] || attr->attlen != -1 ||
+			!VARATT_IS_EXTERNAL(DatumGetPointer(op->d.fieldstore.values[i])))
+			continue;
+		op->d.fieldstore.values[i] = PointerGetDatum(
+						PG_DETOAST_DATUM_PACKED(op->d.fieldstore.values[i]));
+	}
+
 	/* argdesc should already be valid from the DeForm step */
 	tuple = heap_form_tuple(*op->d.fieldstore.argdesc,
 							op->d.fieldstore.values,
 							op->d.fieldstore.nulls);
 
-	*op->resvalue = HeapTupleGetDatum(tuple);
+	*op->resvalue = HeapTupleGetRawDatum(tuple);
 	*op->resnull = false;
 }
 
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 511467280f..c3d464f42b 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2968,7 +2968,7 @@ populate_composite(CompositeIOData *io,
 		/* populate resulting record tuple */
 		tuple = populate_record(io->tupdesc, &io->record_io,
 								defaultval, mcxt, &jso);
-		result = HeapTupleHeaderGetDatum(tuple);
+		result = HeapTupleHeaderGetRawDatum(tuple);
 
 		JsObjectFree(&jso);
 	}
@@ -3387,6 +3387,10 @@ populate_record(TupleDesc tupdesc,
 										  nulls[i] ? (Datum) 0 : values[i],
 										  &field,
 										  &nulls[i]);
+
+		if (!nulls[i] && att->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(values[i])))
+			values[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(values[i]));
 	}
 
 	res = heap_form_tuple(tupdesc, values, nulls);
@@ -3765,7 +3769,7 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
 
 	/* if it's domain over composite, check domain constraints */
 	if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
-		domain_check(HeapTupleHeaderGetDatum(tuphead), false,
+		domain_check(HeapTupleHeaderGetRawDatum(tuphead), false,
 					 cache->argtype,
 					 &cache->c.io.composite.domain_info,
 					 cache->fn_mcxt);
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index b4c70aaa7f..fd073767bc 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -5326,7 +5326,7 @@ exec_eval_datum(PLpgSQL_execstate *estate,
 					elog(ERROR, "row not compatible with its own tupdesc");
 				*typeid = row->rowtupdesc->tdtypeid;
 				*typetypmod = row->rowtupdesc->tdtypmod;
-				*value = HeapTupleGetDatum(tup);
+				*value = HeapTupleGetRawDatum(tup);
 				*isnull = false;
 				MemoryContextSwitchTo(oldcontext);
 				break;
@@ -7300,6 +7300,9 @@ make_tuple_from_row(PLpgSQL_execstate *estate,
 						&dvalues[i], &nulls[i]);
 		if (fieldtypeid != TupleDescAttr(tupdesc, i)->atttypid)
 			return NULL;
+		if (!nulls[i] && TupleDescAttr(tupdesc, i)->attlen == -1 &&
+			VARATT_IS_EXTERNAL(DatumGetPointer(dvalues[i])))
+			dvalues[i] = PointerGetDatum(PG_DETOAST_DATUM_PACKED(dvalues[i]));
 		/* XXX should we insist on typmod match, too? */
 	}
 
-- 
2.17.0


--C94crkcyjafcjHxo
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-Disallow-compressed-data-inside-container-types.patch"



^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* Re: [PATCH] Add native windows on arm64 support
@ 2023-01-17 22:51  Andres Freund <[email protected]>
  0 siblings, 1 reply; 51+ messages in thread

From: Andres Freund @ 2023-01-17 22:51 UTC (permalink / raw)
  To: Niyas Sait <[email protected]>; +Cc: Michael Paquier <[email protected]>; Ian Lawrence Barwick <[email protected]>; Tom Lane <[email protected]>; Thomas Munro <[email protected]>; Julien Rouhaud <[email protected]>; PostgreSQL Hackers <[email protected]>

Hi,

On 2022-12-16 10:52:23 +0000, Niyas Sait wrote:
> Subject: [PATCH v7] Enable postgres native build for windows-arm64 platform

>  elif host_cpu == 'arm' or host_cpu == 'aarch64'
>  
> -  prog = '''
> +  if cc.get_id() == 'msvc'
> +    cdata.set('USE_ARMV8_CRC32C', false)
> +    cdata.set('USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK', 1)
> +    have_optimized_crc = true

I dimly recall that windows might actually require the relevant extension on
arm?


> +  else
> +    prog = '''
>  #include <arm_acle.h>

I'd just make this include #ifdef _MSV_VER (or whatever it is).


>  int main(void)
> @@ -1960,18 +1966,19 @@ int main(void)
>  }
>  '''
>  
> -  if cc.links(prog, name: '__crc32cb, __crc32ch, __crc32cw, and __crc32cd without -march=armv8-a+crc',
> -      args: test_c_args)
> -    # Use ARM CRC Extension unconditionally
> -    cdata.set('USE_ARMV8_CRC32C', 1)
> -    have_optimized_crc = true
> -  elif cc.links(prog, name: '__crc32cb, __crc32ch, __crc32cw, and __crc32cd with -march=armv8-a+crc',
> -      args: test_c_args + ['-march=armv8-a+crc'])
> -    # Use ARM CRC Extension, with runtime check
> -    cflags_crc += '-march=armv8-a+crc'
> -    cdata.set('USE_ARMV8_CRC32C', false)
> -    cdata.set('USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK', 1)
> -    have_optimized_crc = true
> +    if cc.links(prog, name: '__crc32cb, __crc32ch, __crc32cw, and __crc32cd without -march=armv8-a+crc',
> +        args: test_c_args)

Seems like it'd be easier to read if you don't re-indent this, but just have
the cc.get_id() == 'msvc' part of this if/else-if.


Greetings,

Andres Freund






^ permalink  raw  reply  [nested|flat] 51+ messages in thread

* Re: [PATCH] Add native windows on arm64 support
@ 2023-01-19 10:09  Niyas Sait <[email protected]>
  parent: Andres Freund <[email protected]>
  0 siblings, 0 replies; 51+ messages in thread

From: Niyas Sait @ 2023-01-19 10:09 UTC (permalink / raw)
  To: Andres Freund <[email protected]>; +Cc: Michael Paquier <[email protected]>; Ian Lawrence Barwick <[email protected]>; Tom Lane <[email protected]>; Thomas Munro <[email protected]>; Julien Rouhaud <[email protected]>; PostgreSQL Hackers <[email protected]>



On 17/01/2023 22:51, Andres Freund wrote:
> Hi,
> 
> On 2022-12-16 10:52:23 +0000, Niyas Sait wrote:
>> Subject: [PATCH v7] Enable postgres native build for windows-arm64 platform
> 
>>   elif host_cpu == 'arm' or host_cpu == 'aarch64'
>>   
>> -  prog = '''
>> +  if cc.get_id() == 'msvc'
>> +    cdata.set('USE_ARMV8_CRC32C', false)
>> +    cdata.set('USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK', 1)
>> +    have_optimized_crc = true
> 
> I dimly recall that windows might actually require the relevant extension on
> arm?

Do you mean we don't need the runtime checks for CRC ?

CRC is an optional extension for ARMv8 base architecture. I am not sure 
if windows make it mandatory to have this implementation.

> 
>> +  else
>> +    prog = '''
>>   #include <arm_acle.h>
> 
> I'd just make this include #ifdef _MSV_VER (or whatever it is).

The code snippet is not used for MSVC part. I am not sure why we need to 
add the #ifdef _MSC_VER.

>>   int main(void)
>> @@ -1960,18 +1966,19 @@ int main(void)
>>   }
>>   '''
>>   
>> -  if cc.links(prog, name: '__crc32cb, __crc32ch, __crc32cw, and __crc32cd without -march=armv8-a+crc',
>> -      args: test_c_args)
>> -    # Use ARM CRC Extension unconditionally
>> -    cdata.set('USE_ARMV8_CRC32C', 1)
>> -    have_optimized_crc = true
>> -  elif cc.links(prog, name: '__crc32cb, __crc32ch, __crc32cw, and __crc32cd with -march=armv8-a+crc',
>> -      args: test_c_args + ['-march=armv8-a+crc'])
>> -    # Use ARM CRC Extension, with runtime check
>> -    cflags_crc += '-march=armv8-a+crc'
>> -    cdata.set('USE_ARMV8_CRC32C', false)
>> -    cdata.set('USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK', 1)
>> -    have_optimized_crc = true
>> +    if cc.links(prog, name: '__crc32cb, __crc32ch, __crc32cw, and __crc32cd without -march=armv8-a+crc',
>> +        args: test_c_args)
> 
> Seems like it'd be easier to read if you don't re-indent this, but just have
> the cc.get_id() == 'msvc' part of this if/else-if.
> 

Yes that looks better. will do it in next patch.

Thanks Andres for the review.

-- 
Niyas






^ permalink  raw  reply  [nested|flat] 51+ messages in thread

end of thread, other threads:[~2023-01-19 10:09 UTC | newest]

Thread overview: 51+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2017-05-16 09:44 [PATCH 2/2] Change error when tuple-routing sees NULL in range keys amit <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2021-03-04 09:25 [PATCH 2/8] Expand the external data before forming the tuple Dilip Kumar <[email protected]>
2023-01-17 22:51 Re: [PATCH] Add native windows on arm64 support Andres Freund <[email protected]>
2023-01-19 10:09 ` Re: [PATCH] Add native windows on arm64 support Niyas Sait <[email protected]>

This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox