public inbox for [email protected]help / color / mirror / Atom feed
[PATCH 1/3] anyarray_anyelement_operators 2+ messages / 2 participants [nested] [flat]
* [PATCH 1/3] anyarray_anyelement_operators @ 2021-03-13 10:01 Mark Rofail <[email protected]> 0 siblings, 0 replies; 2+ messages in thread From: Mark Rofail @ 2021-03-13 10:01 UTC (permalink / raw) --- doc/src/sgml/func.sgml | 28 +++++ doc/src/sgml/gin.sgml | 8 +- doc/src/sgml/indices.sgml | 2 +- src/backend/access/gin/ginarrayproc.c | 43 +++++-- src/backend/utils/adt/arrayfuncs.c | 137 +++++++++++++++++++++++ src/include/catalog/pg_amop.dat | 3 + src/include/catalog/pg_operator.dat | 14 ++- src/include/catalog/pg_proc.dat | 6 + src/test/regress/expected/arrays.out | 92 +++++++++++++++ src/test/regress/expected/gin.out | 34 ++++++ src/test/regress/expected/opr_sanity.out | 6 +- src/test/regress/sql/arrays.sql | 10 ++ src/test/regress/sql/gin.sql | 8 ++ 13 files changed, 375 insertions(+), 16 deletions(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index b7150510ab..9a3f79e3b7 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -17525,6 +17525,34 @@ SELECT NULLIF(value, '(none)') ... </para></entry> </row> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <type>anyarray</type> <literal>@>></literal> <type>anyelement</type> + <returnvalue>boolean</returnvalue> + </para> + <para> + Does the array contain the specified element? + </para> + <para> + <literal>ARRAY[1,4,3] @>> 3</literal> + <returnvalue>t</returnvalue> + </para></entry> + </row> + + <row> + <entry role="func_table_entry"><para role="func_signature"> + <type>anyelement</type> <literal><<@</literal> <type>anyarray</type> + <returnvalue>boolean</returnvalue> + </para> + <para> + Is the specified element contained in the array? + </para> + <para> + <literal>2 <<@ ARRAY[1,7,4,2,6]</literal> + <returnvalue>t</returnvalue> + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <type>anyarray</type> <literal>&&</literal> <type>anyarray</type> diff --git a/doc/src/sgml/gin.sgml b/doc/src/sgml/gin.sgml index d68d12d515..981513b765 100644 --- a/doc/src/sgml/gin.sgml +++ b/doc/src/sgml/gin.sgml @@ -84,7 +84,7 @@ </thead> <tbody> <row> - <entry morerows="3" valign="middle"><literal>array_ops</literal></entry> + <entry morerows="5" valign="middle"><literal>array_ops</literal></entry> <entry><literal>&& (anyarray,anyarray)</literal></entry> </row> <row> @@ -93,6 +93,12 @@ <row> <entry><literal><@ (anyarray,anyarray)</literal></entry> </row> + <row> + <entry><literal>@>> (anyarray,anyelement)</literal></entry> + </row> + <row> + <entry><literal><<@ (anyelement,anyarray)</literal></entry> + </row> <row> <entry><literal>= (anyarray,anyarray)</literal></entry> </row> diff --git a/doc/src/sgml/indices.sgml b/doc/src/sgml/indices.sgml index 623962d1d8..6de6c33c75 100644 --- a/doc/src/sgml/indices.sgml +++ b/doc/src/sgml/indices.sgml @@ -326,7 +326,7 @@ SELECT * FROM places ORDER BY location <-> point '(101,456)' LIMIT 10; for arrays, which supports indexed queries using these operators: <synopsis> -<@ @> = && +<@ @> <<@ @>> = && </synopsis> (See <xref linkend="functions-array"/> for the meaning of diff --git a/src/backend/access/gin/ginarrayproc.c b/src/backend/access/gin/ginarrayproc.c index bf73e32932..b10bd04ec8 100644 --- a/src/backend/access/gin/ginarrayproc.c +++ b/src/backend/access/gin/ginarrayproc.c @@ -24,6 +24,7 @@ #define GinContainsStrategy 2 #define GinContainedStrategy 3 #define GinEqualStrategy 4 +#define GinContainsElemStrategy 5 /* @@ -78,8 +79,6 @@ ginarrayextract_2args(PG_FUNCTION_ARGS) Datum ginqueryarrayextract(PG_FUNCTION_ARGS) { - /* Make copy of array input to ensure it doesn't disappear while in use */ - ArrayType *array = PG_GETARG_ARRAYTYPE_P_COPY(0); int32 *nkeys = (int32 *) PG_GETARG_POINTER(1); StrategyNumber strategy = PG_GETARG_UINT16(2); @@ -87,21 +86,33 @@ ginqueryarrayextract(PG_FUNCTION_ARGS) /* Pointer *extra_data = (Pointer *) PG_GETARG_POINTER(4); */ bool **nullFlags = (bool **) PG_GETARG_POINTER(5); int32 *searchMode = (int32 *) PG_GETARG_POINTER(6); - int16 elmlen; - bool elmbyval; - char elmalign; Datum *elems; bool *nulls; int nelems; - get_typlenbyvalalign(ARR_ELEMTYPE(array), - &elmlen, &elmbyval, &elmalign); + if (strategy == GinContainsElemStrategy) + { + /* single element is passed, set elems to its pointer */ + elems = &PG_GETARG_DATUM(0); + nulls = &PG_ARGISNULL(0); + nelems = 1; + } + else + { + /* Make copy of array input to ensure it doesn't disappear while in use */ + ArrayType *array = PG_GETARG_ARRAYTYPE_P_COPY(0); + int16 elmlen; + bool elmbyval; + char elmalign; - deconstruct_array(array, - ARR_ELEMTYPE(array), - elmlen, elmbyval, elmalign, - &elems, &nulls, &nelems); + get_typlenbyvalalign(ARR_ELEMTYPE(array), + &elmlen, &elmbyval, &elmalign); + deconstruct_array(array, + ARR_ELEMTYPE(array), + elmlen, elmbyval, elmalign, + &elems, &nulls, &nelems); + } *nkeys = nelems; *nullFlags = nulls; @@ -126,6 +137,14 @@ ginqueryarrayextract(PG_FUNCTION_ARGS) else *searchMode = GIN_SEARCH_MODE_INCLUDE_EMPTY; break; + case GinContainsElemStrategy: + /* + * only items that match the queried element + * are considered candidate + */ + + *searchMode = GIN_SEARCH_MODE_DEFAULT; + break; default: elog(ERROR, "ginqueryarrayextract: unknown strategy number: %d", strategy); @@ -185,6 +204,7 @@ ginarrayconsistent(PG_FUNCTION_ARGS) } } break; + case GinContainsElemStrategy: case GinContainedStrategy: /* we will need recheck */ *recheck = true; @@ -274,6 +294,7 @@ ginarraytriconsistent(PG_FUNCTION_ARGS) } } break; + case GinContainsElemStrategy: case GinContainedStrategy: /* can't do anything else useful here */ res = GIN_MAYBE; diff --git a/src/backend/utils/adt/arrayfuncs.c b/src/backend/utils/adt/arrayfuncs.c index f7012cc5d9..f8cbf64c9e 100644 --- a/src/backend/utils/adt/arrayfuncs.c +++ b/src/backend/utils/adt/arrayfuncs.c @@ -4328,6 +4328,143 @@ arraycontained(PG_FUNCTION_ARGS) } +/* + * array_contains_elem : checks an array for a specific element + * adapted from array_contain_compare() for containment of a single element + */ +static bool +array_contains_elem(AnyArrayType *array, Datum elem, Oid elemtype, + Oid collation, void **fn_extra) +{ + LOCAL_FCINFO(locfcinfo, 2); + Oid arrtype = AARR_ELEMTYPE(array); + TypeCacheEntry *typentry; + int nelems; + int typlen; + bool typbyval; + char typalign; + int i; + array_iter it; + + if (arrtype != elemtype) + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("cannot compare arrays elements with element of different type"))); + + /* + * We arrange to look up the equality function only once per series of + * calls, assuming the element type doesn't change underneath us. The + * typcache is used so that we have no memory leakage when being used as + * an index support function. + */ + typentry = (TypeCacheEntry *) *fn_extra; + if (typentry == NULL || + typentry->type_id != arrtype) + { + typentry = lookup_type_cache(arrtype, + TYPECACHE_EQ_OPR_FINFO); + if (!OidIsValid(typentry->eq_opr_finfo.fn_oid)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_FUNCTION), + errmsg("could not identify an equality operator for type %s", + format_type_be(arrtype)))); + *fn_extra = (void *) typentry; + } + typlen = typentry->typlen; + typbyval = typentry->typbyval; + typalign = typentry->typalign; + + /* + * Apply the comparison operator for the passed element against each + * element in the array + */ + InitFunctionCallInfoData(*locfcinfo, &typentry->eq_opr_finfo, 2, + collation, NULL, NULL); + + /* Loop over source data */ + nelems = ArrayGetNItems(AARR_NDIM(array), AARR_DIMS(array)); + array_iter_setup(&it, array); + + for (i = 0; i < nelems; i++) + { + Datum elt; + bool isnull; + bool oprresult; + + /* Get element, checking for NULL */ + elt = array_iter_next(&it, &isnull, i, typlen, typbyval, typalign); + + /* + * We assume that the comparison operator is strict, so a NULL can't + * match anything. refer to the comment in array_contain_compare() + */ + if (isnull) + continue; + + /* + * Apply the operator to the element pair; treat NULL as false + */ + locfcinfo->args[0].value = elt; + locfcinfo->args[0].isnull = false; + locfcinfo->args[1].value = elem; + locfcinfo->args[1].isnull = false; + locfcinfo->isnull = false; + oprresult = DatumGetBool(FunctionCallInvoke(locfcinfo)); + if (!locfcinfo->isnull && oprresult) + return true; + } + + return false; +} + +Datum +arraycontainselem(PG_FUNCTION_ARGS) +{ + AnyArrayType *array = PG_GETARG_ANY_ARRAY_P(0); + Datum elem = PG_GETARG_DATUM(1); + Oid elemtype = get_fn_expr_argtype(fcinfo->flinfo, 1); + Oid collation = PG_GET_COLLATION(); + bool result; + + /* + * we don't need to check if the elem is null or if the elem datatype and + * array datatype match since this is handled within internal calls already + * (a property of polymorphic functions) + */ + + result = array_contains_elem(array, elem, elemtype, collation, + &fcinfo->flinfo->fn_extra); + + /* Avoid leaking memory when handed toasted input */ + AARR_FREE_IF_COPY(array, 0); + + PG_RETURN_BOOL(result); +} + +Datum +arrayelemcontained(PG_FUNCTION_ARGS) +{ + AnyArrayType *array = PG_GETARG_ANY_ARRAY_P(1); + Datum elem = PG_GETARG_DATUM(0); + Oid elemtype = get_fn_expr_argtype(fcinfo->flinfo, 0); + Oid collation = PG_GET_COLLATION(); + bool result; + + /* + * we don't need to check if the elem is null or if the elem datatype and + * array datatype match since this is handled within internal calls already + * (a property of polymorphic functions) + */ + + result = array_contains_elem(array, elem, elemtype, collation, + &fcinfo->flinfo->fn_extra); + + /* Avoid leaking memory when handed toasted input */ + AARR_FREE_IF_COPY(array, 1); + + PG_RETURN_BOOL(result); +} + /*----------------------------------------------------------------------------- * Array iteration functions * These functions are used to iterate efficiently through arrays diff --git a/src/include/catalog/pg_amop.dat b/src/include/catalog/pg_amop.dat index 0f7ff63669..8a14fc7140 100644 --- a/src/include/catalog/pg_amop.dat +++ b/src/include/catalog/pg_amop.dat @@ -1242,6 +1242,9 @@ { amopfamily => 'gin/array_ops', amoplefttype => 'anyarray', amoprighttype => 'anyarray', amopstrategy => '4', amopopr => '=(anyarray,anyarray)', amopmethod => 'gin' }, +{ amopfamily => 'gin/array_ops', amoplefttype => 'anyarray', + amoprighttype => 'anyelement', amopstrategy => '5', + amopopr => '@>>(anyarray,anyelement)', amopmethod => 'gin' }, # btree enum_ops { amopfamily => 'btree/enum_ops', amoplefttype => 'anyenum', diff --git a/src/include/catalog/pg_operator.dat b/src/include/catalog/pg_operator.dat index 0d4eac8f96..7ef071135c 100644 --- a/src/include/catalog/pg_operator.dat +++ b/src/include/catalog/pg_operator.dat @@ -2761,7 +2761,7 @@ oprresult => 'bool', oprcode => 'circle_overabove', oprrest => 'positionsel', oprjoin => 'positionjoinsel' }, -# overlap/contains/contained for arrays +# overlap/contains/contained/elemcontained/containselem for arrays { oid => '2750', oid_symbol => 'OID_ARRAY_OVERLAP_OP', descr => 'overlaps', oprname => '&&', oprleft => 'anyarray', oprright => 'anyarray', oprresult => 'bool', oprcom => '&&(anyarray,anyarray)', @@ -2778,6 +2778,18 @@ oprresult => 'bool', oprcom => '@>(anyarray,anyarray)', oprcode => 'arraycontained', oprrest => 'arraycontsel', oprjoin => 'arraycontjoinsel' }, +{ oid => '6108', oid_symbol => 'OID_ARRAY_ELEMCONTAINED_OP', + descr => 'elem is contained by', + oprname => '<<@', oprleft => 'anyelement', oprright => 'anyarray', + oprresult => 'bool', oprcom => '@>>(anyarray,anyelement)', + oprcode => 'arrayelemcontained', oprrest => 'arraycontsel', + oprjoin => 'arraycontjoinsel' }, +{ oid => '6105', oid_symbol => 'OID_ARRAY_CONTAINSELEM_OP', + descr => 'contains elem', + oprname => '@>>', oprleft => 'anyarray', oprright => 'anyelement', + oprresult => 'bool', oprcom => '<<@(anyelement,anyarray)', + oprcode => 'arraycontainselem', oprrest => 'arraycontsel', + oprjoin => 'arraycontjoinsel' }, # capturing operators to preserve pre-8.3 behavior of text concatenation { oid => '2779', descr => 'concatenate', diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 4e0c9be58c..8bc05707c7 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -8180,6 +8180,12 @@ { oid => '2749', proname => 'arraycontained', prorettype => 'bool', proargtypes => 'anyarray anyarray', prosrc => 'arraycontained' }, +{ oid => '6109', + proname => 'arrayelemcontained', prorettype => 'bool', + proargtypes => 'anyelement anyarray', prosrc => 'arrayelemcontained' }, +{ oid => '6107', + proname => 'arraycontainselem', prorettype => 'bool', + proargtypes => 'anyarray anyelement', prosrc => 'arraycontainselem' }, # BRIN minmax { oid => '3383', descr => 'BRIN minmax support', diff --git a/src/test/regress/expected/arrays.out b/src/test/regress/expected/arrays.out index 8bc7721e7d..95c9ae5443 100644 --- a/src/test/regress/expected/arrays.out +++ b/src/test/regress/expected/arrays.out @@ -758,6 +758,28 @@ SELECT * FROM array_op_test WHERE i @> '{32}' ORDER BY seqno; 100 | {85,32,57,39,49,84,32,3,30} | {AAAAAAA80240,AAAAAAAAAAAAAAAA1729,AAAAA60038,AAAAAAAAAAA92631,AAAAAAAA9523} (6 rows) +SELECT * FROM array_op_test WHERE i @>> 32 ORDER BY seqno; + seqno | i | t +-------+---------------------------------+------------------------------------------------------------------------------------------------------------------------------------ + 6 | {39,35,5,94,17,92,60,32} | {AAAAAAAAAAAAAAA35875,AAAAAAAAAAAAAAAA23657} + 74 | {32} | {AAAAAAAAAAAAAAAA1729,AAAAAAAAAAAAA22860,AAAAAA99807,AAAAA17383,AAAAAAAAAAAAAAA67062,AAAAAAAAAAA15165,AAAAAAAAAAA50956} + 77 | {97,15,32,17,55,59,18,37,50,39} | {AAAAAAAAAAAA67946,AAAAAA54032,AAAAAAAA81587,55847,AAAAAAAAAAAAAA28620,AAAAAAAAAAAAAAAAA43052,AAAAAA75463,AAAA49534,AAAAAAAA44066} + 89 | {40,32,17,6,30,88} | {AA44673,AAAAAAAAAAA6119,AAAAAAAAAAAAAAAA23657,AAAAAAAAAAAAAAAAAA47955,AAAAAAAAAAAAAAAA33598,AAAAAAAAAAA33576,AA44673} + 98 | {38,34,32,89} | {AAAAAAAAAAAAAAAAAA71621,AAAA8857,AAAAAAAAAAAAAAAAAAA65037,AAAAAAAAAAAAAAAA31334,AAAAAAAAAA48845} + 100 | {85,32,57,39,49,84,32,3,30} | {AAAAAAA80240,AAAAAAAAAAAAAAAA1729,AAAAA60038,AAAAAAAAAAA92631,AAAAAAAA9523} +(6 rows) + +SELECT * FROM array_op_test WHERE 32 <<@ i ORDER BY seqno; + seqno | i | t +-------+---------------------------------+------------------------------------------------------------------------------------------------------------------------------------ + 6 | {39,35,5,94,17,92,60,32} | {AAAAAAAAAAAAAAA35875,AAAAAAAAAAAAAAAA23657} + 74 | {32} | {AAAAAAAAAAAAAAAA1729,AAAAAAAAAAAAA22860,AAAAAA99807,AAAAA17383,AAAAAAAAAAAAAAA67062,AAAAAAAAAAA15165,AAAAAAAAAAA50956} + 77 | {97,15,32,17,55,59,18,37,50,39} | {AAAAAAAAAAAA67946,AAAAAA54032,AAAAAAAA81587,55847,AAAAAAAAAAAAAA28620,AAAAAAAAAAAAAAAAA43052,AAAAAA75463,AAAA49534,AAAAAAAA44066} + 89 | {40,32,17,6,30,88} | {AA44673,AAAAAAAAAAA6119,AAAAAAAAAAAAAAAA23657,AAAAAAAAAAAAAAAAAA47955,AAAAAAAAAAAAAAAA33598,AAAAAAAAAAA33576,AA44673} + 98 | {38,34,32,89} | {AAAAAAAAAAAAAAAAAA71621,AAAA8857,AAAAAAAAAAAAAAAAAAA65037,AAAAAAAAAAAAAAAA31334,AAAAAAAAAA48845} + 100 | {85,32,57,39,49,84,32,3,30} | {AAAAAAA80240,AAAAAAAAAAAAAAAA1729,AAAAA60038,AAAAAAAAAAA92631,AAAAAAAA9523} +(6 rows) + SELECT * FROM array_op_test WHERE i && '{32}' ORDER BY seqno; seqno | i | t -------+---------------------------------+------------------------------------------------------------------------------------------------------------------------------------ @@ -782,6 +804,32 @@ SELECT * FROM array_op_test WHERE i @> '{17}' ORDER BY seqno; 89 | {40,32,17,6,30,88} | {AA44673,AAAAAAAAAAA6119,AAAAAAAAAAAAAAAA23657,AAAAAAAAAAAAAAAAAA47955,AAAAAAAAAAAAAAAA33598,AAAAAAAAAAA33576,AA44673} (8 rows) +SELECT * FROM array_op_test WHERE 17 <<@ i ORDER BY seqno; + seqno | i | t +-------+---------------------------------+------------------------------------------------------------------------------------------------------------------------------------ + 6 | {39,35,5,94,17,92,60,32} | {AAAAAAAAAAAAAAA35875,AAAAAAAAAAAAAAAA23657} + 12 | {17,99,18,52,91,72,0,43,96,23} | {AAAAA33250,AAAAAAAAAAAAAAAAAAA85420,AAAAAAAAAAA33576} + 15 | {17,14,16,63,67} | {AA6416,AAAAAAAAAA646,AAAAA95309} + 19 | {52,82,17,74,23,46,69,51,75} | {AAAAAAAAAAAAA73084,AAAAA75968,AAAAAAAAAAAAAAAA14047,AAAAAAA80240,AAAAAAAAAAAAAAAAAAA1205,A68938} + 53 | {38,17} | {AAAAAAAAAAA21658} + 65 | {61,5,76,59,17} | {AAAAAA99807,AAAAA64741,AAAAAAAAAAA53908,AA21643,AAAAAAAAA10012} + 77 | {97,15,32,17,55,59,18,37,50,39} | {AAAAAAAAAAAA67946,AAAAAA54032,AAAAAAAA81587,55847,AAAAAAAAAAAAAA28620,AAAAAAAAAAAAAAAAA43052,AAAAAA75463,AAAA49534,AAAAAAAA44066} + 89 | {40,32,17,6,30,88} | {AA44673,AAAAAAAAAAA6119,AAAAAAAAAAAAAAAA23657,AAAAAAAAAAAAAAAAAA47955,AAAAAAAAAAAAAAAA33598,AAAAAAAAAAA33576,AA44673} +(8 rows) + +SELECT * FROM array_op_test WHERE i @>> 17 ORDER BY seqno; + seqno | i | t +-------+---------------------------------+------------------------------------------------------------------------------------------------------------------------------------ + 6 | {39,35,5,94,17,92,60,32} | {AAAAAAAAAAAAAAA35875,AAAAAAAAAAAAAAAA23657} + 12 | {17,99,18,52,91,72,0,43,96,23} | {AAAAA33250,AAAAAAAAAAAAAAAAAAA85420,AAAAAAAAAAA33576} + 15 | {17,14,16,63,67} | {AA6416,AAAAAAAAAA646,AAAAA95309} + 19 | {52,82,17,74,23,46,69,51,75} | {AAAAAAAAAAAAA73084,AAAAA75968,AAAAAAAAAAAAAAAA14047,AAAAAAA80240,AAAAAAAAAAAAAAAAAAA1205,A68938} + 53 | {38,17} | {AAAAAAAAAAA21658} + 65 | {61,5,76,59,17} | {AAAAAA99807,AAAAA64741,AAAAAAAAAAA53908,AA21643,AAAAAAAAA10012} + 77 | {97,15,32,17,55,59,18,37,50,39} | {AAAAAAAAAAAA67946,AAAAAA54032,AAAAAAAA81587,55847,AAAAAAAAAAAAAA28620,AAAAAAAAAAAAAAAAA43052,AAAAAA75463,AAAA49534,AAAAAAAA44066} + 89 | {40,32,17,6,30,88} | {AA44673,AAAAAAAAAAA6119,AAAAAAAAAAAAAAAA23657,AAAAAAAAAAAAAAAAAA47955,AAAAAAAAAAAAAAAA33598,AAAAAAAAAAA33576,AA44673} +(8 rows) + SELECT * FROM array_op_test WHERE i && '{17}' ORDER BY seqno; seqno | i | t -------+---------------------------------+------------------------------------------------------------------------------------------------------------------------------------ @@ -963,6 +1011,16 @@ SELECT * FROM array_op_test WHERE i @> '{NULL}' ORDER BY seqno; -------+---+--- (0 rows) +SELECT * FROM array_op_test WHERE i @>> NULL ORDER BY seqno; + seqno | i | t +-------+---+--- +(0 rows) + +SELECT * FROM array_op_test WHERE NULL <<@ i ORDER BY seqno; + seqno | i | t +-------+---+--- +(0 rows) + SELECT * FROM array_op_test WHERE i && '{NULL}' ORDER BY seqno; seqno | i | t -------+---+--- @@ -983,6 +1041,24 @@ SELECT * FROM array_op_test WHERE t @> '{AAAAAAAA72908}' ORDER BY seqno; 79 | {45} | {AAAAAAAAAA646,AAAAAAAAAAAAAAAAAAA70415,AAAAAA43678,AAAAAAAA72908} (4 rows) +SELECT * FROM array_op_test WHERE t @>> 'AAAAAAAA72908' ORDER BY seqno; + seqno | i | t +-------+-----------------------+-------------------------------------------------------------------------------------------------------------------------------------------- + 22 | {11,6,56,62,53,30} | {AAAAAAAA72908} + 45 | {99,45} | {AAAAAAAA72908,AAAAAAAAAAAAAAAAAAA17075,AA88409,AAAAAAAAAAAAAAAAAA36842,AAAAAAA48038,AAAAAAAAAAAAAA10611} + 72 | {22,1,16,78,20,91,83} | {47735,AAAAAAA56483,AAAAAAAAAAAAA93788,AA42406,AAAAAAAAAAAAA73084,AAAAAAAA72908,AAAAAAAAAAAAAAAAAA61286,AAAAA66674,AAAAAAAAAAAAAAAAA50407} + 79 | {45} | {AAAAAAAAAA646,AAAAAAAAAAAAAAAAAAA70415,AAAAAA43678,AAAAAAAA72908} +(4 rows) + +SELECT * FROM array_op_test WHERE 'AAAAAAAA72908' <<@ t ORDER BY seqno; + seqno | i | t +-------+-----------------------+-------------------------------------------------------------------------------------------------------------------------------------------- + 22 | {11,6,56,62,53,30} | {AAAAAAAA72908} + 45 | {99,45} | {AAAAAAAA72908,AAAAAAAAAAAAAAAAAAA17075,AA88409,AAAAAAAAAAAAAAAAAA36842,AAAAAAA48038,AAAAAAAAAAAAAA10611} + 72 | {22,1,16,78,20,91,83} | {47735,AAAAAAA56483,AAAAAAAAAAAAA93788,AA42406,AAAAAAAAAAAAA73084,AAAAAAAA72908,AAAAAAAAAAAAAAAAAA61286,AAAAA66674,AAAAAAAAAAAAAAAAA50407} + 79 | {45} | {AAAAAAAAAA646,AAAAAAAAAAAAAAAAAAA70415,AAAAAA43678,AAAAAAAA72908} +(4 rows) + SELECT * FROM array_op_test WHERE t && '{AAAAAAAA72908}' ORDER BY seqno; seqno | i | t -------+-----------------------+-------------------------------------------------------------------------------------------------------------------------------------------- @@ -1000,6 +1076,22 @@ SELECT * FROM array_op_test WHERE t @> '{AAAAAAAAAA646}' ORDER BY seqno; 96 | {23,97,43} | {AAAAAAAAAA646,A87088} (3 rows) +SELECT * FROM array_op_test WHERE t @>> 'AAAAAAAAAA646' ORDER BY seqno; + seqno | i | t +-------+------------------+-------------------------------------------------------------------- + 15 | {17,14,16,63,67} | {AA6416,AAAAAAAAAA646,AAAAA95309} + 79 | {45} | {AAAAAAAAAA646,AAAAAAAAAAAAAAAAAAA70415,AAAAAA43678,AAAAAAAA72908} + 96 | {23,97,43} | {AAAAAAAAAA646,A87088} +(3 rows) + +SELECT * FROM array_op_test WHERE 'AAAAAAAAAA646' <<@ t ORDER BY seqno; + seqno | i | t +-------+------------------+-------------------------------------------------------------------- + 15 | {17,14,16,63,67} | {AA6416,AAAAAAAAAA646,AAAAA95309} + 79 | {45} | {AAAAAAAAAA646,AAAAAAAAAAAAAAAAAAA70415,AAAAAA43678,AAAAAAAA72908} + 96 | {23,97,43} | {AAAAAAAAAA646,A87088} +(3 rows) + SELECT * FROM array_op_test WHERE t && '{AAAAAAAAAA646}' ORDER BY seqno; seqno | i | t -------+------------------+-------------------------------------------------------------------- diff --git a/src/test/regress/expected/gin.out b/src/test/regress/expected/gin.out index 6402e89c7f..698d322e14 100644 --- a/src/test/regress/expected/gin.out +++ b/src/test/regress/expected/gin.out @@ -53,6 +53,40 @@ select count(*) from gin_test_tbl where i @> array[1, 999]; 3 (1 row) +explain (costs off) +select count(*) from gin_test_tbl where i @>> 1; + QUERY PLAN +----------------------------------------------- + Aggregate + -> Bitmap Heap Scan on gin_test_tbl + Recheck Cond: (i @>> 1) + -> Bitmap Index Scan on gin_test_idx + Index Cond: (i @>> 1) +(5 rows) + +explain (costs off) +select count(*) from gin_test_tbl where i @>> 999; + QUERY PLAN +----------------------------------------------- + Aggregate + -> Bitmap Heap Scan on gin_test_tbl + Recheck Cond: (i @>> 999) + -> Bitmap Index Scan on gin_test_idx + Index Cond: (i @>> 999) +(5 rows) + +select count(*) from gin_test_tbl where i @>> 1; + count +------- + 3 +(1 row) + +select count(*) from gin_test_tbl where i @>> 999; + count +------- + 0 +(1 row) + -- Very weak test for gin_fuzzy_search_limit set gin_fuzzy_search_limit = 1000; explain (costs off) diff --git a/src/test/regress/expected/opr_sanity.out b/src/test/regress/expected/opr_sanity.out index 254ca06d3d..5de5ab6d13 100644 --- a/src/test/regress/expected/opr_sanity.out +++ b/src/test/regress/expected/opr_sanity.out @@ -1173,6 +1173,7 @@ ORDER BY 1, 2; <-> | <-> << | >> <<= | >>= + <<@ | @>> <= | >= <> | <> <@ | @> @@ -1188,7 +1189,7 @@ ORDER BY 1, 2; ~<=~ | ~>=~ ~<~ | ~>~ ~= | ~= -(29 rows) +(30 rows) -- Likewise for negator pairs. SELECT DISTINCT o1.oprname AS op1, o2.oprname AS op2 @@ -2029,6 +2030,7 @@ ORDER BY 1, 2, 3; 2742 | 2 | @@@ 2742 | 3 | <@ 2742 | 4 | = + 2742 | 5 | @>> 2742 | 7 | @> 2742 | 9 | ? 2742 | 10 | ?| @@ -2100,7 +2102,7 @@ ORDER BY 1, 2, 3; 4000 | 28 | ^@ 4000 | 29 | <^ 4000 | 30 | >^ -(123 rows) +(124 rows) -- Check that all opclass search operators have selectivity estimators. -- This is not absolutely required, but it seems a reasonable thing diff --git a/src/test/regress/sql/arrays.sql b/src/test/regress/sql/arrays.sql index c40619a8d5..b5eec945f7 100644 --- a/src/test/regress/sql/arrays.sql +++ b/src/test/regress/sql/arrays.sql @@ -319,8 +319,12 @@ SELECT 0 || ARRAY[1,2] || 3 AS "{0,1,2,3}"; SELECT ARRAY[1.1] || ARRAY[2,3,4]; SELECT * FROM array_op_test WHERE i @> '{32}' ORDER BY seqno; +SELECT * FROM array_op_test WHERE i @>> 32 ORDER BY seqno; +SELECT * FROM array_op_test WHERE 32 <<@ i ORDER BY seqno; SELECT * FROM array_op_test WHERE i && '{32}' ORDER BY seqno; SELECT * FROM array_op_test WHERE i @> '{17}' ORDER BY seqno; +SELECT * FROM array_op_test WHERE 17 <<@ i ORDER BY seqno; +SELECT * FROM array_op_test WHERE i @>> 17 ORDER BY seqno; SELECT * FROM array_op_test WHERE i && '{17}' ORDER BY seqno; SELECT * FROM array_op_test WHERE i @> '{32,17}' ORDER BY seqno; SELECT * FROM array_op_test WHERE i && '{32,17}' ORDER BY seqno; @@ -331,12 +335,18 @@ SELECT * FROM array_op_test WHERE i && '{}' ORDER BY seqno; SELECT * FROM array_op_test WHERE i <@ '{}' ORDER BY seqno; SELECT * FROM array_op_test WHERE i = '{NULL}' ORDER BY seqno; SELECT * FROM array_op_test WHERE i @> '{NULL}' ORDER BY seqno; +SELECT * FROM array_op_test WHERE i @>> NULL ORDER BY seqno; +SELECT * FROM array_op_test WHERE NULL <<@ i ORDER BY seqno; SELECT * FROM array_op_test WHERE i && '{NULL}' ORDER BY seqno; SELECT * FROM array_op_test WHERE i <@ '{NULL}' ORDER BY seqno; SELECT * FROM array_op_test WHERE t @> '{AAAAAAAA72908}' ORDER BY seqno; +SELECT * FROM array_op_test WHERE t @>> 'AAAAAAAA72908' ORDER BY seqno; +SELECT * FROM array_op_test WHERE 'AAAAAAAA72908' <<@ t ORDER BY seqno; SELECT * FROM array_op_test WHERE t && '{AAAAAAAA72908}' ORDER BY seqno; SELECT * FROM array_op_test WHERE t @> '{AAAAAAAAAA646}' ORDER BY seqno; +SELECT * FROM array_op_test WHERE t @>> 'AAAAAAAAAA646' ORDER BY seqno; +SELECT * FROM array_op_test WHERE 'AAAAAAAAAA646' <<@ t ORDER BY seqno; SELECT * FROM array_op_test WHERE t && '{AAAAAAAAAA646}' ORDER BY seqno; SELECT * FROM array_op_test WHERE t @> '{AAAAAAAA72908,AAAAAAAAAA646}' ORDER BY seqno; SELECT * FROM array_op_test WHERE t && '{AAAAAAAA72908,AAAAAAAAAA646}' ORDER BY seqno; diff --git a/src/test/regress/sql/gin.sql b/src/test/regress/sql/gin.sql index 5194afcc1f..c9b40903c6 100644 --- a/src/test/regress/sql/gin.sql +++ b/src/test/regress/sql/gin.sql @@ -41,6 +41,14 @@ select count(*) from gin_test_tbl where i @> array[1, 999]; select count(*) from gin_test_tbl where i @> array[1, 999]; +explain (costs off) +select count(*) from gin_test_tbl where i @>> 1; +explain (costs off) +select count(*) from gin_test_tbl where i @>> 999; + +select count(*) from gin_test_tbl where i @>> 1; +select count(*) from gin_test_tbl where i @>> 999; + -- Very weak test for gin_fuzzy_search_limit set gin_fuzzy_search_limit = 1000; -- 2.17.0 --opg8F0UgoHELSI+9 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0002-fix.patch" ^ permalink raw reply [nested|flat] 2+ messages in thread
* [PATCH v40 3/4] Serialize decoded tuples without flattening. @ 2026-03-09 10:35 Antonin Houska <[email protected]> 0 siblings, 0 replies; 2+ messages in thread From: Antonin Houska @ 2026-03-09 10:35 UTC (permalink / raw) So far we used to flatten the tuples produced by logical decoding in order to write them to a temporary file. That means that all the TOASTed attributes ended up in a single chunk of memory. The problem is that even if individual attributes do fit into the maximum chunk size (currently 1 GB), the whole tuple is not guaranteed to fit. The flattening might be a problem in the future anyway, since there's an effort to restrict the impact of REPACK (CONCURRENTLY) on the VACUUM xmin horizon. However, as the flattening assumes scans of the TOAST relaation, the less we restrict VACUUM, the more likely it is that the TOAST attributes needed for the flattening won't be available at decoding time. (This patch also removes unnecessary field 'tupdesc' from RepackDecodingState and allocates it in the appropriate memory context.) --- src/backend/commands/cluster.c | 264 +++++++++++++++--- .../pgoutput_repack/pgoutput_repack.c | 122 ++++++-- src/include/commands/cluster.h | 6 +- .../expected/repack_toast.out | 1 + .../injection_points/specs/repack_toast.spec | 7 + 5 files changed, 331 insertions(+), 69 deletions(-) diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index fe419ef808e..77e301b7c63 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -33,6 +33,7 @@ #include "postgres.h" #include "access/amapi.h" +#include "access/detoast.h" #include "access/heapam.h" #include "access/multixact.h" #include "access/relscan.h" @@ -285,6 +286,10 @@ static void apply_concurrent_update(Relation rel, HeapTuple tup, IndexInsertState *iistate, TupleTableSlot *index_slot); static void apply_concurrent_delete(Relation rel, HeapTuple tup_target); +static HeapTuple restore_tuple(BufFile *file, Relation relation, + MemoryContext cxt); +static HeapTuple adjust_toast_pointers(Relation relation, HeapTuple tup_dst, + HeapTuple tup_src, MemoryContext cxt); static HeapTuple find_target_tuple(Relation rel, ChangeDest *dest, HeapTuple tup_key, TupleTableSlot *ident_slot); @@ -2622,6 +2627,7 @@ setup_logical_decoding(Oid relid) LogicalDecodingContext *ctx; NameData slotname; RepackDecodingState *dstate; + MemoryContext oldcxt; /* * REPACK CONCURRENTLY is not allowed in a transaction block, so this @@ -2672,14 +2678,6 @@ setup_logical_decoding(Oid relid) */ ctx->reader->routine.page_read = read_local_xlog_page_no_wait; - /* - * read_local_xlog_page_no_wait() needs to be able to indicate the end of - * WAL. - */ - ctx->reader->private_data = MemoryContextAllocZero(ctx->context, - sizeof(ReadLocalXLogPageNoWaitPrivate)); - - /* Some WAL records should have been read. */ Assert(ctx->reader->EndRecPtr != InvalidXLogRecPtr); @@ -2690,18 +2688,27 @@ setup_logical_decoding(Oid relid) XLByteToSeg(ctx->reader->EndRecPtr, repack_current_segment, wal_segment_size); - dstate = palloc0_object(RepackDecodingState); - dstate->relid = relid; + /* Our private state belongs to the decoding context. */ + oldcxt = MemoryContextSwitchTo(ctx->context); /* - * Tuple descriptor may be needed to flatten a tuple before we write it to - * a file. A copy is needed because the decoding worker invalidates system - * caches before it starts to do the actual work. + * read_local_xlog_page_no_wait() needs to be able to indicate the end of + * WAL. */ - rel = table_open(relid, AccessShareLock); - dstate->tupdesc = CreateTupleDescCopy(RelationGetDescr(rel)); + ctx->reader->private_data = palloc0_object(ReadLocalXLogPageNoWaitPrivate); + dstate = palloc0_object(RepackDecodingState); + MemoryContextSwitchTo(oldcxt); + +#ifdef USE_ASSERT_CHECKING + dstate->relid = relid; +#endif + + dstate->change_cxt = AllocSetContextCreate(ctx->context, + "REPACK - change", + ALLOCSET_DEFAULT_SIZES); /* Avoid logical decoding of other relations. */ + rel = table_open(relid, AccessShareLock); repacked_rel_locator = rel->rd_locator; toastrelid = rel->rd_rel->reltoastrelid; if (OidIsValid(toastrelid)) @@ -2870,10 +2877,10 @@ static void apply_concurrent_changes(BufFile *file, ChangeDest *dest) { char kind; - uint32 t_len; Relation rel = dest->rel; TupleTableSlot *index_slot, *ident_slot; + MemoryContext apply_cxt; HeapTuple tup_old = NULL; /* TupleTableSlot is needed to pass the tuple to ExecInsertIndexTuples(). */ @@ -2883,6 +2890,14 @@ apply_concurrent_changes(BufFile *file, ChangeDest *dest) /* A slot to fetch tuples from identity index. */ ident_slot = table_slot_create(rel, NULL); + /* + * Use a separate memory context for the tuples and any memory needed to + * process them. + */ + apply_cxt = AllocSetContextCreate(TopTransactionContext, + "REPACK - apply", + ALLOCSET_DEFAULT_SIZES); + while (true) { size_t nread; @@ -2897,14 +2912,7 @@ apply_concurrent_changes(BufFile *file, ChangeDest *dest) break; /* Read the tuple. */ - BufFileReadExact(file, &t_len, sizeof(t_len)); - tup = (HeapTuple) palloc(HEAPTUPLESIZE + t_len); - tup->t_data = (HeapTupleHeader) ((char *) tup + HEAPTUPLESIZE); - BufFileReadExact(file, tup->t_data, t_len); - tup->t_len = t_len; - ItemPointerSetInvalid(&tup->t_self); - tup->t_tableOid = RelationGetRelid(dest->rel); - + tup = restore_tuple(file, rel, apply_cxt); if (kind == CHANGE_UPDATE_OLD) { Assert(tup_old == NULL); @@ -2916,7 +2924,7 @@ apply_concurrent_changes(BufFile *file, ChangeDest *dest) apply_concurrent_insert(rel, tup, dest->iistate, index_slot); - pfree(tup); + MemoryContextReset(apply_cxt); } else if (kind == CHANGE_UPDATE_NEW || kind == CHANGE_DELETE) { @@ -2940,18 +2948,28 @@ apply_concurrent_changes(BufFile *file, ChangeDest *dest) elog(ERROR, "failed to find target tuple"); if (kind == CHANGE_UPDATE_NEW) + { + /* + * If 'tup' contains TOAST pointers, they point to the old + * relation's toast. Copy the corresponding TOAST pointers for + * the new relation from the existing tuple. (The fact that we + * received a TOAST pointer here implies that the attribute + * hasn't changed.) + */ + if (HeapTupleHasExternal(tup)) + tup = adjust_toast_pointers(rel, tup, tup_exist, + apply_cxt); + apply_concurrent_update(rel, tup, tup_exist, dest->iistate, index_slot); + } else apply_concurrent_delete(rel, tup_exist); if (tup_old != NULL) - { - pfree(tup_old); tup_old = NULL; - } - pfree(tup); + MemoryContextReset(apply_cxt); } else elog(ERROR, "unrecognized kind of change: %d", kind); @@ -2968,6 +2986,7 @@ apply_concurrent_changes(BufFile *file, ChangeDest *dest) } /* Cleanup. */ + MemoryContextDelete(apply_cxt); ExecDropSingleTupleTableSlot(index_slot); ExecDropSingleTupleTableSlot(ident_slot); } @@ -3077,6 +3096,186 @@ apply_concurrent_delete(Relation rel, HeapTuple tup_target) pgstat_progress_incr_param(PROGRESS_REPACK_HEAP_TUPLES_DELETED, 1); } +/* + * Read tuple from file and store it in given memory context. + * + * External attributes are stored in separate memory chunks, in order to avoid + * exceeding MaxAllocSize - that could happen if the individual attributes are + * smaller than MaxAllocSize but the whole tuple is bigger. + */ +static HeapTuple +restore_tuple(BufFile *file, Relation relation, MemoryContext cxt) +{ + uint32 t_len; + HeapTuple tup; + MemoryContext oldcxt; + uint32 natt_ext; + List *attrs_ext = NIL; + + oldcxt = MemoryContextSwitchTo(cxt); + + /* First, read the external attributes if there are some. */ + BufFileReadExact(file, &natt_ext, sizeof(natt_ext)); + for (int i = 0; i < natt_ext; i++) + { + varlena varhdr; + char *ext_val; + Size ext_val_size; + + /* + * Read the header into properly aligned memory before accessing it. + */ + BufFileReadExact(file, &varhdr, VARHDRSZ); + ext_val_size = VARSIZE_ANY(&varhdr); + ext_val = palloc0(ext_val_size); + memcpy(ext_val, &varhdr, VARHDRSZ); + BufFileReadExact(file, ext_val + VARHDRSZ, ext_val_size - VARHDRSZ); + + attrs_ext = lappend(attrs_ext, ext_val); + } + + /* Read the tuple. */ + BufFileReadExact(file, &t_len, sizeof(t_len)); + tup = (HeapTuple) palloc(HEAPTUPLESIZE + t_len); + tup->t_data = (HeapTupleHeader) ((char *) tup + HEAPTUPLESIZE); + BufFileReadExact(file, tup->t_data, t_len); + tup->t_len = t_len; + ItemPointerSetInvalid(&tup->t_self); + tup->t_tableOid = RelationGetRelid(relation); + + /* + * If there are "external indirect" attributes, make the tuple reference + * them. + */ + if (natt_ext > 0) + { + TupleDesc desc; + Datum *attrs; + bool *isnull; + ListCell *lc; + + desc = RelationGetDescr(relation); + attrs = palloc0_array(Datum, desc->natts); + isnull = palloc0_array(bool, desc->natts); + + heap_deform_tuple(tup, desc, attrs, isnull); + + lc = list_head(attrs_ext); + for (int i = 0; i < desc->natts; i++) + { + CompactAttribute *attr = TupleDescCompactAttr(desc, i); + varlena *varlena_pointer, + *new_datum; + varatt_indirect redirect_pointer; + + /* Find the external attributes, like in store_change(). */ + if (attr->attisdropped) + continue; + if (attr->attlen != -1) + continue; + if (isnull[i]) + continue; + varlena_pointer = (varlena *) DatumGetPointer(attrs[i]); + if (!VARATT_IS_EXTERNAL_INDIRECT(varlena_pointer)) + continue; + + /* Update the attribute. */ + memset(&redirect_pointer, 0, sizeof(redirect_pointer)); + redirect_pointer.pointer = lfirst(lc); + + new_datum = (varlena *) palloc0(INDIRECT_POINTER_SIZE); + SET_VARTAG_EXTERNAL(new_datum, VARTAG_INDIRECT); + memcpy(VARDATA_EXTERNAL(new_datum), &redirect_pointer, + sizeof(redirect_pointer)); + + attrs[i] = PointerGetDatum(new_datum); + + lc = lnext(attrs_ext, lc); + } + Assert(lc == NULL); + + tup = heap_form_tuple(desc, attrs, isnull); + } + + MemoryContextSwitchTo(oldcxt); + + return tup; +} + +/* + * Copy TOAST pointers from 'tup_src' to 'tup_dst' and return the modified + * 'tup_dst'. Use 'cxt' for all memory allocations. + */ +static HeapTuple +adjust_toast_pointers(Relation relation, HeapTuple tup_dst, + HeapTuple tup_src, MemoryContext cxt) +{ + TupleDesc desc; + Datum *attrs_dst; + bool *isnull_dst; + Datum *attrs_src = NULL; + bool *isnull_src = NULL; + MemoryContext oldcxt; + HeapTuple result; + + oldcxt = MemoryContextSwitchTo(cxt); + + desc = RelationGetDescr(relation); + attrs_dst = palloc0_array(Datum, desc->natts); + isnull_dst = palloc0_array(bool, desc->natts); + + heap_deform_tuple(tup_dst, desc, attrs_dst, isnull_dst); + + for (int i = 0; i < desc->natts; i++) + { + CompactAttribute *attr = TupleDescCompactAttr(desc, i); + varlena *varlena_dst, + *varlena_src; + + if (attr->attisdropped) + continue; + if (attr->attlen != -1) + continue; + if (isnull_dst[i]) + continue; + + varlena_dst = (varlena *) DatumGetPointer(attrs_dst[i]); + if (!VARATT_IS_EXTERNAL_ONDISK(varlena_dst)) + continue; + + /* + * The first TOAST pointer? The caller can easily check if the tuple + * has external attributes(s) in general, but is not supposed to check + * specifically for TOAST. + */ + if (attrs_src == NULL) + { + attrs_src = palloc0_array(Datum, desc->natts); + isnull_src = palloc0_array(bool, desc->natts); + + heap_deform_tuple(tup_src, desc, attrs_src, isnull_src); + } + + varlena_src = (varlena *) DatumGetPointer(attrs_src[i]); + + /* + * Logical decoding only uses TOAST pointer for attributes that + * haven't changed. + */ + Assert(!isnull_src[i]); + Assert(VARATT_IS_EXTERNAL_ONDISK(varlena_src)); + + /* Copy the pointer. */ + memcpy(VARDATA_EXTERNAL(varlena_dst), VARDATA_EXTERNAL(varlena_src), + sizeof(varatt_external)); + } + result = heap_form_tuple(desc, attrs_dst, isnull_dst); + + MemoryContextSwitchTo(oldcxt); + + return result; +} + /* * Find the tuple to be updated or deleted. * @@ -3299,15 +3498,8 @@ free_index_insert_state(IndexInsertState *iistate) static void cleanup_logical_decoding(LogicalDecodingContext *ctx) { - RepackDecodingState *dstate; - - dstate = (RepackDecodingState *) ctx->output_writer_private; - - FreeTupleDesc(dstate->tupdesc); FreeDecodingContext(ctx); - ReplicationSlotDropAcquired(); - pfree(dstate); } /* diff --git a/src/backend/replication/pgoutput_repack/pgoutput_repack.c b/src/backend/replication/pgoutput_repack/pgoutput_repack.c index d0c464a98d5..707940c1127 100644 --- a/src/backend/replication/pgoutput_repack/pgoutput_repack.c +++ b/src/backend/replication/pgoutput_repack/pgoutput_repack.c @@ -12,7 +12,7 @@ */ #include "postgres.h" -#include "access/heaptoast.h" +#include "access/detoast.h" #include "commands/cluster.h" #include "replication/snapbuild.h" #include "utils/memutils.h" @@ -28,7 +28,7 @@ static void plugin_commit_txn(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, XLogRecPtr commit_lsn); static void plugin_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, Relation rel, ReorderBufferChange *change); -static void store_change(LogicalDecodingContext *ctx, +static void store_change(LogicalDecodingContext *ctx, Relation relation, ConcurrentChangeKind kind, HeapTuple tuple); void @@ -97,9 +97,8 @@ plugin_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, dstate = (RepackDecodingState *) ctx->output_writer_private; - /* Only interested in one particular relation. */ - if (relation->rd_id != dstate->relid) - return; + /* Changes of other relation should not have been decoded. */ + Assert(RelationGetRelid(relation) == dstate->relid); /* Decode entry depending on its type */ switch (change->action) @@ -117,7 +116,7 @@ plugin_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, if (newtuple == NULL) elog(ERROR, "incomplete insert info."); - store_change(ctx, CHANGE_INSERT, newtuple); + store_change(ctx, relation, CHANGE_INSERT, newtuple); } break; case REORDER_BUFFER_CHANGE_UPDATE: @@ -132,9 +131,9 @@ plugin_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, elog(ERROR, "incomplete update info."); if (oldtuple != NULL) - store_change(ctx, CHANGE_UPDATE_OLD, oldtuple); + store_change(ctx, relation, CHANGE_UPDATE_OLD, oldtuple); - store_change(ctx, CHANGE_UPDATE_NEW, newtuple); + store_change(ctx, relation, CHANGE_UPDATE_NEW, newtuple); } break; case REORDER_BUFFER_CHANGE_DELETE: @@ -146,7 +145,7 @@ plugin_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, if (oldtuple == NULL) elog(ERROR, "incomplete delete info."); - store_change(ctx, CHANGE_DELETE, oldtuple); + store_change(ctx, relation, CHANGE_DELETE, oldtuple); } break; default: @@ -164,39 +163,100 @@ plugin_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, /* Store concurrent data change. */ static void -store_change(LogicalDecodingContext *ctx, ConcurrentChangeKind kind, - HeapTuple tuple) +store_change(LogicalDecodingContext *ctx, Relation relation, + ConcurrentChangeKind kind, HeapTuple tuple) { RepackDecodingState *dstate; + MemoryContext oldcxt; + BufFile *file; char kind_byte = (char) kind; - bool flattened = false; + List *attrs_ext = NIL; + uint32 natt_ext; dstate = (RepackDecodingState *) ctx->output_writer_private; + file = dstate->file; /* Store the change kind. */ - BufFileWrite(dstate->file, &kind_byte, 1); + BufFileWrite(file, &kind_byte, 1); + + /* Make sure there are no memory leaks. */ + oldcxt = MemoryContextSwitchTo(dstate->change_cxt); /* - * ReorderBufferCommit() stores the TOAST chunks in its private memory - * context and frees them after having called apply_change(). Therefore - * we need flat copy (including TOAST) that we eventually copy into the - * memory context which is available to decode_concurrent_changes(). + * If the tuple contains "external indirect" attributes, we need to write + * the contents to the file because we have no control over that memory. */ if (HeapTupleHasExternal(tuple)) { - /* - * toast_flatten_tuple_to_datum() might be more convenient but we - * don't want the decompression it does. - */ - tuple = toast_flatten_tuple(tuple, dstate->tupdesc); - flattened = true; - } - /* Store the tuple size ... */ - BufFileWrite(dstate->file, &tuple->t_len, sizeof(tuple->t_len)); - /* ... and the tuple itself. */ - BufFileWrite(dstate->file, tuple->t_data, tuple->t_len); + TupleDesc desc; + Datum *attrs; + bool *isnull; - /* Free the flat copy if created above. */ - if (flattened) - pfree(tuple); + desc = RelationGetDescr(relation); + attrs = palloc0_array(Datum, desc->natts); + isnull = palloc0_array(bool, desc->natts); + + heap_deform_tuple(tuple, desc, attrs, isnull); + + /* First, gather and count the "external indirect" attributes. */ + for (int i = 0; i < desc->natts; i++) + { + CompactAttribute *attr = TupleDescCompactAttr(desc, i); + varlena *varlena_pointer; + + if (attr->attisdropped) + continue; + + /* not a varlena datatype */ + if (attr->attlen != -1) + continue; + + /* no data */ + if (isnull[i]) + continue; + + /* ok, we know we have a toast datum */ + varlena_pointer = (varlena *) DatumGetPointer(attrs[i]); + + if (!VARATT_IS_EXTERNAL(varlena_pointer)) + continue; + + if (VARATT_IS_EXTERNAL_INDIRECT(varlena_pointer)) + attrs_ext = lappend(attrs_ext, varlena_pointer); + else + { + /* + * Logical decoding should not produce "external expanded" + * attributes (those actually should never appear on disk), so + * only TOASTed attribute can be seen here. + */ + Assert(VARATT_IS_EXTERNAL_ONDISK(varlena_pointer)); + } + } + natt_ext = list_length(attrs_ext); + } + else + natt_ext = 0; + + /* Write the number of external attributes. */ + BufFileWrite(file, &natt_ext, sizeof(natt_ext)); + /* ... and the attributes themselves, if there are some. */ + foreach_ptr(varlena, attr_val, attrs_ext) + { + varlena *ext_val; + Size ext_val_size; + + ext_val = detoast_external_attr(attr_val); + ext_val_size = VARSIZE_ANY(ext_val); + BufFileWrite(file, ext_val, ext_val_size); + } + + /* Finally write the tuple size ... */ + BufFileWrite(file, &tuple->t_len, sizeof(tuple->t_len)); + /* ... and the tuple itself. */ + BufFileWrite(file, tuple->t_data, tuple->t_len); + + /* Cleanup. */ + MemoryContextSwitchTo(oldcxt); + MemoryContextReset(dstate->change_cxt); } diff --git a/src/include/commands/cluster.h b/src/include/commands/cluster.h index 1b05d5d418b..34b7208de9b 100644 --- a/src/include/commands/cluster.h +++ b/src/include/commands/cluster.h @@ -65,11 +65,13 @@ typedef enum */ typedef struct RepackDecodingState { +#ifdef USE_ASSERT_CHECKING /* The relation whose changes we're decoding. */ Oid relid; +#endif - /* Tuple descriptor of the relation being processed. */ - TupleDesc tupdesc; + /* Per-change memory context. */ + MemoryContext change_cxt; /* The current output file. */ BufFile *file; diff --git a/src/test/modules/injection_points/expected/repack_toast.out b/src/test/modules/injection_points/expected/repack_toast.out index 4f866a74e32..b56dde134f8 100644 --- a/src/test/modules/injection_points/expected/repack_toast.out +++ b/src/test/modules/injection_points/expected/repack_toast.out @@ -13,6 +13,7 @@ step change: UPDATE repack_test SET j=get_long_string() where i=2; DELETE FROM repack_test WHERE i=3; INSERT INTO repack_test(i, j) VALUES (4, get_long_string()); + UPDATE repack_test SET i=3 where i=1; step check2: INSERT INTO relfilenodes(node) diff --git a/src/test/modules/injection_points/specs/repack_toast.spec b/src/test/modules/injection_points/specs/repack_toast.spec index b48abf21450..b878b198971 100644 --- a/src/test/modules/injection_points/specs/repack_toast.spec +++ b/src/test/modules/injection_points/specs/repack_toast.spec @@ -74,10 +74,17 @@ teardown session s2 step change +# Separately test UPDATE of both plain ("i") and TOASTed ("j") attribute. In +# the first case, the new tuple we get from reorderbuffer.c contains "j" as a +# TOAST pointer, which we need to update so it points to the new heap. In the +# latter case, we receive "j" as "external indirect" value - here we test that +# the decoding worker writes the tuple to a file correctly and that the +# backend executing REPACK manages to restore it. { UPDATE repack_test SET j=get_long_string() where i=2; DELETE FROM repack_test WHERE i=3; INSERT INTO repack_test(i, j) VALUES (4, get_long_string()); + UPDATE repack_test SET i=3 where i=1; } # Check the table from the perspective of s2. step check2 -- 2.47.3 --k4rn5ob3nx2ufkij Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v40-0004-Use-BulkInsertState-when-copying-data-to-the-new.patch" ^ permalink raw reply [nested|flat] 2+ messages in thread
end of thread, other threads:[~2026-03-09 10:35 UTC | newest] Thread overview: 2+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2021-03-13 10:01 [PATCH 1/3] anyarray_anyelement_operators Mark Rofail <[email protected]> 2026-03-09 10:35 [PATCH v40 3/4] Serialize decoded tuples without flattening. Antonin Houska <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox