public inbox for [email protected]
help / color / mirror / Atom feed[PATCH v7 2/3] Infrastructure for asynchronous execution
12+ messages / 7 participants
[nested] [flat]
* [PATCH v7 2/3] Infrastructure for asynchronous execution
@ 2018-05-15 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 12+ messages in thread
From: Kyotaro Horiguchi @ 2018-05-15 11:21 UTC (permalink / raw)
This patch add an infrastructure for asynchronous execution. As a PoC
this makes only Append capable to handle asynchronously executable
subnodes.
---
src/backend/commands/explain.c | 17 ++
src/backend/executor/Makefile | 1 +
src/backend/executor/execAsync.c | 152 +++++++++++
src/backend/executor/nodeAppend.c | 342 ++++++++++++++++++++----
src/backend/executor/nodeForeignscan.c | 21 ++
src/backend/nodes/bitmapset.c | 72 +++++
src/backend/nodes/copyfuncs.c | 3 +
src/backend/nodes/outfuncs.c | 3 +
src/backend/nodes/readfuncs.c | 3 +
src/backend/optimizer/path/allpaths.c | 24 ++
src/backend/optimizer/path/costsize.c | 55 +++-
src/backend/optimizer/plan/createplan.c | 45 +++-
src/backend/postmaster/pgstat.c | 3 +
src/backend/utils/adt/ruleutils.c | 8 +-
src/backend/utils/resowner/resowner.c | 4 +-
src/include/executor/execAsync.h | 22 ++
src/include/executor/executor.h | 1 +
src/include/executor/nodeForeignscan.h | 3 +
src/include/foreign/fdwapi.h | 11 +
src/include/nodes/bitmapset.h | 1 +
src/include/nodes/execnodes.h | 23 +-
src/include/nodes/plannodes.h | 9 +
src/include/optimizer/paths.h | 2 +
src/include/pgstat.h | 3 +-
24 files changed, 756 insertions(+), 72 deletions(-)
create mode 100644 src/backend/executor/execAsync.c
create mode 100644 src/include/executor/execAsync.h
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index c98c9b5547..097355f6f9 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -86,6 +86,7 @@ static void show_incremental_sort_keys(IncrementalSortState *incrsortstate,
List *ancestors, ExplainState *es);
static void show_merge_append_keys(MergeAppendState *mstate, List *ancestors,
ExplainState *es);
+static void show_append_info(AppendState *astate, ExplainState *es);
static void show_agg_keys(AggState *astate, List *ancestors,
ExplainState *es);
static void show_grouping_sets(PlanState *planstate, Agg *agg,
@@ -1377,6 +1378,8 @@ ExplainNode(PlanState *planstate, List *ancestors,
}
if (plan->parallel_aware)
appendStringInfoString(es->str, "Parallel ");
+ if (plan->async_capable)
+ appendStringInfoString(es->str, "Async ");
appendStringInfoString(es->str, pname);
es->indent++;
}
@@ -1958,6 +1961,11 @@ ExplainNode(PlanState *planstate, List *ancestors,
case T_Hash:
show_hash_info(castNode(HashState, planstate), es);
break;
+
+ case T_Append:
+ show_append_info(castNode(AppendState, planstate), es);
+ break;
+
default:
break;
}
@@ -2311,6 +2319,15 @@ show_merge_append_keys(MergeAppendState *mstate, List *ancestors,
ancestors, es);
}
+static void
+show_append_info(AppendState *astate, ExplainState *es)
+{
+ Append *plan = (Append *) astate->ps.plan;
+
+ if (plan->nasyncplans > 0)
+ ExplainPropertyInteger("Async subplans", "", plan->nasyncplans, es);
+}
+
/*
* Show the grouping keys for an Agg node.
*/
diff --git a/src/backend/executor/Makefile b/src/backend/executor/Makefile
index f990c6473a..1004647d4f 100644
--- a/src/backend/executor/Makefile
+++ b/src/backend/executor/Makefile
@@ -14,6 +14,7 @@ include $(top_builddir)/src/Makefile.global
OBJS = \
execAmi.o \
+ execAsync.o \
execCurrent.o \
execExpr.o \
execExprInterp.o \
diff --git a/src/backend/executor/execAsync.c b/src/backend/executor/execAsync.c
new file mode 100644
index 0000000000..2b7d1877e0
--- /dev/null
+++ b/src/backend/executor/execAsync.c
@@ -0,0 +1,152 @@
+/*-------------------------------------------------------------------------
+ *
+ * execAsync.c
+ * Support routines for asynchronous execution.
+ *
+ * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ * src/backend/executor/execAsync.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "executor/execAsync.h"
+#include "executor/nodeAppend.h"
+#include "executor/nodeForeignscan.h"
+#include "miscadmin.h"
+#include "pgstat.h"
+#include "utils/memutils.h"
+#include "utils/resowner.h"
+
+/*
+ * ExecAsyncConfigureWait: Add wait event to the WaitEventSet if needed.
+ *
+ * If reinit is true, the caller didn't reuse existing WaitEventSet.
+ */
+bool
+ExecAsyncConfigureWait(WaitEventSet *wes, PlanState *node,
+ void *data, bool reinit)
+{
+ switch (nodeTag(node))
+ {
+ case T_ForeignScanState:
+ return ExecForeignAsyncConfigureWait((ForeignScanState *)node,
+ wes, data, reinit);
+ break;
+ default:
+ elog(ERROR, "unrecognized node type: %d",
+ (int) nodeTag(node));
+ }
+}
+
+/*
+ * struct for memory context callback argument used in ExecAsyncEventWait
+ */
+typedef struct {
+ int **p_refind;
+ int *p_refindsize;
+} ExecAsync_mcbarg;
+
+/*
+ * callback function to reset static variables pointing to the memory in
+ * TopTransactionContext in ExecAsyncEventWait.
+ */
+static void ExecAsyncMemoryContextCallback(void *arg)
+{
+ /* arg is the address of the variable refind in ExecAsyncEventWait */
+ ExecAsync_mcbarg *mcbarg = (ExecAsync_mcbarg *) arg;
+ *mcbarg->p_refind = NULL;
+ *mcbarg->p_refindsize = 0;
+}
+
+#define EVENT_BUFFER_SIZE 16
+
+/*
+ * ExecAsyncEventWait:
+ *
+ * Wait for async events to fire. Returns the Bitmapset of fired events.
+ */
+Bitmapset *
+ExecAsyncEventWait(PlanState **nodes, Bitmapset *waitnodes, long timeout)
+{
+ static int *refind = NULL;
+ static int refindsize = 0;
+ WaitEventSet *wes;
+ WaitEvent occurred_event[EVENT_BUFFER_SIZE];
+ int noccurred = 0;
+ Bitmapset *fired_events = NULL;
+ int i;
+ int n;
+
+ n = bms_num_members(waitnodes);
+ wes = CreateWaitEventSet(TopTransactionContext,
+ TopTransactionResourceOwner, n);
+ if (refindsize < n)
+ {
+ if (refindsize == 0)
+ refindsize = EVENT_BUFFER_SIZE; /* XXX */
+ while (refindsize < n)
+ refindsize *= 2;
+ if (refind)
+ refind = (int *) repalloc(refind, refindsize * sizeof(int));
+ else
+ {
+ static ExecAsync_mcbarg mcb_arg =
+ { &refind, &refindsize };
+ static MemoryContextCallback mcb =
+ { ExecAsyncMemoryContextCallback, (void *)&mcb_arg, NULL };
+ MemoryContext oldctxt =
+ MemoryContextSwitchTo(TopTransactionContext);
+
+ /*
+ * refind points to a memory block in
+ * TopTransactionContext. Register a callback to reset it.
+ */
+ MemoryContextRegisterResetCallback(TopTransactionContext, &mcb);
+ refind = (int *) palloc(refindsize * sizeof(int));
+ MemoryContextSwitchTo(oldctxt);
+ }
+ }
+
+ /* Prepare WaitEventSet for waiting on the waitnodes. */
+ n = 0;
+ for (i = bms_next_member(waitnodes, -1) ; i >= 0 ;
+ i = bms_next_member(waitnodes, i))
+ {
+ refind[i] = i;
+ if (ExecAsyncConfigureWait(wes, nodes[i], refind + i, true))
+ n++;
+ }
+
+ /* Return immediately if no node to wait. */
+ if (n == 0)
+ {
+ FreeWaitEventSet(wes);
+ return NULL;
+ }
+
+ noccurred = WaitEventSetWait(wes, timeout, occurred_event,
+ EVENT_BUFFER_SIZE,
+ WAIT_EVENT_ASYNC_WAIT);
+ FreeWaitEventSet(wes);
+ if (noccurred == 0)
+ return NULL;
+
+ for (i = 0 ; i < noccurred ; i++)
+ {
+ WaitEvent *w = &occurred_event[i];
+
+ if ((w->events & (WL_SOCKET_READABLE | WL_SOCKET_WRITEABLE)) != 0)
+ {
+ int n = *(int*)w->user_data;
+
+ fired_events = bms_add_member(fired_events, n);
+ }
+ }
+
+ return fired_events;
+}
diff --git a/src/backend/executor/nodeAppend.c b/src/backend/executor/nodeAppend.c
index 88919e62fa..60c36ee048 100644
--- a/src/backend/executor/nodeAppend.c
+++ b/src/backend/executor/nodeAppend.c
@@ -60,6 +60,7 @@
#include "executor/execdebug.h"
#include "executor/execPartition.h"
#include "executor/nodeAppend.h"
+#include "executor/execAsync.h"
#include "miscadmin.h"
/* Shared state for parallel-aware Append. */
@@ -80,6 +81,7 @@ struct ParallelAppendState
#define INVALID_SUBPLAN_INDEX -1
static TupleTableSlot *ExecAppend(PlanState *pstate);
+static TupleTableSlot *ExecAppendAsync(PlanState *pstate);
static bool choose_next_subplan_locally(AppendState *node);
static bool choose_next_subplan_for_leader(AppendState *node);
static bool choose_next_subplan_for_worker(AppendState *node);
@@ -103,22 +105,22 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
PlanState **appendplanstates;
Bitmapset *validsubplans;
int nplans;
+ int nasyncplans;
int firstvalid;
int i,
j;
/* check for unsupported flags */
- Assert(!(eflags & EXEC_FLAG_MARK));
+ Assert(!(eflags & (EXEC_FLAG_MARK | EXEC_FLAG_ASYNC)));
/*
* create new AppendState for our append node
*/
appendstate->ps.plan = (Plan *) node;
appendstate->ps.state = estate;
- appendstate->ps.ExecProcNode = ExecAppend;
/* Let choose_next_subplan_* function handle setting the first subplan */
- appendstate->as_whichplan = INVALID_SUBPLAN_INDEX;
+ appendstate->as_whichsyncplan = INVALID_SUBPLAN_INDEX;
/* If run-time partition pruning is enabled, then set that up now */
if (node->part_prune_info != NULL)
@@ -152,11 +154,12 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
/*
* When no run-time pruning is required and there's at least one
- * subplan, we can fill as_valid_subplans immediately, preventing
+ * subplan, we can fill as_valid_syncsubplans immediately, preventing
* later calls to ExecFindMatchingSubPlans.
*/
if (!prunestate->do_exec_prune && nplans > 0)
- appendstate->as_valid_subplans = bms_add_range(NULL, 0, nplans - 1);
+ appendstate->as_valid_syncsubplans =
+ bms_add_range(NULL, node->nasyncplans, nplans - 1);
}
else
{
@@ -167,8 +170,9 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
* subplans as valid; they must also all be initialized.
*/
Assert(nplans > 0);
- appendstate->as_valid_subplans = validsubplans =
- bms_add_range(NULL, 0, nplans - 1);
+ validsubplans = bms_add_range(NULL, 0, nplans - 1);
+ appendstate->as_valid_syncsubplans =
+ bms_add_range(NULL, node->nasyncplans, nplans - 1);
appendstate->as_prune_state = NULL;
}
@@ -192,10 +196,20 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
*/
j = 0;
firstvalid = nplans;
+ nasyncplans = 0;
+
i = -1;
while ((i = bms_next_member(validsubplans, i)) >= 0)
{
Plan *initNode = (Plan *) list_nth(node->appendplans, i);
+ int sub_eflags = eflags;
+
+ /* Let async-capable subplans run asynchronously */
+ if (i < node->nasyncplans)
+ {
+ sub_eflags |= EXEC_FLAG_ASYNC;
+ nasyncplans++;
+ }
/*
* Record the lowest appendplans index which is a valid partial plan.
@@ -203,13 +217,46 @@ ExecInitAppend(Append *node, EState *estate, int eflags)
if (i >= node->first_partial_plan && j < firstvalid)
firstvalid = j;
- appendplanstates[j++] = ExecInitNode(initNode, estate, eflags);
+ appendplanstates[j++] = ExecInitNode(initNode, estate, sub_eflags);
}
appendstate->as_first_partial_plan = firstvalid;
appendstate->appendplans = appendplanstates;
appendstate->as_nplans = nplans;
+ /* fill in async stuff */
+ appendstate->as_nasyncplans = nasyncplans;
+ appendstate->as_syncdone = (nasyncplans == nplans);
+ appendstate->as_exec_prune = false;
+
+ /* choose appropriate version of Exec function */
+ if (appendstate->as_nasyncplans == 0)
+ appendstate->ps.ExecProcNode = ExecAppend;
+ else
+ appendstate->ps.ExecProcNode = ExecAppendAsync;
+
+ if (appendstate->as_nasyncplans)
+ {
+ appendstate->as_asyncresult = (TupleTableSlot **)
+ palloc0(appendstate->as_nasyncplans * sizeof(TupleTableSlot *));
+
+ /* initially, all async requests need a request */
+ appendstate->as_needrequest =
+ bms_add_range(NULL, 0, appendstate->as_nasyncplans - 1);
+
+ /*
+ * ExecAppendAsync needs as_valid_syncsubplans to handle async
+ * subnodes.
+ */
+ if (appendstate->as_prune_state != NULL &&
+ appendstate->as_prune_state->do_exec_prune)
+ {
+ Assert(appendstate->as_valid_syncsubplans == NULL);
+
+ appendstate->as_exec_prune = true;
+ }
+ }
+
/*
* Miscellaneous initialization
*/
@@ -233,7 +280,7 @@ ExecAppend(PlanState *pstate)
{
AppendState *node = castNode(AppendState, pstate);
- if (node->as_whichplan < 0)
+ if (node->as_whichsyncplan < 0)
{
/* Nothing to do if there are no subplans */
if (node->as_nplans == 0)
@@ -243,11 +290,13 @@ ExecAppend(PlanState *pstate)
* If no subplan has been chosen, we must choose one before
* proceeding.
*/
- if (node->as_whichplan == INVALID_SUBPLAN_INDEX &&
+ if (node->as_whichsyncplan == INVALID_SUBPLAN_INDEX &&
!node->choose_next_subplan(node))
return ExecClearTuple(node->ps.ps_ResultTupleSlot);
}
+ Assert(node->as_nasyncplans == 0);
+
for (;;)
{
PlanState *subnode;
@@ -258,8 +307,9 @@ ExecAppend(PlanState *pstate)
/*
* figure out which subplan we are currently processing
*/
- Assert(node->as_whichplan >= 0 && node->as_whichplan < node->as_nplans);
- subnode = node->appendplans[node->as_whichplan];
+ Assert(node->as_whichsyncplan >= 0 &&
+ node->as_whichsyncplan < node->as_nplans);
+ subnode = node->appendplans[node->as_whichsyncplan];
/*
* get a tuple from the subplan
@@ -282,6 +332,172 @@ ExecAppend(PlanState *pstate)
}
}
+static TupleTableSlot *
+ExecAppendAsync(PlanState *pstate)
+{
+ AppendState *node = castNode(AppendState, pstate);
+ Bitmapset *needrequest;
+ int i;
+
+ Assert(node->as_nasyncplans > 0);
+
+restart:
+ if (node->as_nasyncresult > 0)
+ {
+ --node->as_nasyncresult;
+ return node->as_asyncresult[node->as_nasyncresult];
+ }
+
+ if (node->as_exec_prune)
+ {
+ Bitmapset *valid_subplans =
+ ExecFindMatchingSubPlans(node->as_prune_state);
+
+ /* Distribute valid subplans into sync and async */
+ node->as_needrequest =
+ bms_intersect(node->as_needrequest, valid_subplans);
+ node->as_valid_syncsubplans =
+ bms_difference(valid_subplans, node->as_needrequest);
+
+ node->as_exec_prune = false;
+ }
+
+ needrequest = node->as_needrequest;
+ node->as_needrequest = NULL;
+ while ((i = bms_first_member(needrequest)) >= 0)
+ {
+ TupleTableSlot *slot;
+ PlanState *subnode = node->appendplans[i];
+
+ slot = ExecProcNode(subnode);
+ if (subnode->asyncstate == AS_AVAILABLE)
+ {
+ if (!TupIsNull(slot))
+ {
+ node->as_asyncresult[node->as_nasyncresult++] = slot;
+ node->as_needrequest = bms_add_member(node->as_needrequest, i);
+ }
+ }
+ else
+ node->as_pending_async = bms_add_member(node->as_pending_async, i);
+ }
+ bms_free(needrequest);
+
+ for (;;)
+ {
+ TupleTableSlot *result;
+
+ /* return now if a result is available */
+ if (node->as_nasyncresult > 0)
+ {
+ --node->as_nasyncresult;
+ return node->as_asyncresult[node->as_nasyncresult];
+ }
+
+ while (!bms_is_empty(node->as_pending_async))
+ {
+ /* Don't wait for async nodes if any sync node exists. */
+ long timeout = node->as_syncdone ? -1 : 0;
+ Bitmapset *fired;
+ int i;
+
+ fired = ExecAsyncEventWait(node->appendplans,
+ node->as_pending_async,
+ timeout);
+
+ if (bms_is_empty(fired) && node->as_syncdone)
+ {
+ /*
+ * We come here when all the subnodes had fired before
+ * waiting. Retry fetching from the nodes.
+ */
+ node->as_needrequest = node->as_pending_async;
+ node->as_pending_async = NULL;
+ goto restart;
+ }
+
+ while ((i = bms_first_member(fired)) >= 0)
+ {
+ TupleTableSlot *slot;
+ PlanState *subnode = node->appendplans[i];
+ slot = ExecProcNode(subnode);
+
+ Assert(subnode->asyncstate == AS_AVAILABLE);
+
+ if (!TupIsNull(slot))
+ {
+ node->as_asyncresult[node->as_nasyncresult++] = slot;
+ node->as_needrequest =
+ bms_add_member(node->as_needrequest, i);
+ }
+
+ node->as_pending_async =
+ bms_del_member(node->as_pending_async, i);
+ }
+ bms_free(fired);
+
+ /* return now if a result is available */
+ if (node->as_nasyncresult > 0)
+ {
+ --node->as_nasyncresult;
+ return node->as_asyncresult[node->as_nasyncresult];
+ }
+
+ if (!node->as_syncdone)
+ break;
+ }
+
+ /*
+ * If there is no asynchronous activity still pending and the
+ * synchronous activity is also complete, we're totally done scanning
+ * this node. Otherwise, we're done with the asynchronous stuff but
+ * must continue scanning the synchronous children.
+ */
+
+ if (!node->as_syncdone &&
+ node->as_whichsyncplan == INVALID_SUBPLAN_INDEX)
+ node->as_syncdone = !node->choose_next_subplan(node);
+
+ if (node->as_syncdone)
+ {
+ Assert(bms_is_empty(node->as_pending_async));
+ return ExecClearTuple(node->ps.ps_ResultTupleSlot);
+ }
+
+ /*
+ * get a tuple from the subplan
+ */
+ result = ExecProcNode(node->appendplans[node->as_whichsyncplan]);
+
+ if (!TupIsNull(result))
+ {
+ /*
+ * If the subplan gave us something then return it as-is. We do
+ * NOT make use of the result slot that was set up in
+ * ExecInitAppend; there's no need for it.
+ */
+ return result;
+ }
+
+ /*
+ * Go on to the "next" subplan. If no more subplans, return the empty
+ * slot set up for us by ExecInitAppend, unless there are async plans
+ * we have yet to finish.
+ */
+ if (!node->choose_next_subplan(node))
+ {
+ node->as_syncdone = true;
+ if (bms_is_empty(node->as_pending_async))
+ {
+ Assert(bms_is_empty(node->as_needrequest));
+ return ExecClearTuple(node->ps.ps_ResultTupleSlot);
+ }
+ }
+
+ /* Else loop back and try to get a tuple from the new subplan */
+ }
+}
+
/* ----------------------------------------------------------------
* ExecEndAppend
*
@@ -324,10 +540,18 @@ ExecReScanAppend(AppendState *node)
bms_overlap(node->ps.chgParam,
node->as_prune_state->execparamids))
{
- bms_free(node->as_valid_subplans);
- node->as_valid_subplans = NULL;
+ bms_free(node->as_valid_syncsubplans);
+ node->as_valid_syncsubplans = NULL;
}
+ /* Reset async state. */
+ for (i = 0; i < node->as_nasyncplans; ++i)
+ ExecShutdownNode(node->appendplans[i]);
+
+ node->as_nasyncresult = 0;
+ node->as_needrequest = bms_add_range(NULL, 0, node->as_nasyncplans - 1);
+ node->as_syncdone = (node->as_nasyncplans == node->as_nplans);
+
for (i = 0; i < node->as_nplans; i++)
{
PlanState *subnode = node->appendplans[i];
@@ -348,7 +572,7 @@ ExecReScanAppend(AppendState *node)
}
/* Let choose_next_subplan_* function handle setting the first subplan */
- node->as_whichplan = INVALID_SUBPLAN_INDEX;
+ node->as_whichsyncplan = INVALID_SUBPLAN_INDEX;
}
/* ----------------------------------------------------------------
@@ -436,7 +660,7 @@ ExecAppendInitializeWorker(AppendState *node, ParallelWorkerContext *pwcxt)
static bool
choose_next_subplan_locally(AppendState *node)
{
- int whichplan = node->as_whichplan;
+ int whichplan = node->as_whichsyncplan;
int nextplan;
/* We should never be called when there are no subplans */
@@ -451,10 +675,18 @@ choose_next_subplan_locally(AppendState *node)
*/
if (whichplan == INVALID_SUBPLAN_INDEX)
{
- if (node->as_valid_subplans == NULL)
- node->as_valid_subplans =
+ /* Shouldn't have an active async node */
+ Assert(bms_is_empty(node->as_needrequest));
+
+ if (node->as_valid_syncsubplans == NULL)
+ node->as_valid_syncsubplans =
ExecFindMatchingSubPlans(node->as_prune_state);
+ /* Exclude async plans */
+ if (node->as_nasyncplans > 0)
+ bms_del_range(node->as_valid_syncsubplans,
+ 0, node->as_nasyncplans - 1);
+
whichplan = -1;
}
@@ -462,14 +694,14 @@ choose_next_subplan_locally(AppendState *node)
Assert(whichplan >= -1 && whichplan <= node->as_nplans);
if (ScanDirectionIsForward(node->ps.state->es_direction))
- nextplan = bms_next_member(node->as_valid_subplans, whichplan);
+ nextplan = bms_next_member(node->as_valid_syncsubplans, whichplan);
else
- nextplan = bms_prev_member(node->as_valid_subplans, whichplan);
+ nextplan = bms_prev_member(node->as_valid_syncsubplans, whichplan);
if (nextplan < 0)
return false;
- node->as_whichplan = nextplan;
+ node->as_whichsyncplan = nextplan;
return true;
}
@@ -490,29 +722,29 @@ choose_next_subplan_for_leader(AppendState *node)
/* Backward scan is not supported by parallel-aware plans */
Assert(ScanDirectionIsForward(node->ps.state->es_direction));
- /* We should never be called when there are no subplans */
- Assert(node->as_nplans > 0);
+ /* We should never be called when there are no sync subplans */
+ Assert(node->as_nplans > node->as_nasyncplans);
LWLockAcquire(&pstate->pa_lock, LW_EXCLUSIVE);
- if (node->as_whichplan != INVALID_SUBPLAN_INDEX)
+ if (node->as_whichsyncplan != INVALID_SUBPLAN_INDEX)
{
/* Mark just-completed subplan as finished. */
- node->as_pstate->pa_finished[node->as_whichplan] = true;
+ node->as_pstate->pa_finished[node->as_whichsyncplan] = true;
}
else
{
/* Start with last subplan. */
- node->as_whichplan = node->as_nplans - 1;
+ node->as_whichsyncplan = node->as_nplans - 1;
/*
* If we've yet to determine the valid subplans then do so now. If
* run-time pruning is disabled then the valid subplans will always be
* set to all subplans.
*/
- if (node->as_valid_subplans == NULL)
+ if (node->as_valid_syncsubplans == NULL)
{
- node->as_valid_subplans =
+ node->as_valid_syncsubplans =
ExecFindMatchingSubPlans(node->as_prune_state);
/*
@@ -524,26 +756,26 @@ choose_next_subplan_for_leader(AppendState *node)
}
/* Loop until we find a subplan to execute. */
- while (pstate->pa_finished[node->as_whichplan])
+ while (pstate->pa_finished[node->as_whichsyncplan])
{
- if (node->as_whichplan == 0)
+ if (node->as_whichsyncplan == 0)
{
pstate->pa_next_plan = INVALID_SUBPLAN_INDEX;
- node->as_whichplan = INVALID_SUBPLAN_INDEX;
+ node->as_whichsyncplan = INVALID_SUBPLAN_INDEX;
LWLockRelease(&pstate->pa_lock);
return false;
}
/*
- * We needn't pay attention to as_valid_subplans here as all invalid
+ * We needn't pay attention to as_valid_syncsubplans here as all invalid
* plans have been marked as finished.
*/
- node->as_whichplan--;
+ node->as_whichsyncplan--;
}
/* If non-partial, immediately mark as finished. */
- if (node->as_whichplan < node->as_first_partial_plan)
- node->as_pstate->pa_finished[node->as_whichplan] = true;
+ if (node->as_whichsyncplan < node->as_first_partial_plan)
+ node->as_pstate->pa_finished[node->as_whichsyncplan] = true;
LWLockRelease(&pstate->pa_lock);
@@ -571,23 +803,23 @@ choose_next_subplan_for_worker(AppendState *node)
/* Backward scan is not supported by parallel-aware plans */
Assert(ScanDirectionIsForward(node->ps.state->es_direction));
- /* We should never be called when there are no subplans */
- Assert(node->as_nplans > 0);
+ /* We should never be called when there are no sync subplans */
+ Assert(node->as_nplans > node->as_nasyncplans);
LWLockAcquire(&pstate->pa_lock, LW_EXCLUSIVE);
/* Mark just-completed subplan as finished. */
- if (node->as_whichplan != INVALID_SUBPLAN_INDEX)
- node->as_pstate->pa_finished[node->as_whichplan] = true;
+ if (node->as_whichsyncplan != INVALID_SUBPLAN_INDEX)
+ node->as_pstate->pa_finished[node->as_whichsyncplan] = true;
/*
* If we've yet to determine the valid subplans then do so now. If
* run-time pruning is disabled then the valid subplans will always be set
* to all subplans.
*/
- else if (node->as_valid_subplans == NULL)
+ else if (node->as_valid_syncsubplans == NULL)
{
- node->as_valid_subplans =
+ node->as_valid_syncsubplans =
ExecFindMatchingSubPlans(node->as_prune_state);
mark_invalid_subplans_as_finished(node);
}
@@ -600,30 +832,30 @@ choose_next_subplan_for_worker(AppendState *node)
}
/* Save the plan from which we are starting the search. */
- node->as_whichplan = pstate->pa_next_plan;
+ node->as_whichsyncplan = pstate->pa_next_plan;
/* Loop until we find a valid subplan to execute. */
while (pstate->pa_finished[pstate->pa_next_plan])
{
int nextplan;
- nextplan = bms_next_member(node->as_valid_subplans,
+ nextplan = bms_next_member(node->as_valid_syncsubplans,
pstate->pa_next_plan);
if (nextplan >= 0)
{
/* Advance to the next valid plan. */
pstate->pa_next_plan = nextplan;
}
- else if (node->as_whichplan > node->as_first_partial_plan)
+ else if (node->as_whichsyncplan > node->as_first_partial_plan)
{
/*
* Try looping back to the first valid partial plan, if there is
* one. If there isn't, arrange to bail out below.
*/
- nextplan = bms_next_member(node->as_valid_subplans,
+ nextplan = bms_next_member(node->as_valid_syncsubplans,
node->as_first_partial_plan - 1);
pstate->pa_next_plan =
- nextplan < 0 ? node->as_whichplan : nextplan;
+ nextplan < 0 ? node->as_whichsyncplan : nextplan;
}
else
{
@@ -631,10 +863,10 @@ choose_next_subplan_for_worker(AppendState *node)
* At last plan, and either there are no partial plans or we've
* tried them all. Arrange to bail out.
*/
- pstate->pa_next_plan = node->as_whichplan;
+ pstate->pa_next_plan = node->as_whichsyncplan;
}
- if (pstate->pa_next_plan == node->as_whichplan)
+ if (pstate->pa_next_plan == node->as_whichsyncplan)
{
/* We've tried everything! */
pstate->pa_next_plan = INVALID_SUBPLAN_INDEX;
@@ -644,8 +876,8 @@ choose_next_subplan_for_worker(AppendState *node)
}
/* Pick the plan we found, and advance pa_next_plan one more time. */
- node->as_whichplan = pstate->pa_next_plan;
- pstate->pa_next_plan = bms_next_member(node->as_valid_subplans,
+ node->as_whichsyncplan = pstate->pa_next_plan;
+ pstate->pa_next_plan = bms_next_member(node->as_valid_syncsubplans,
pstate->pa_next_plan);
/*
@@ -654,7 +886,7 @@ choose_next_subplan_for_worker(AppendState *node)
*/
if (pstate->pa_next_plan < 0)
{
- int nextplan = bms_next_member(node->as_valid_subplans,
+ int nextplan = bms_next_member(node->as_valid_syncsubplans,
node->as_first_partial_plan - 1);
if (nextplan >= 0)
@@ -671,8 +903,8 @@ choose_next_subplan_for_worker(AppendState *node)
}
/* If non-partial, immediately mark as finished. */
- if (node->as_whichplan < node->as_first_partial_plan)
- node->as_pstate->pa_finished[node->as_whichplan] = true;
+ if (node->as_whichsyncplan < node->as_first_partial_plan)
+ node->as_pstate->pa_finished[node->as_whichsyncplan] = true;
LWLockRelease(&pstate->pa_lock);
@@ -699,13 +931,13 @@ mark_invalid_subplans_as_finished(AppendState *node)
Assert(node->as_prune_state);
/* Nothing to do if all plans are valid */
- if (bms_num_members(node->as_valid_subplans) == node->as_nplans)
+ if (bms_num_members(node->as_valid_syncsubplans) == node->as_nplans)
return;
/* Mark all non-valid plans as finished */
for (i = 0; i < node->as_nplans; i++)
{
- if (!bms_is_member(i, node->as_valid_subplans))
+ if (!bms_is_member(i, node->as_valid_syncsubplans))
node->as_pstate->pa_finished[i] = true;
}
}
diff --git a/src/backend/executor/nodeForeignscan.c b/src/backend/executor/nodeForeignscan.c
index 513471ab9b..3bf4aaa63d 100644
--- a/src/backend/executor/nodeForeignscan.c
+++ b/src/backend/executor/nodeForeignscan.c
@@ -141,6 +141,10 @@ ExecInitForeignScan(ForeignScan *node, EState *estate, int eflags)
scanstate->ss.ps.plan = (Plan *) node;
scanstate->ss.ps.state = estate;
scanstate->ss.ps.ExecProcNode = ExecForeignScan;
+ scanstate->ss.ps.asyncstate = AS_AVAILABLE;
+
+ if ((eflags & EXEC_FLAG_ASYNC) != 0)
+ scanstate->fs_async = true;
/*
* Miscellaneous initialization
@@ -384,3 +388,20 @@ ExecShutdownForeignScan(ForeignScanState *node)
if (fdwroutine->ShutdownForeignScan)
fdwroutine->ShutdownForeignScan(node);
}
+
+/* ----------------------------------------------------------------
+ * ExecAsyncForeignScanConfigureWait
+ *
+ * In async mode, configure for a wait
+ * ----------------------------------------------------------------
+ */
+bool
+ExecForeignAsyncConfigureWait(ForeignScanState *node, WaitEventSet *wes,
+ void *caller_data, bool reinit)
+{
+ FdwRoutine *fdwroutine = node->fdwroutine;
+
+ Assert(fdwroutine->ForeignAsyncConfigureWait != NULL);
+ return fdwroutine->ForeignAsyncConfigureWait(node, wes,
+ caller_data, reinit);
+}
diff --git a/src/backend/nodes/bitmapset.c b/src/backend/nodes/bitmapset.c
index 2719ea45a3..05b625783b 100644
--- a/src/backend/nodes/bitmapset.c
+++ b/src/backend/nodes/bitmapset.c
@@ -895,6 +895,78 @@ bms_add_range(Bitmapset *a, int lower, int upper)
return a;
}
+/*
+ * bms_del_range
+ * Delete members in the range of 'lower' to 'upper' from the set.
+ *
+ * Note this could also be done by calling bms_del_member in a loop, however,
+ * using this function will be faster when the range is large as we work at
+ * the bitmapword level rather than at bit level.
+ */
+Bitmapset *
+bms_del_range(Bitmapset *a, int lower, int upper)
+{
+ int lwordnum,
+ lbitnum,
+ uwordnum,
+ ushiftbits,
+ wordnum;
+
+ if (lower < 0 || upper < 0)
+ elog(ERROR, "negative bitmapset member not allowed");
+ if (lower > upper)
+ elog(ERROR, "lower range must not be above upper range");
+ uwordnum = WORDNUM(upper);
+
+ if (a == NULL)
+ {
+ a = (Bitmapset *) palloc0(BITMAPSET_SIZE(uwordnum + 1));
+ a->nwords = uwordnum + 1;
+ }
+
+ /* ensure we have enough words to store the upper bit */
+ else if (uwordnum >= a->nwords)
+ {
+ int oldnwords = a->nwords;
+ int i;
+
+ a = (Bitmapset *) repalloc(a, BITMAPSET_SIZE(uwordnum + 1));
+ a->nwords = uwordnum + 1;
+ /* zero out the enlarged portion */
+ for (i = oldnwords; i < a->nwords; i++)
+ a->words[i] = 0;
+ }
+
+ wordnum = lwordnum = WORDNUM(lower);
+
+ lbitnum = BITNUM(lower);
+ ushiftbits = BITNUM(upper) + 1;
+
+ /*
+ * Special case when lwordnum is the same as uwordnum we must perform the
+ * upper and lower masking on the word.
+ */
+ if (lwordnum == uwordnum)
+ {
+ a->words[lwordnum] &= ((bitmapword) (((bitmapword) 1 << lbitnum) - 1)
+ | (~(bitmapword) 0) << ushiftbits);
+ }
+ else
+ {
+ /* turn off lbitnum and all bits left of it */
+ a->words[wordnum++] &= (bitmapword) (((bitmapword) 1 << lbitnum) - 1);
+
+ /* turn off all bits for any intermediate words */
+ while (wordnum < uwordnum)
+ a->words[wordnum++] = (bitmapword) 0;
+
+ /* turn off upper's bit and all bits right of it. */
+ a->words[uwordnum] &= (~(bitmapword) 0) << ushiftbits;
+ }
+
+ return a;
+}
+
/*
* bms_int_members - like bms_intersect, but left input is recycled
*/
diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c
index 0409a40b82..4eff3712b7 100644
--- a/src/backend/nodes/copyfuncs.c
+++ b/src/backend/nodes/copyfuncs.c
@@ -121,6 +121,7 @@ CopyPlanFields(const Plan *from, Plan *newnode)
COPY_SCALAR_FIELD(plan_width);
COPY_SCALAR_FIELD(parallel_aware);
COPY_SCALAR_FIELD(parallel_safe);
+ COPY_SCALAR_FIELD(async_capable);
COPY_SCALAR_FIELD(plan_node_id);
COPY_NODE_FIELD(targetlist);
COPY_NODE_FIELD(qual);
@@ -246,6 +247,8 @@ _copyAppend(const Append *from)
COPY_NODE_FIELD(appendplans);
COPY_SCALAR_FIELD(first_partial_plan);
COPY_NODE_FIELD(part_prune_info);
+ COPY_SCALAR_FIELD(nasyncplans);
+ COPY_SCALAR_FIELD(referent);
return newnode;
}
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index f0386480ab..2b1b0e9141 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -334,6 +334,7 @@ _outPlanInfo(StringInfo str, const Plan *node)
WRITE_INT_FIELD(plan_width);
WRITE_BOOL_FIELD(parallel_aware);
WRITE_BOOL_FIELD(parallel_safe);
+ WRITE_BOOL_FIELD(async_capable);
WRITE_INT_FIELD(plan_node_id);
WRITE_NODE_FIELD(targetlist);
WRITE_NODE_FIELD(qual);
@@ -436,6 +437,8 @@ _outAppend(StringInfo str, const Append *node)
WRITE_NODE_FIELD(appendplans);
WRITE_INT_FIELD(first_partial_plan);
WRITE_NODE_FIELD(part_prune_info);
+ WRITE_INT_FIELD(nasyncplans);
+ WRITE_INT_FIELD(referent);
}
static void
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index 42050ab719..63af7c02d8 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -1572,6 +1572,7 @@ ReadCommonPlan(Plan *local_node)
READ_INT_FIELD(plan_width);
READ_BOOL_FIELD(parallel_aware);
READ_BOOL_FIELD(parallel_safe);
+ READ_BOOL_FIELD(async_capable);
READ_INT_FIELD(plan_node_id);
READ_NODE_FIELD(targetlist);
READ_NODE_FIELD(qual);
@@ -1672,6 +1673,8 @@ _readAppend(void)
READ_NODE_FIELD(appendplans);
READ_INT_FIELD(first_partial_plan);
READ_NODE_FIELD(part_prune_info);
+ READ_INT_FIELD(nasyncplans);
+ READ_INT_FIELD(referent);
READ_DONE();
}
diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c
index b399592ff8..17e9a7a897 100644
--- a/src/backend/optimizer/path/allpaths.c
+++ b/src/backend/optimizer/path/allpaths.c
@@ -3973,6 +3973,30 @@ generate_partitionwise_join_paths(PlannerInfo *root, RelOptInfo *rel)
list_free(live_children);
}
+/*
+ * is_projection_capable_path
+ * Check whether a given Path node is async-capable.
+ */
+bool
+is_async_capable_path(Path *path)
+{
+ switch (nodeTag(path))
+ {
+ case T_ForeignPath:
+ {
+ FdwRoutine *fdwroutine = path->parent->fdwroutine;
+
+ Assert(fdwroutine != NULL);
+ if (fdwroutine->IsForeignPathAsyncCapable != NULL &&
+ fdwroutine->IsForeignPathAsyncCapable((ForeignPath *) path))
+ return true;
+ }
+ default:
+ break;
+ }
+ return false;
+}
+
/*****************************************************************************
* DEBUG SUPPORT
diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index cd3716d494..143e00b13e 100644
--- a/src/backend/optimizer/path/costsize.c
+++ b/src/backend/optimizer/path/costsize.c
@@ -2048,22 +2048,59 @@ cost_append(AppendPath *apath)
if (pathkeys == NIL)
{
- Path *subpath = (Path *) linitial(apath->subpaths);
-
- /*
- * For an unordered, non-parallel-aware Append we take the startup
- * cost as the startup cost of the first subpath.
- */
- apath->path.startup_cost = subpath->startup_cost;
+ Cost first_nonasync_startup_cost = -1.0;
+ Cost async_min_startup_cost = -1;
+ Cost async_max_cost = 0.0;
/* Compute rows and costs as sums of subplan rows and costs. */
foreach(l, apath->subpaths)
{
Path *subpath = (Path *) lfirst(l);
+ /*
+ * For an unordered, non-parallel-aware Append we take the
+ * startup cost as the startup cost of the first
+ * nonasync-capable subpath or the minimum startup cost of
+ * async-capable subpaths.
+ */
+ if (!is_async_capable_path(subpath))
+ {
+ if (first_nonasync_startup_cost < 0.0)
+ first_nonasync_startup_cost = subpath->startup_cost;
+
+ apath->path.total_cost += subpath->total_cost;
+ }
+ else
+ {
+ if (async_min_startup_cost < 0.0 ||
+ async_min_startup_cost > subpath->startup_cost)
+ async_min_startup_cost = subpath->startup_cost;
+
+ /*
+ * It's not obvious how to determine the total cost of
+ * async subnodes. Although it is not always true, we
+ * assume it is the maximum cost among all async subnodes.
+ */
+ if (async_max_cost < subpath->total_cost)
+ async_max_cost = subpath->total_cost;
+ }
+
apath->path.rows += subpath->rows;
- apath->path.total_cost += subpath->total_cost;
}
+
+ /*
+ * If there's an sync subnodes, the startup cost is the startup
+ * cost of the first sync subnode. Otherwise it's the minimal
+ * startup cost of async subnodes.
+ */
+ if (first_nonasync_startup_cost >= 0.0)
+ apath->path.startup_cost = first_nonasync_startup_cost;
+ else
+ apath->path.startup_cost = async_min_startup_cost;
+
+ /* Use async maximum cost if it exceeds the sync total cost */
+ if (async_max_cost > apath->path.total_cost)
+ apath->path.total_cost = async_max_cost;
}
else
{
@@ -2084,6 +2121,8 @@ cost_append(AppendPath *apath)
* This case is also different from the above in that we have to
* account for possibly injecting sorts into subpaths that aren't
* natively ordered.
+ *
+ * Note: An ordered append won't be run asynchronously.
*/
foreach(l, apath->subpaths)
{
diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c
index 3d7a4e373f..3ae46ed6f1 100644
--- a/src/backend/optimizer/plan/createplan.c
+++ b/src/backend/optimizer/plan/createplan.c
@@ -1082,6 +1082,11 @@ create_append_plan(PlannerInfo *root, AppendPath *best_path, int flags)
bool tlist_was_changed = false;
List *pathkeys = best_path->path.pathkeys;
List *subplans = NIL;
+ List *asyncplans = NIL;
+ List *syncplans = NIL;
+ List *asyncpaths = NIL;
+ List *syncpaths = NIL;
+ List *newsubpaths = NIL;
ListCell *subpaths;
RelOptInfo *rel = best_path->path.parent;
PartitionPruneInfo *partpruneinfo = NULL;
@@ -1090,6 +1095,9 @@ create_append_plan(PlannerInfo *root, AppendPath *best_path, int flags)
Oid *nodeSortOperators = NULL;
Oid *nodeCollations = NULL;
bool *nodeNullsFirst = NULL;
+ int nasyncplans = 0;
+ bool first = true;
+ bool referent_is_sync = true;
/*
* The subpaths list could be empty, if every child was proven empty by
@@ -1219,9 +1227,40 @@ create_append_plan(PlannerInfo *root, AppendPath *best_path, int flags)
}
}
- subplans = lappend(subplans, subplan);
+ /*
+ * Classify as async-capable or not. If we have decided to run the
+ * children in parallel, we cannot any one of them run asynchronously.
+ * Planner thinks that all subnodes are executed in order if this
+ * append is orderd. No subpaths cannot be run asynchronously in that
+ * case.
+ */
+ if (pathkeys == NIL &&
+ !best_path->path.parallel_safe && is_async_capable_path(subpath))
+ {
+ subplan->async_capable = true;
+ asyncplans = lappend(asyncplans, subplan);
+ asyncpaths = lappend(asyncpaths, subpath);
+ ++nasyncplans;
+ if (first)
+ referent_is_sync = false;
+ }
+ else
+ {
+ syncplans = lappend(syncplans, subplan);
+ syncpaths = lappend(syncpaths, subpath);
+ }
+
+ first = false;
}
+ /*
+ * subplan contains asyncplans in the first half, if any, and sync plans in
+ * another half, if any. We need that the same for subpaths to make
+ * partition pruning information in sync with subplans.
+ */
+ subplans = list_concat(asyncplans, syncplans);
+ newsubpaths = list_concat(asyncpaths, syncpaths);
+
/*
* If any quals exist, they may be useful to perform further partition
* pruning during execution. Gather information needed by the executor to
@@ -1249,7 +1288,7 @@ create_append_plan(PlannerInfo *root, AppendPath *best_path, int flags)
if (prunequal != NIL)
partpruneinfo =
make_partition_pruneinfo(root, rel,
- best_path->subpaths,
+ newsubpaths,
best_path->partitioned_rels,
prunequal);
}
@@ -1257,6 +1296,8 @@ create_append_plan(PlannerInfo *root, AppendPath *best_path, int flags)
plan->appendplans = subplans;
plan->first_partial_plan = best_path->first_partial_path;
plan->part_prune_info = partpruneinfo;
+ plan->nasyncplans = nasyncplans;
+ plan->referent = referent_is_sync ? nasyncplans : 0;
copy_generic_path_info(&plan->plan, (Path *) best_path);
diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c
index 30020f8cda..faed3f2442 100644
--- a/src/backend/postmaster/pgstat.c
+++ b/src/backend/postmaster/pgstat.c
@@ -3878,6 +3878,9 @@ pgstat_get_wait_ipc(WaitEventIPC w)
case WAIT_EVENT_XACT_GROUP_UPDATE:
event_name = "XactGroupUpdate";
break;
+ case WAIT_EVENT_ASYNC_WAIT:
+ event_name = "AsyncExecWait";
+ break;
/* no default case, so that compiler will warn */
}
diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c
index 62023c20b2..07aeb43a7f 100644
--- a/src/backend/utils/adt/ruleutils.c
+++ b/src/backend/utils/adt/ruleutils.c
@@ -4574,10 +4574,14 @@ set_deparse_plan(deparse_namespace *dpns, Plan *plan)
* tlists according to one of the children, and the first one is the most
* natural choice. Likewise special-case ModifyTable to pretend that the
* first child plan is the OUTER referent; this is to support RETURNING
- * lists containing references to non-target relations.
+ * lists containing references to non-target relations. For Append, use the
+ * explicitly specified referent.
*/
if (IsA(plan, Append))
- dpns->outer_plan = linitial(((Append *) plan)->appendplans);
+ {
+ Append *app = (Append *) plan;
+ dpns->outer_plan = list_nth(app->appendplans, app->referent);
+ }
else if (IsA(plan, MergeAppend))
dpns->outer_plan = linitial(((MergeAppend *) plan)->mergeplans);
else if (IsA(plan, ModifyTable))
diff --git a/src/backend/utils/resowner/resowner.c b/src/backend/utils/resowner/resowner.c
index 237ca9fa30..27742a1641 100644
--- a/src/backend/utils/resowner/resowner.c
+++ b/src/backend/utils/resowner/resowner.c
@@ -1416,7 +1416,7 @@ void
ResourceOwnerForgetWES(ResourceOwner owner, WaitEventSet *events)
{
/*
- * XXXX: There's no property to show as an identier of a wait event set,
+ * XXXX: There's no property to show as an identifier of a wait event set,
* use its pointer instead.
*/
if (!ResourceArrayRemove(&(owner->wesarr), PointerGetDatum(events)))
@@ -1431,7 +1431,7 @@ static void
PrintWESLeakWarning(WaitEventSet *events)
{
/*
- * XXXX: There's no property to show as an identier of a wait event set,
+ * XXXX: There's no property to show as an identifier of a wait event set,
* use its pointer instead.
*/
elog(WARNING, "wait event set leak: %p still referenced",
diff --git a/src/include/executor/execAsync.h b/src/include/executor/execAsync.h
new file mode 100644
index 0000000000..3b6bf4a516
--- /dev/null
+++ b/src/include/executor/execAsync.h
@@ -0,0 +1,22 @@
+/*--------------------------------------------------------------------
+ * execAsync.c
+ * Support functions for asynchronous query execution
+ *
+ * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ * src/backend/executor/execAsync.c
+ *--------------------------------------------------------------------
+ */
+#ifndef EXECASYNC_H
+#define EXECASYNC_H
+
+#include "nodes/execnodes.h"
+#include "storage/latch.h"
+
+extern bool ExecAsyncConfigureWait(WaitEventSet *wes, PlanState *node,
+ void *data, bool reinit);
+extern Bitmapset *ExecAsyncEventWait(PlanState **nodes, Bitmapset *waitnodes,
+ long timeout);
+#endif /* EXECASYNC_H */
diff --git a/src/include/executor/executor.h b/src/include/executor/executor.h
index 415e117407..9cf2c1f676 100644
--- a/src/include/executor/executor.h
+++ b/src/include/executor/executor.h
@@ -59,6 +59,7 @@
#define EXEC_FLAG_MARK 0x0008 /* need mark/restore */
#define EXEC_FLAG_SKIP_TRIGGERS 0x0010 /* skip AfterTrigger calls */
#define EXEC_FLAG_WITH_NO_DATA 0x0020 /* rel scannability doesn't matter */
+#define EXEC_FLAG_ASYNC 0x0040 /* request async execution */
/* Hook for plugins to get control in ExecutorStart() */
diff --git a/src/include/executor/nodeForeignscan.h b/src/include/executor/nodeForeignscan.h
index 326d713ebf..71a233b41f 100644
--- a/src/include/executor/nodeForeignscan.h
+++ b/src/include/executor/nodeForeignscan.h
@@ -30,5 +30,8 @@ extern void ExecForeignScanReInitializeDSM(ForeignScanState *node,
extern void ExecForeignScanInitializeWorker(ForeignScanState *node,
ParallelWorkerContext *pwcxt);
extern void ExecShutdownForeignScan(ForeignScanState *node);
+extern bool ExecForeignAsyncConfigureWait(ForeignScanState *node,
+ WaitEventSet *wes,
+ void *caller_data, bool reinit);
#endif /* NODEFOREIGNSCAN_H */
diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h
index 95556dfb15..853ba2b5ad 100644
--- a/src/include/foreign/fdwapi.h
+++ b/src/include/foreign/fdwapi.h
@@ -169,6 +169,11 @@ typedef bool (*IsForeignScanParallelSafe_function) (PlannerInfo *root,
typedef List *(*ReparameterizeForeignPathByChild_function) (PlannerInfo *root,
List *fdw_private,
RelOptInfo *child_rel);
+typedef bool (*IsForeignPathAsyncCapable_function) (ForeignPath *path);
+typedef bool (*ForeignAsyncConfigureWait_function) (ForeignScanState *node,
+ WaitEventSet *wes,
+ void *caller_data,
+ bool reinit);
/*
* FdwRoutine is the struct returned by a foreign-data wrapper's handler
@@ -190,6 +195,7 @@ typedef struct FdwRoutine
GetForeignPlan_function GetForeignPlan;
BeginForeignScan_function BeginForeignScan;
IterateForeignScan_function IterateForeignScan;
+ IterateForeignScan_function IterateForeignScanAsync;
ReScanForeignScan_function ReScanForeignScan;
EndForeignScan_function EndForeignScan;
@@ -242,6 +248,11 @@ typedef struct FdwRoutine
InitializeDSMForeignScan_function InitializeDSMForeignScan;
ReInitializeDSMForeignScan_function ReInitializeDSMForeignScan;
InitializeWorkerForeignScan_function InitializeWorkerForeignScan;
+
+ /* Support functions for asynchronous execution */
+ IsForeignPathAsyncCapable_function IsForeignPathAsyncCapable;
+ ForeignAsyncConfigureWait_function ForeignAsyncConfigureWait;
+
ShutdownForeignScan_function ShutdownForeignScan;
/* Support functions for path reparameterization. */
diff --git a/src/include/nodes/bitmapset.h b/src/include/nodes/bitmapset.h
index d113c271ee..177e6218cb 100644
--- a/src/include/nodes/bitmapset.h
+++ b/src/include/nodes/bitmapset.h
@@ -107,6 +107,7 @@ extern Bitmapset *bms_add_members(Bitmapset *a, const Bitmapset *b);
extern Bitmapset *bms_add_range(Bitmapset *a, int lower, int upper);
extern Bitmapset *bms_int_members(Bitmapset *a, const Bitmapset *b);
extern Bitmapset *bms_del_members(Bitmapset *a, const Bitmapset *b);
+extern Bitmapset *bms_del_range(Bitmapset *a, int lower, int upper);
extern Bitmapset *bms_join(Bitmapset *a, Bitmapset *b);
/* support for iterating through the integer elements of a set: */
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index ef448d67c7..dce7fb0e07 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -925,6 +925,12 @@ typedef TupleTableSlot *(*ExecProcNodeMtd) (struct PlanState *pstate);
* abstract superclass for all PlanState-type nodes.
* ----------------
*/
+typedef enum AsyncState
+{
+ AS_AVAILABLE,
+ AS_WAITING
+} AsyncState;
+
typedef struct PlanState
{
NodeTag type;
@@ -1013,6 +1019,11 @@ typedef struct PlanState
bool outeropsset;
bool inneropsset;
bool resultopsset;
+
+ /* Async subnode execution stuff */
+ AsyncState asyncstate;
+
+ int32 padding; /* to keep alignment of derived types */
} PlanState;
/* ----------------
@@ -1208,14 +1219,21 @@ struct AppendState
PlanState ps; /* its first field is NodeTag */
PlanState **appendplans; /* array of PlanStates for my inputs */
int as_nplans;
- int as_whichplan;
+ int as_whichsyncplan; /* which sync plan is being executed */
int as_first_partial_plan; /* Index of 'appendplans' containing
* the first partial plan */
+ int as_nasyncplans; /* # of async-capable children */
ParallelAppendState *as_pstate; /* parallel coordination info */
Size pstate_len; /* size of parallel coordination info */
struct PartitionPruneState *as_prune_state;
- Bitmapset *as_valid_subplans;
+ Bitmapset *as_valid_syncsubplans;
bool (*choose_next_subplan) (AppendState *);
+ bool as_syncdone; /* all synchronous plans done? */
+ Bitmapset *as_needrequest; /* async plans needing a new request */
+ Bitmapset *as_pending_async; /* pending async plans */
+ TupleTableSlot **as_asyncresult; /* results of each async plan */
+ int as_nasyncresult; /* # of valid entries in as_asyncresult */
+ bool as_exec_prune; /* runtime pruning needed for async exec? */
};
/* ----------------
@@ -1783,6 +1801,7 @@ typedef struct ForeignScanState
Size pscan_len; /* size of parallel coordination information */
/* use struct pointer to avoid including fdwapi.h here */
struct FdwRoutine *fdwroutine;
+ bool fs_async;
void *fdw_state; /* foreign-data wrapper can keep state here */
} ForeignScanState;
diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h
index 83e01074ed..abad89b327 100644
--- a/src/include/nodes/plannodes.h
+++ b/src/include/nodes/plannodes.h
@@ -135,6 +135,11 @@ typedef struct Plan
bool parallel_aware; /* engage parallel-aware logic? */
bool parallel_safe; /* OK to use as part of parallel plan? */
+ /*
+ * information needed for asynchronous execution
+ */
+ bool async_capable; /* engage asynchronous execution logic? */
+
/*
* Common structural data for all Plan types.
*/
@@ -262,6 +267,10 @@ typedef struct Append
/* Info for run-time subplan pruning; NULL if we're not doing that */
struct PartitionPruneInfo *part_prune_info;
+
+ /* Async child node execution stuff */
+ int nasyncplans; /* # async subplans, always at start of list */
+ int referent; /* index of inheritance tree referent */
} Append;
/* ----------------
diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h
index 10b6e81079..53876b2d8b 100644
--- a/src/include/optimizer/paths.h
+++ b/src/include/optimizer/paths.h
@@ -241,4 +241,6 @@ extern PathKey *make_canonical_pathkey(PlannerInfo *root,
extern void add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel,
List *live_childrels);
+extern bool is_async_capable_path(Path *path);
+
#endif /* PATHS_H */
diff --git a/src/include/pgstat.h b/src/include/pgstat.h
index 0dfbac46b4..d673f9da6b 100644
--- a/src/include/pgstat.h
+++ b/src/include/pgstat.h
@@ -887,7 +887,8 @@ typedef enum
WAIT_EVENT_REPLICATION_SLOT_DROP,
WAIT_EVENT_SAFE_SNAPSHOT,
WAIT_EVENT_SYNC_REP,
- WAIT_EVENT_XACT_GROUP_UPDATE
+ WAIT_EVENT_XACT_GROUP_UPDATE,
+ WAIT_EVENT_ASYNC_WAIT
} WaitEventIPC;
/* ----------
--
2.18.4
----Next_Part(Thu_Oct__1_13_43_31_2020_721)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline; filename="v7-0003-async-postgres_fdw.patch"
^ permalink raw reply [nested|flat] 12+ messages in thread
* Re: Move defaults toward ICU in 16?
@ 2023-02-02 13:44 Robert Haas <[email protected]>
2023-02-02 16:31 ` Re: Move defaults toward ICU in 16? Jeff Davis <[email protected]>
0 siblings, 1 reply; 12+ messages in thread
From: Robert Haas @ 2023-02-02 13:44 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: pgsql-hackers
On Thu, Feb 2, 2023 at 8:13 AM Jeff Davis <[email protected]> wrote:
> If we don't want to nudge users toward ICU, is it because we are
> waiting for something, or is there a lack of consensus that ICU is
> actually better?
Do you think it's better?
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 12+ messages in thread
* Re: Move defaults toward ICU in 16?
2023-02-02 13:44 Re: Move defaults toward ICU in 16? Robert Haas <[email protected]>
@ 2023-02-02 16:31 ` Jeff Davis <[email protected]>
2023-02-02 22:14 ` Re: Move defaults toward ICU in 16? Thomas Munro <[email protected]>
0 siblings, 1 reply; 12+ messages in thread
From: Jeff Davis @ 2023-02-02 16:31 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: pgsql-hackers
On Thu, 2023-02-02 at 08:44 -0500, Robert Haas wrote:
> On Thu, Feb 2, 2023 at 8:13 AM Jeff Davis <[email protected]> wrote:
> > If we don't want to nudge users toward ICU, is it because we are
> > waiting for something, or is there a lack of consensus that ICU is
> > actually better?
>
> Do you think it's better?
Yes:
* ICU more featureful: e.g. supports case-insensitive collations (the
citext docs suggest looking at ICU instead).
* It's faster: a simple non-contrived sort is something like 70%
faster[1] than one using glibc.
* It can provide consistent semantics across platforms.
I believe the above reasons are enough to call ICU "better", but it
also seems like a better path for addressing/mitigating collator
versioning problems:
* Easier for users to control what library version is available on
their system. We can also ask packagers to keep some old versions of
ICU available for an extended period of time.
* If one of the ICU multilib patches makes it in, it will be easier
for users to select which of the library versions Postgres will use.
* Reports versions for indiividual collators, distinct from the
library version.
The biggest disadvantage (rather, the flip side of its advantages) is
that it's a separate dependency. Will ICU still be maintained in 10
years or will we end up stuck maintaining it ourselves? Then again,
we've already been shipping it, so I don't know if we can avoid that
problem entirely now even if we wanted to.
I don't mean that ICU solves all of our problems -- far from it. But
you asked if I think it's better, and my answer is yes.
Regards,
Jeff Davis
[1]
https://postgr.es/m/[email protected]
^ permalink raw reply [nested|flat] 12+ messages in thread
* Re: Move defaults toward ICU in 16?
2023-02-02 13:44 Re: Move defaults toward ICU in 16? Robert Haas <[email protected]>
2023-02-02 16:31 ` Re: Move defaults toward ICU in 16? Jeff Davis <[email protected]>
@ 2023-02-02 22:14 ` Thomas Munro <[email protected]>
2023-02-02 22:48 ` Re: Move defaults toward ICU in 16? Peter Geoghegan <[email protected]>
2023-02-02 23:10 ` Re: Move defaults toward ICU in 16? Tom Lane <[email protected]>
0 siblings, 2 replies; 12+ messages in thread
From: Thomas Munro @ 2023-02-02 22:14 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: Robert Haas <[email protected]>; pgsql-hackers
On Fri, Feb 3, 2023 at 5:31 AM Jeff Davis <[email protected]> wrote:
> On Thu, 2023-02-02 at 08:44 -0500, Robert Haas wrote:
> > On Thu, Feb 2, 2023 at 8:13 AM Jeff Davis <[email protected]> wrote:
> > > If we don't want to nudge users toward ICU, is it because we are
> > > waiting for something, or is there a lack of consensus that ICU is
> > > actually better?
> >
> > Do you think it's better?
>
> Yes:
>
> * ICU more featureful: e.g. supports case-insensitive collations (the
> citext docs suggest looking at ICU instead).
> * It's faster: a simple non-contrived sort is something like 70%
> faster[1] than one using glibc.
> * It can provide consistent semantics across platforms.
+1
> * Easier for users to control what library version is available on
> their system. We can also ask packagers to keep some old versions of
> ICU available for an extended period of time.
> * If one of the ICU multilib patches makes it in, it will be easier
> for users to select which of the library versions Postgres will use.
> * Reports versions for indiividual collators, distinct from the
> library version.
+1
> The biggest disadvantage (rather, the flip side of its advantages) is
> that it's a separate dependency. Will ICU still be maintained in 10
> years or will we end up stuck maintaining it ourselves? Then again,
> we've already been shipping it, so I don't know if we can avoid that
> problem entirely now even if we wanted to.
It has a pretty special status, with an absolutely enormous amount of
technology depending on it.
http://blog.unicode.org/2016/05/icu-joins-unicode-consortium.html
https://unicode.org/consortium/consort.html
https://home.unicode.org/membership/members/
https://home.unicode.org/about-unicode/
I mean, who knows what the future holds, but ultimately what we're
doing here is taking the de facto reference implementation of the
Unicode collation algorithm. Are Unicode and the consortium still
going to be here in 10 years? We're all in on Unicode, and it's also
tangled up with ISO standards, as are parts of the collation stuff.
Sure, there could be a clean-room implementation that replaces it in
some sense (just as there is a Java implementation) but it would very
likely be "the same" because the real thing we're buying here is the
set of algorithms and data maintenance that the whole industry has
agreed on.
Unless Britain decides to exit the Latin alphabet, terminate
membership of ISO and revert to anglo-saxon runes with a sort order
that is defined in the new constitution as "the opposite of whatever
Unicode says", it's hard to see obstacles to ICU's long term universal
applicability.
It's still important to have libc support as an option, though,
because it's a totally reasonable thing to want sort order to agree
with the "sort" command on the same host, and you are willing to deal
with all the complexities that we're trying to escape.
^ permalink raw reply [nested|flat] 12+ messages in thread
* Re: Move defaults toward ICU in 16?
2023-02-02 13:44 Re: Move defaults toward ICU in 16? Robert Haas <[email protected]>
2023-02-02 16:31 ` Re: Move defaults toward ICU in 16? Jeff Davis <[email protected]>
2023-02-02 22:14 ` Re: Move defaults toward ICU in 16? Thomas Munro <[email protected]>
@ 2023-02-02 22:48 ` Peter Geoghegan <[email protected]>
1 sibling, 0 replies; 12+ messages in thread
From: Peter Geoghegan @ 2023-02-02 22:48 UTC (permalink / raw)
To: Thomas Munro <[email protected]>; +Cc: Jeff Davis <[email protected]>; Robert Haas <[email protected]>; pgsql-hackers
On Thu, Feb 2, 2023 at 2:15 PM Thomas Munro <[email protected]> wrote:
> Sure, there could be a clean-room implementation that replaces it in
> some sense (just as there is a Java implementation) but it would very
> likely be "the same" because the real thing we're buying here is the
> set of algorithms and data maintenance that the whole industry has
> agreed on.
I don't think that a clean room implementation is implausible. They
seem to already exist, and be explicitly provided for by CLDR, which
is not joined at the hip to ICU:
https://github.com/elixir-cldr/cldr
Most of the value that we tend to think of as coming from ICU actually
comes from CLDR itself, as well as related Unicode Consortium and IETF
standards/RFCs such as BCP-47.
> Unless Britain decides to exit the Latin alphabet, terminate
> membership of ISO and revert to anglo-saxon runes with a sort order
> that is defined in the new constitution as "the opposite of whatever
> Unicode says", it's hard to see obstacles to ICU's long term universal
> applicability.
It would have to literally be defined as "not unicode" for it to
present a real problem. A key goal of Unicode is to accommodate
political and cultural shifts, since even countries can come and go.
In principle Unicode should be able to accommodate just about any
change in preferences, except when there is an irreconcilable
difference of opinion among people that are from the same natural
language group. For example it can accommodate relatively minor
differences of opinion about how text should be sorted among groups
that each speak a regional dialect of the same language. Hardly
anybody even notices this.
Accommodating these variations can only come from making a huge
investment. Most of the work is actually done by natural language
scholars, not technologists. That effort is very unlikely to be
duplicated by some other group with its own conflicting goals. AFAICT
there is no great need for any schisms, since differences of opinion
can usually be accommodated under the umbrella of Unicode.
--
Peter Geoghegan
^ permalink raw reply [nested|flat] 12+ messages in thread
* Re: Move defaults toward ICU in 16?
2023-02-02 13:44 Re: Move defaults toward ICU in 16? Robert Haas <[email protected]>
2023-02-02 16:31 ` Re: Move defaults toward ICU in 16? Jeff Davis <[email protected]>
2023-02-02 22:14 ` Re: Move defaults toward ICU in 16? Thomas Munro <[email protected]>
@ 2023-02-02 23:10 ` Tom Lane <[email protected]>
2023-02-08 20:16 ` Re: Move defaults toward ICU in 16? Jeff Davis <[email protected]>
1 sibling, 1 reply; 12+ messages in thread
From: Tom Lane @ 2023-02-02 23:10 UTC (permalink / raw)
To: Thomas Munro <[email protected]>; +Cc: Jeff Davis <[email protected]>; Robert Haas <[email protected]>; pgsql-hackers
Thomas Munro <[email protected]> writes:
> It's still important to have libc support as an option, though,
> because it's a totally reasonable thing to want sort order to agree
> with the "sort" command on the same host, and you are willing to deal
> with all the complexities that we're trying to escape.
Yeah. I would be resistant to making ICU a required dependency,
but it doesn't seem unreasonable to start moving towards it being
our default collation support.
regards, tom lane
^ permalink raw reply [nested|flat] 12+ messages in thread
* Re: Move defaults toward ICU in 16?
2023-02-02 13:44 Re: Move defaults toward ICU in 16? Robert Haas <[email protected]>
2023-02-02 16:31 ` Re: Move defaults toward ICU in 16? Jeff Davis <[email protected]>
2023-02-02 22:14 ` Re: Move defaults toward ICU in 16? Thomas Munro <[email protected]>
2023-02-02 23:10 ` Re: Move defaults toward ICU in 16? Tom Lane <[email protected]>
@ 2023-02-08 20:16 ` Jeff Davis <[email protected]>
2023-02-09 02:22 ` Re: Move defaults toward ICU in 16? Andres Freund <[email protected]>
0 siblings, 1 reply; 12+ messages in thread
From: Jeff Davis @ 2023-02-08 20:16 UTC (permalink / raw)
To: Tom Lane <[email protected]>; Thomas Munro <[email protected]>; +Cc: Robert Haas <[email protected]>; pgsql-hackers
On Thu, 2023-02-02 at 18:10 -0500, Tom Lane wrote:
> Yeah. I would be resistant to making ICU a required dependency,
> but it doesn't seem unreasonable to start moving towards it being
> our default collation support.
Patch attached.
To get the default locale, the patch initializes a UCollator with NULL
for the locale name, and then queries it for the locale name. Then it's
converted to a language tag, which is consistent with the initial
collation import. I'm not sure that's the best way, but it seems
reasonable.
If it's a user-provided locale (--icu-locale=), then the patch leaves
it as-is, and does not convert it to a language tag (consistent with
CREATE COLLATION and CREATE DATABASE).
I opened another discussion about whether we want to try harder to
validate or canonicalize the locale name:
https://www.postgresql.org/message-id/[email protected]
--
Jeff Davis
PostgreSQL Contributor Team - AWS
Attachments:
[text/x-patch] v1-0001-Use-ICU-by-default-at-initdb-time.patch (3.1K, ../../[email protected]/2-v1-0001-Use-ICU-by-default-at-initdb-time.patch)
download | inline diff:
From 1b7d940c0f12062185b8b42bf8d3c0a6f05a74d4 Mon Sep 17 00:00:00 2001
From: Jeff Davis <[email protected]>
Date: Wed, 8 Feb 2023 12:06:26 -0800
Subject: [PATCH v1] Use ICU by default at initdb time.
If the ICU locale is not specified, initialize the default collator
and retrieve the locale name from that.
Discussion: https://postgr.es/m/[email protected]
---
src/bin/initdb/initdb.c | 74 +++++++++++++++++++++++++++++++++++++++--
1 file changed, 72 insertions(+), 2 deletions(-)
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index 7a58c33ace..7321652db3 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -53,6 +53,7 @@
#include <netdb.h>
#include <sys/socket.h>
#include <sys/stat.h>
+#include <unicode/ucol.h>
#include <unistd.h>
#include <signal.h>
#include <time.h>
@@ -133,7 +134,11 @@ static char *lc_monetary = NULL;
static char *lc_numeric = NULL;
static char *lc_time = NULL;
static char *lc_messages = NULL;
+#ifdef USE_ICU
+static char locale_provider = COLLPROVIDER_ICU;
+#else
static char locale_provider = COLLPROVIDER_LIBC;
+#endif
static char *icu_locale = NULL;
static const char *default_text_search_config = NULL;
static char *username = NULL;
@@ -2024,6 +2029,72 @@ check_icu_locale_encoding(int user_enc)
return true;
}
+/*
+ * Check that ICU accepts the locale name; or if not specified, retrieve the
+ * default ICU locale.
+ */
+static void
+check_icu_locale()
+{
+#ifdef USE_ICU
+ UCollator *collator;
+ UErrorCode status;
+
+ status = U_ZERO_ERROR;
+ collator = ucol_open(icu_locale, &status);
+ if (U_FAILURE(status))
+ {
+ if (icu_locale)
+ pg_fatal("ICU locale \"%s\" could not be opened: %s",
+ icu_locale, u_errorName(status));
+ else
+ pg_fatal("default ICU locale could not be opened: %s",
+ u_errorName(status));
+ }
+
+ /* if not specified, get locale from default collator */
+ if (icu_locale == NULL)
+ {
+ const char *default_locale;
+
+ status = U_ZERO_ERROR;
+ default_locale = ucol_getLocaleByType(collator, ULOC_VALID_LOCALE,
+ &status);
+ if (U_FAILURE(status))
+ {
+ ucol_close(collator);
+ pg_fatal("could not determine default ICU locale");
+ }
+
+ if (U_ICU_VERSION_MAJOR_NUM >= 54)
+ {
+ const bool strict = true;
+ char *langtag;
+ int len;
+
+ len = uloc_toLanguageTag(default_locale, NULL, 0, strict, &status);
+ langtag = pg_malloc(len + 1);
+ status = U_ZERO_ERROR;
+ uloc_toLanguageTag(default_locale, langtag, len + 1, strict,
+ &status);
+
+ if (U_FAILURE(status))
+ {
+ ucol_close(collator);
+ pg_fatal("could not determine language tag for default locale \"%s\": %s",
+ default_locale, u_errorName(status));
+ }
+
+ icu_locale = langtag;
+ }
+ else
+ icu_locale = pg_strdup(default_locale);
+ }
+
+ ucol_close(collator);
+#endif
+}
+
/*
* set up the locale variables
*
@@ -2077,8 +2148,7 @@ setlocales(void)
if (locale_provider == COLLPROVIDER_ICU)
{
- if (!icu_locale)
- pg_fatal("ICU locale must be specified");
+ check_icu_locale();
/*
* In supported builds, the ICU locale ID will be checked by the
--
2.34.1
^ permalink raw reply [nested|flat] 12+ messages in thread
* Re: Move defaults toward ICU in 16?
2023-02-02 13:44 Re: Move defaults toward ICU in 16? Robert Haas <[email protected]>
2023-02-02 16:31 ` Re: Move defaults toward ICU in 16? Jeff Davis <[email protected]>
2023-02-02 22:14 ` Re: Move defaults toward ICU in 16? Thomas Munro <[email protected]>
2023-02-02 23:10 ` Re: Move defaults toward ICU in 16? Tom Lane <[email protected]>
2023-02-08 20:16 ` Re: Move defaults toward ICU in 16? Jeff Davis <[email protected]>
@ 2023-02-09 02:22 ` Andres Freund <[email protected]>
2023-02-11 00:17 ` Re: Move defaults toward ICU in 16? Jeff Davis <[email protected]>
0 siblings, 1 reply; 12+ messages in thread
From: Andres Freund @ 2023-02-09 02:22 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: Tom Lane <[email protected]>; Thomas Munro <[email protected]>; Robert Haas <[email protected]>; pgsql-hackers
On 2023-02-08 12:16:46 -0800, Jeff Davis wrote:
> On Thu, 2023-02-02 at 18:10 -0500, Tom Lane wrote:
> > Yeah. I would be resistant to making ICU a required dependency,
> > but it doesn't seem unreasonable to start moving towards it being
> > our default collation support.
>
> Patch attached.
Unfortunately this fails widely on CI, with both compile time and runtime
issues:
https://cirrus-ci.com/build/5116408950947840
^ permalink raw reply [nested|flat] 12+ messages in thread
* Re: Move defaults toward ICU in 16?
2023-02-02 13:44 Re: Move defaults toward ICU in 16? Robert Haas <[email protected]>
2023-02-02 16:31 ` Re: Move defaults toward ICU in 16? Jeff Davis <[email protected]>
2023-02-02 22:14 ` Re: Move defaults toward ICU in 16? Thomas Munro <[email protected]>
2023-02-02 23:10 ` Re: Move defaults toward ICU in 16? Tom Lane <[email protected]>
2023-02-08 20:16 ` Re: Move defaults toward ICU in 16? Jeff Davis <[email protected]>
2023-02-09 02:22 ` Re: Move defaults toward ICU in 16? Andres Freund <[email protected]>
@ 2023-02-11 00:17 ` Jeff Davis <[email protected]>
2023-02-11 02:00 ` Re: Move defaults toward ICU in 16? Andres Freund <[email protected]>
0 siblings, 1 reply; 12+ messages in thread
From: Jeff Davis @ 2023-02-11 00:17 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Tom Lane <[email protected]>; Thomas Munro <[email protected]>; Robert Haas <[email protected]>; pgsql-hackers
On Wed, 2023-02-08 at 18:22 -0800, Andres Freund wrote:
> On 2023-02-08 12:16:46 -0800, Jeff Davis wrote:
> > On Thu, 2023-02-02 at 18:10 -0500, Tom Lane wrote:
> > > Yeah. I would be resistant to making ICU a required dependency,
> > > but it doesn't seem unreasonable to start moving towards it being
> > > our default collation support.
> >
> > Patch attached.
>
> Unfortunately this fails widely on CI, with both compile time and
> runtime
New patches attached.
0001: build defaults to requiring ICU
0002: initdb defaults to using ICU (if server built with ICU)
One CI test is failing: "Windows - Server 2019, VS 2019 - Meson &
ninja"; if I apply Andres patch (
https://github.com/anarazel/postgres/commit/dde7c68 ), then it works.
I ran into one annoyance with pg_upgrade, which is that a v15 cluster
initialized with the defaults requires that the v16 cluster is
initialized with --locale-provider=libc, because otherwise the old and
new cluster will have mismatching template databases. Simple to fix
once you see the error, but I wonder how many initdb scripts might be
broken? I suppose it's just the cost of changing a default? Would an
environment variable help for cases where it's difficult to pass that
extra option down through a script?
I also considered posting another patch to change the default for
CREATE COLLATION, but there are a few issues I'm not sure about. Should
the default be based on whether ICU support is available? Or the
datlocprovider for the current database? And/or some kind of
compatibility GUC?
Notes on the tests I needed to fix, in case they are interesting or
point to some kind of larger problem:
* ecpg has a test that involves setting the client_encoding to LATIN1
which required a compatible server encoding so it was setting
ENCODING=SQL_ASCII, which ICU doesn't support. The ecpg test did not
look particularly sensitive to the locale, so I changed it to use
client_encoding=SQL_ASCII instead, so that the server encoding doesn't
matter.
* citext has a test involving Turkish characters, which works for all
libc locales, but in ICU the test only works in Turkish locales. I skip
the test if datlocprovider='i', because citext doesn't seem very
important in an ICU world.
* unaccent is broken if the database provider is ICU and LC_CTYPE=C,
because the t_isspace() (etc.) functions do not properly handle ICU.
Probably some other things are broken with that combination, but only
this test seems to exercise it. I just skipped the test for that broken
combination, but perhaps it should be fixed in the future.
* initdb was being built with ICU as a dependency in meson, but not
autoconf. I assume it's fine to link ICU into initdb, so I changed the
Makefile.
* I changed a couple tests to initialize with --locale-provider=libc.
They were testing that creating a database with the ICU provider but no
ICU locale fails, and that's easiest to test if the template is libc.
* The CI test CompilerWarnings:mingw_cross_warning was failing because
ICU is not available. I added --without-icu in the .cirrus.yml file and
it works.
--
Jeff Davis
PostgreSQL Contributor Team - AWS
Attachments:
[text/x-patch] v2-0001-Build-ICU-support-by-default.patch (12.9K, ../../[email protected]/2-v2-0001-Build-ICU-support-by-default.patch)
download | inline diff:
From b1772b12be3c47a00a3723d2937421cb5bbfe3a3 Mon Sep 17 00:00:00 2001
From: Jeff Davis <[email protected]>
Date: Fri, 10 Feb 2023 12:08:11 -0800
Subject: [PATCH v2 1/2] Build ICU support by default.
Discussion: https://postgr.es/m/[email protected]
---
.cirrus.yml | 1 +
configure | 46 ++++++++------------
configure.ac | 8 +++-
doc/src/sgml/installation.sgml | 78 +++++++++++++++++++---------------
meson.build | 12 +++++-
meson_options.txt | 2 +-
6 files changed, 78 insertions(+), 69 deletions(-)
diff --git a/.cirrus.yml b/.cirrus.yml
index 6b98980075..24302efea6 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -775,6 +775,7 @@ task:
time ./configure \
--host=x86_64-w64-mingw32 \
--enable-cassert \
+ --without-icu \
CC="ccache x86_64-w64-mingw32-gcc" \
CXX="ccache x86_64-w64-mingw32-g++"
make -s -j${BUILD_JOBS} clean
diff --git a/configure b/configure
index 5d07fd0bb9..ddb75f7f4b 100755
--- a/configure
+++ b/configure
@@ -1558,7 +1558,7 @@ Optional Packages:
set WAL block size in kB [8]
--with-CC=CMD set compiler (deprecated)
--with-llvm build with LLVM based JIT support
- --with-icu build with ICU support
+ --without-icu build without ICU support
--with-tcl build Tcl modules (PL/Tcl)
--with-tclconfig=DIR tclConfig.sh is in DIR
--with-perl build Perl modules (PL/Perl)
@@ -8401,7 +8401,9 @@ $as_echo "#define USE_ICU 1" >>confdefs.h
esac
else
- with_icu=no
+ with_icu=yes
+
+$as_echo "#define USE_ICU 1" >>confdefs.h
fi
@@ -8470,31 +8472,17 @@ fi
# Put the nasty error message in config.log where it belongs
echo "$ICU_PKG_ERRORS" >&5
- as_fn_error $? "Package requirements (icu-uc icu-i18n) were not met:
-
-$ICU_PKG_ERRORS
-
-Consider adjusting the PKG_CONFIG_PATH environment variable if you
-installed software in a non-standard prefix.
-
-Alternatively, you may set the environment variables ICU_CFLAGS
-and ICU_LIBS to avoid the need to call pkg-config.
-See the pkg-config man page for more details." "$LINENO" 5
+ as_fn_error $? "ICU library not found
+If you have ICU already installed, see config.log for details on the
+failure. It is possible the compiler isn't looking in the proper directory.
+Use --without-icu to disable ICU support." "$LINENO" 5
elif test $pkg_failed = untried; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
- { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
-$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
-is in your PATH or set the PKG_CONFIG environment variable to the full
-path to pkg-config.
-
-Alternatively, you may set the environment variables ICU_CFLAGS
-and ICU_LIBS to avoid the need to call pkg-config.
-See the pkg-config man page for more details.
-
-To get pkg-config, see <http://pkg-config.freedesktop.org/>.
-See \`config.log' for more details" "$LINENO" 5; }
+ as_fn_error $? "ICU library not found
+If you have ICU already installed, see config.log for details on the
+failure. It is possible the compiler isn't looking in the proper directory.
+Use --without-icu to disable ICU support." "$LINENO" 5
else
ICU_CFLAGS=$pkg_cv_ICU_CFLAGS
ICU_LIBS=$pkg_cv_ICU_LIBS
@@ -15323,7 +15311,7 @@ else
We can't simply define LARGE_OFF_T to be 9223372036854775807,
since some C++ compilers masquerading as C compilers
incorrectly reject 9223372036854775807. */
-#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
+#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
&& LARGE_OFF_T % 2147483647 == 1)
? 1 : -1];
@@ -15369,7 +15357,7 @@ else
We can't simply define LARGE_OFF_T to be 9223372036854775807,
since some C++ compilers masquerading as C compilers
incorrectly reject 9223372036854775807. */
-#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
+#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
&& LARGE_OFF_T % 2147483647 == 1)
? 1 : -1];
@@ -15393,7 +15381,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
We can't simply define LARGE_OFF_T to be 9223372036854775807,
since some C++ compilers masquerading as C compilers
incorrectly reject 9223372036854775807. */
-#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
+#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
&& LARGE_OFF_T % 2147483647 == 1)
? 1 : -1];
@@ -15438,7 +15426,7 @@ else
We can't simply define LARGE_OFF_T to be 9223372036854775807,
since some C++ compilers masquerading as C compilers
incorrectly reject 9223372036854775807. */
-#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
+#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
&& LARGE_OFF_T % 2147483647 == 1)
? 1 : -1];
@@ -15462,7 +15450,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
We can't simply define LARGE_OFF_T to be 9223372036854775807,
since some C++ compilers masquerading as C compilers
incorrectly reject 9223372036854775807. */
-#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
+#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
&& LARGE_OFF_T % 2147483647 == 1)
? 1 : -1];
diff --git a/configure.ac b/configure.ac
index e9b74ced6c..909f5dba3c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -853,13 +853,17 @@ AC_SUBST(enable_thread_safety)
# ICU
#
AC_MSG_CHECKING([whether to build with ICU support])
-PGAC_ARG_BOOL(with, icu, no, [build with ICU support],
+PGAC_ARG_BOOL(with, icu, yes, [build without ICU support],
[AC_DEFINE([USE_ICU], 1, [Define to build with ICU support. (--with-icu)])])
AC_MSG_RESULT([$with_icu])
AC_SUBST(with_icu)
if test "$with_icu" = yes; then
- PKG_CHECK_MODULES(ICU, icu-uc icu-i18n)
+ PKG_CHECK_MODULES(ICU, icu-uc icu-i18n, [],
+ [AC_MSG_ERROR([ICU library not found
+If you have ICU already installed, see config.log for details on the
+failure. It is possible the compiler isn't looking in the proper directory.
+Use --without-icu to disable ICU support.])])
fi
#
diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml
index 6619e69462..b5f88d27df 100644
--- a/doc/src/sgml/installation.sgml
+++ b/doc/src/sgml/installation.sgml
@@ -146,6 +146,35 @@ documentation. See standalone-profile.xsl for details.
<application>pg_restore</application>.
</para>
</listitem>
+
+ <listitem>
+ <para>
+ The ICU locale provider (see <xref linkend="locale-providers"/>) is used by default. If you don't want to use it then you must specify the <option>--without-icu</option> option to <filename>configure</filename>. Using this option disables support for ICU collation features (see <xref linkend="collation"/>).
+ </para>
+ <para>
+ ICU support requires the <productname>ICU4C</productname> package to be
+ installed. The minimum required version of
+ <productname>ICU4C</productname> is currently 4.2.
+ </para>
+
+ <para>
+ By default,
+ <productname>pkg-config</productname><indexterm><primary>pkg-config</primary></indexterm>
+ will be used to find the required compilation options. This is
+ supported for <productname>ICU4C</productname> version 4.6 and later.
+ For older versions, or if <productname>pkg-config</productname> is not
+ available, the variables <envar>ICU_CFLAGS</envar> and
+ <envar>ICU_LIBS</envar> can be specified to
+ <filename>configure</filename>, like in this example:
+<programlisting>
+./configure ... ICU_CFLAGS='-I/some/where/include' ICU_LIBS='-L/some/where/lib -licui18n -licuuc -licudata'
+</programlisting>
+ (If <productname>ICU4C</productname> is in the default search path
+ for the compiler, then you still need to specify nonempty strings in
+ order to avoid use of <productname>pkg-config</productname>, for
+ example, <literal>ICU_CFLAGS=' '</literal>.)
+ </para>
+ </listitem>
</itemizedlist>
</para>
@@ -926,40 +955,6 @@ build-postgresql:
</listitem>
</varlistentry>
- <varlistentry id="configure-option-with-icu">
- <term><option>--with-icu</option></term>
- <listitem>
- <para>
- Build with support for
- the <productname>ICU</productname><indexterm><primary>ICU</primary></indexterm>
- library, enabling use of ICU collation
- features<phrase condition="standalone-ignore"> (see
- <xref linkend="collation"/>)</phrase>.
- This requires the <productname>ICU4C</productname> package
- to be installed. The minimum required version
- of <productname>ICU4C</productname> is currently 4.2.
- </para>
-
- <para>
- By default,
- <productname>pkg-config</productname><indexterm><primary>pkg-config</primary></indexterm>
- will be used to find the required compilation options. This is
- supported for <productname>ICU4C</productname> version 4.6 and later.
- For older versions, or if <productname>pkg-config</productname> is
- not available, the variables <envar>ICU_CFLAGS</envar>
- and <envar>ICU_LIBS</envar> can be specified
- to <filename>configure</filename>, like in this example:
-<programlisting>
-./configure ... --with-icu ICU_CFLAGS='-I/some/where/include' ICU_LIBS='-L/some/where/lib -licui18n -licuuc -licudata'
-</programlisting>
- (If <productname>ICU4C</productname> is in the default search path
- for the compiler, then you still need to specify nonempty strings in
- order to avoid use of <productname>pkg-config</productname>, for
- example, <literal>ICU_CFLAGS=' '</literal>.)
- </para>
- </listitem>
- </varlistentry>
-
<varlistentry id="configure-with-llvm">
<term><option>--with-llvm</option></term>
<listitem>
@@ -1231,6 +1226,19 @@ build-postgresql:
<variablelist>
+ <varlistentry id="configure-option-without-icu">
+ <term><option>--without-icu</option></term>
+ <listitem>
+ <para>
+ Build without support for the
+ <productname>ICU</productname><indexterm><primary>ICU</primary></indexterm>
+ library, disabling the use of ICU collation features<phrase
+ condition="standalone-ignore"> (see <xref
+ linkend="collation"/>)</phrase>.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry id="configure-option-without-readline">
<term><option>--without-readline</option></term>
<listitem>
@@ -2419,7 +2427,7 @@ ninja install
<productname>ICU</productname><indexterm><primary>ICU</primary></indexterm>
library, enabling use of ICU collation features<phrase
condition="standalone-ignore"> (see <xref
- linkend="collation"/>)</phrase>. Defaults to auto and requires the
+ linkend="collation"/>)</phrase>. Defaults to enabled and requires the
<productname>ICU4C</productname> package to be installed. The minimum
required version of <productname>ICU4C</productname> is currently 4.2.
</para>
diff --git a/meson.build b/meson.build
index e379a252a5..9cd7491fda 100644
--- a/meson.build
+++ b/meson.build
@@ -721,8 +721,16 @@ endif
icuopt = get_option('icu')
if not icuopt.disabled()
- icu = dependency('icu-uc', required: icuopt.enabled())
- icu_i18n = dependency('icu-i18n', required: icuopt.enabled())
+ icu = dependency('icu-uc', required: false)
+ icu_i18n = dependency('icu-i18n', required: false)
+
+ icu_found = icu.found() and icu_i18n.found()
+ if icuopt.enabled() and not icu_found
+ error('''ICU library not found
+If you have ICU already installed, see meson-log/meson-log.txt for details on the
+failure. It is possible the compiler isn't looking in the proper directory.
+Use -Dicu=disabled to disable ICU support.''')
+ endif
if icu.found()
cdata.set('USE_ICU', 1)
diff --git a/meson_options.txt b/meson_options.txt
index 9d3ef4aa20..fbd4e69356 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -82,7 +82,7 @@ option('dtrace', type : 'feature', value: 'disabled',
option('gssapi', type : 'feature', value: 'auto',
description: 'GSSAPI support')
-option('icu', type : 'feature', value: 'auto',
+option('icu', type : 'feature', value: 'enabled',
description: 'ICU support')
option('ldap', type : 'feature', value: 'auto',
--
2.34.1
[text/x-patch] v2-0002-Use-ICU-by-default-at-initdb-time.patch (17.6K, ../../[email protected]/3-v2-0002-Use-ICU-by-default-at-initdb-time.patch)
download | inline diff:
From 64ad3a3b0c8bcdf95df90a9db05edb255a3f18c2 Mon Sep 17 00:00:00 2001
From: Jeff Davis <[email protected]>
Date: Wed, 8 Feb 2023 12:06:26 -0800
Subject: [PATCH v2 2/2] Use ICU by default at initdb time.
If the ICU locale is not specified, initialize the default collator
and retrieve the locale name from that.
Discussion: https://postgr.es/m/[email protected]
---
contrib/citext/expected/citext_utf8.out | 4 +-
contrib/citext/expected/citext_utf8_1.out | 4 +-
contrib/citext/sql/citext_utf8.sql | 4 +-
contrib/unaccent/expected/unaccent.out | 9 +++
contrib/unaccent/expected/unaccent_1.out | 8 ++
contrib/unaccent/sql/unaccent.sql | 11 +++
doc/src/sgml/ref/initdb.sgml | 48 +++++++-----
src/bin/initdb/Makefile | 4 +-
src/bin/initdb/initdb.c | 76 ++++++++++++++++++-
src/bin/initdb/t/001_initdb.pl | 7 +-
src/bin/pg_dump/t/002_pg_dump.pl | 2 +-
src/bin/scripts/t/020_createdb.pl | 2 +-
src/interfaces/ecpg/test/Makefile | 3 -
src/interfaces/ecpg/test/connect/test5.pgc | 2 +-
.../ecpg/test/expected/connect-test5.c | 2 +-
.../ecpg/test/expected/connect-test5.stderr | 2 +-
src/interfaces/ecpg/test/meson.build | 1 -
src/test/icu/t/010_database.pl | 2 +-
18 files changed, 149 insertions(+), 42 deletions(-)
create mode 100644 contrib/unaccent/expected/unaccent_1.out
diff --git a/contrib/citext/expected/citext_utf8.out b/contrib/citext/expected/citext_utf8.out
index 666b07ccec..85ce9c3b64 100644
--- a/contrib/citext/expected/citext_utf8.out
+++ b/contrib/citext/expected/citext_utf8.out
@@ -3,7 +3,9 @@
* and a Unicode-aware locale.
*/
SELECT getdatabaseencoding() <> 'UTF8' OR
- current_setting('lc_ctype') = 'C'
+ current_setting('lc_ctype') = 'C' OR
+ (SELECT datlocprovider='i' FROM pg_database
+ WHERE datname=current_database())
AS skip_test \gset
\if :skip_test
\quit
diff --git a/contrib/citext/expected/citext_utf8_1.out b/contrib/citext/expected/citext_utf8_1.out
index 433e985349..60ddebd841 100644
--- a/contrib/citext/expected/citext_utf8_1.out
+++ b/contrib/citext/expected/citext_utf8_1.out
@@ -3,7 +3,9 @@
* and a Unicode-aware locale.
*/
SELECT getdatabaseencoding() <> 'UTF8' OR
- current_setting('lc_ctype') = 'C'
+ current_setting('lc_ctype') = 'C' OR
+ (SELECT datlocprovider='i' FROM pg_database
+ WHERE datname=current_database())
AS skip_test \gset
\if :skip_test
\quit
diff --git a/contrib/citext/sql/citext_utf8.sql b/contrib/citext/sql/citext_utf8.sql
index d068000b42..e9c504e764 100644
--- a/contrib/citext/sql/citext_utf8.sql
+++ b/contrib/citext/sql/citext_utf8.sql
@@ -4,7 +4,9 @@
*/
SELECT getdatabaseencoding() <> 'UTF8' OR
- current_setting('lc_ctype') = 'C'
+ current_setting('lc_ctype') = 'C' OR
+ (SELECT datlocprovider='i' FROM pg_database
+ WHERE datname=current_database())
AS skip_test \gset
\if :skip_test
\quit
diff --git a/contrib/unaccent/expected/unaccent.out b/contrib/unaccent/expected/unaccent.out
index ee0ac71a1c..cef98ee60c 100644
--- a/contrib/unaccent/expected/unaccent.out
+++ b/contrib/unaccent/expected/unaccent.out
@@ -1,3 +1,12 @@
+-- unaccent is broken if the default collation is provided by ICU and
+-- LC_CTYPE=C
+SELECT current_setting('lc_ctype') = 'C' AND
+ (SELECT datlocprovider='i' FROM pg_database
+ WHERE datname=current_database())
+ AS skip_test \gset
+\if :skip_test
+\quit
+\endif
CREATE EXTENSION unaccent;
-- must have a UTF8 database
SELECT getdatabaseencoding();
diff --git a/contrib/unaccent/expected/unaccent_1.out b/contrib/unaccent/expected/unaccent_1.out
new file mode 100644
index 0000000000..0a4a3838ab
--- /dev/null
+++ b/contrib/unaccent/expected/unaccent_1.out
@@ -0,0 +1,8 @@
+-- unaccent is broken if the default collation is provided by ICU and
+-- LC_CTYPE=C
+SELECT current_setting('lc_ctype') = 'C' AND
+ (SELECT datlocprovider='i' FROM pg_database
+ WHERE datname=current_database())
+ AS skip_test \gset
+\if :skip_test
+\quit
diff --git a/contrib/unaccent/sql/unaccent.sql b/contrib/unaccent/sql/unaccent.sql
index 3fc0c706be..027dfb964a 100644
--- a/contrib/unaccent/sql/unaccent.sql
+++ b/contrib/unaccent/sql/unaccent.sql
@@ -1,3 +1,14 @@
+
+-- unaccent is broken if the default collation is provided by ICU and
+-- LC_CTYPE=C
+SELECT current_setting('lc_ctype') = 'C' AND
+ (SELECT datlocprovider='i' FROM pg_database
+ WHERE datname=current_database())
+ AS skip_test \gset
+\if :skip_test
+\quit
+\endif
+
CREATE EXTENSION unaccent;
-- must have a UTF8 database
diff --git a/doc/src/sgml/ref/initdb.sgml b/doc/src/sgml/ref/initdb.sgml
index 5b2bdac101..4f37386ea3 100644
--- a/doc/src/sgml/ref/initdb.sgml
+++ b/doc/src/sgml/ref/initdb.sgml
@@ -89,10 +89,28 @@ PostgreSQL documentation
and character set encoding. These can also be set separately for each
database when it is created. <command>initdb</command> determines those
settings for the template databases, which will serve as the default for
- all other databases. By default, <command>initdb</command> uses the
- locale provider <literal>libc</literal>, takes the locale settings from
- the environment, and determines the encoding from the locale settings.
- This is almost always sufficient, unless there are special requirements.
+ all other databases.
+ </para>
+
+ <para>
+ By default, <command>initdb</command> uses the ICU library to provide
+ locale services if the server was built with ICU support; otherwise it uses
+ the <literal>libc</literal> locale provider (see <xref
+ linkend="locale-providers"/>). To choose the specific ICU locale ID to
+ apply, use the option <option>--icu-locale</option>. Note that for
+ implementation reasons and to support legacy code,
+ <command>initdb</command> will still select and initialize libc locale
+ settings when the ICU locale provider is used.
+ </para>
+
+ <para>
+ Alternatively, <command>initdb</command> can use the locale provider
+ <literal>libc</literal>. To select this option, specify
+ <literal>--locale-provider=libc</literal>, or build the server without ICU
+ support. The <literal>libc</literal> locale provider takes the locale
+ settings from the environment, and determines the encoding from the locale
+ settings. This is almost always sufficient, unless there are special
+ requirements.
</para>
<para>
@@ -103,17 +121,6 @@ PostgreSQL documentation
categories can give nonsensical results, so this should be used with care.
</para>
- <para>
- Alternatively, the ICU library can be used to provide locale services.
- (Again, this only sets the default for subsequently created databases.) To
- select this option, specify <literal>--locale-provider=icu</literal>.
- To choose the specific ICU locale ID to apply, use the option
- <option>--icu-locale</option>. Note that
- for implementation reasons and to support legacy code,
- <command>initdb</command> will still select and initialize libc locale
- settings when the ICU locale provider is used.
- </para>
-
<para>
When <command>initdb</command> runs, it will print out the locale settings
it has chosen. If you have complex requirements or specified multiple
@@ -234,7 +241,8 @@ PostgreSQL documentation
<term><option>--icu-locale=<replaceable>locale</replaceable></option></term>
<listitem>
<para>
- Specifies the ICU locale ID, if the ICU locale provider is used.
+ Specifies the ICU locale ID, if the ICU locale provider is used. By
+ default, ICU obtains the ICU locale from the ICU default collator.
</para>
</listitem>
</varlistentry>
@@ -297,10 +305,12 @@ PostgreSQL documentation
<term><option>--locale-provider={<literal>libc</literal>|<literal>icu</literal>}</option></term>
<listitem>
<para>
- This option sets the locale provider for databases created in the
- new cluster. It can be overridden in the <command>CREATE
+ This option sets the locale provider for databases created in the new
+ cluster. It can be overridden in the <command>CREATE
DATABASE</command> command when new databases are subsequently
- created. The default is <literal>libc</literal>.
+ created. The default is <literal>icu</literal> if the server was
+ built with ICU support; otherwise the default is
+ <literal>libc</literal> (see <xref linkend="locale-providers"/>).
</para>
</listitem>
</varlistentry>
diff --git a/src/bin/initdb/Makefile b/src/bin/initdb/Makefile
index eab89c5501..d69bd89572 100644
--- a/src/bin/initdb/Makefile
+++ b/src/bin/initdb/Makefile
@@ -16,7 +16,7 @@ subdir = src/bin/initdb
top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
-override CPPFLAGS := -I$(libpq_srcdir) -I$(top_srcdir)/src/timezone $(CPPFLAGS)
+override CPPFLAGS := -I$(libpq_srcdir) -I$(top_srcdir)/src/timezone $(ICU_CFLAGS) $(CPPFLAGS)
# Note: it's important that we link to encnames.o from libpgcommon, not
# from libpq, else we have risks of version skew if we run with a libpq
@@ -24,7 +24,7 @@ override CPPFLAGS := -I$(libpq_srcdir) -I$(top_srcdir)/src/timezone $(CPPFLAGS)
# should ensure that that happens.
#
# We need libpq only because fe_utils does.
-LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
+LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport) $(ICU_LIBS)
# use system timezone data?
ifneq (,$(with_system_tzdata))
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index 7a58c33ace..0776294499 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -53,6 +53,9 @@
#include <netdb.h>
#include <sys/socket.h>
#include <sys/stat.h>
+#ifdef USE_ICU
+#include <unicode/ucol.h>
+#endif
#include <unistd.h>
#include <signal.h>
#include <time.h>
@@ -133,7 +136,11 @@ static char *lc_monetary = NULL;
static char *lc_numeric = NULL;
static char *lc_time = NULL;
static char *lc_messages = NULL;
+#ifdef USE_ICU
+static char locale_provider = COLLPROVIDER_ICU;
+#else
static char locale_provider = COLLPROVIDER_LIBC;
+#endif
static char *icu_locale = NULL;
static const char *default_text_search_config = NULL;
static char *username = NULL;
@@ -2024,6 +2031,72 @@ check_icu_locale_encoding(int user_enc)
return true;
}
+/*
+ * Check that ICU accepts the locale name; or if not specified, retrieve the
+ * default ICU locale.
+ */
+static void
+check_icu_locale()
+{
+#ifdef USE_ICU
+ UCollator *collator;
+ UErrorCode status;
+
+ status = U_ZERO_ERROR;
+ collator = ucol_open(icu_locale, &status);
+ if (U_FAILURE(status))
+ {
+ if (icu_locale)
+ pg_fatal("could not open collator for locale \"%s\": %s",
+ icu_locale, u_errorName(status));
+ else
+ pg_fatal("could not open collator for default locale: %s",
+ u_errorName(status));
+ }
+
+ /* if not specified, get locale from default collator */
+ if (icu_locale == NULL)
+ {
+ const char *default_locale;
+
+ status = U_ZERO_ERROR;
+ default_locale = ucol_getLocaleByType(collator, ULOC_VALID_LOCALE,
+ &status);
+ if (U_FAILURE(status))
+ {
+ ucol_close(collator);
+ pg_fatal("could not determine default ICU locale");
+ }
+
+ if (U_ICU_VERSION_MAJOR_NUM >= 54)
+ {
+ const bool strict = true;
+ char *langtag;
+ int len;
+
+ len = uloc_toLanguageTag(default_locale, NULL, 0, strict, &status);
+ langtag = pg_malloc(len + 1);
+ status = U_ZERO_ERROR;
+ uloc_toLanguageTag(default_locale, langtag, len + 1, strict,
+ &status);
+
+ if (U_FAILURE(status))
+ {
+ ucol_close(collator);
+ pg_fatal("could not determine language tag for default locale \"%s\": %s",
+ default_locale, u_errorName(status));
+ }
+
+ icu_locale = langtag;
+ }
+ else
+ icu_locale = pg_strdup(default_locale);
+ }
+
+ ucol_close(collator);
+#endif
+}
+
/*
* set up the locale variables
*
@@ -2077,8 +2150,7 @@ setlocales(void)
if (locale_provider == COLLPROVIDER_ICU)
{
- if (!icu_locale)
- pg_fatal("ICU locale must be specified");
+ check_icu_locale();
/*
* In supported builds, the ICU locale ID will be checked by the
diff --git a/src/bin/initdb/t/001_initdb.pl b/src/bin/initdb/t/001_initdb.pl
index 772769acab..e5d214e09c 100644
--- a/src/bin/initdb/t/001_initdb.pl
+++ b/src/bin/initdb/t/001_initdb.pl
@@ -97,11 +97,6 @@ SKIP:
if ($ENV{with_icu} eq 'yes')
{
- command_fails_like(
- [ 'initdb', '--no-sync', '--locale-provider=icu', "$tempdir/data2" ],
- qr/initdb: error: ICU locale must be specified/,
- 'locale provider ICU requires --icu-locale');
-
command_ok(
[
'initdb', '--no-sync',
@@ -116,7 +111,7 @@ if ($ENV{with_icu} eq 'yes')
'--locale-provider=icu', '--icu-locale=@colNumeric=lower',
"$tempdir/dataX"
],
- qr/FATAL: could not open collator for locale/,
+ qr/error: could not open collator for locale/,
'fails for invalid ICU locale');
command_fails_like(
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index d92247c915..30294f381c 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -1684,7 +1684,7 @@ my %tests = (
create_sql =>
"CREATE DATABASE dump_test2 LOCALE = 'C' TEMPLATE = template0;",
regexp => qr/^
- \QCREATE DATABASE dump_test2 \E.*\QLOCALE = 'C';\E
+ \QCREATE DATABASE dump_test2 \E.*\QLOCALE = 'C'\E
/xm,
like => { pg_dumpall_dbprivs => 1, },
},
diff --git a/src/bin/scripts/t/020_createdb.pl b/src/bin/scripts/t/020_createdb.pl
index 3ad4fbb00c..8ec58cdd64 100644
--- a/src/bin/scripts/t/020_createdb.pl
+++ b/src/bin/scripts/t/020_createdb.pl
@@ -13,7 +13,7 @@ program_version_ok('createdb');
program_options_handling_ok('createdb');
my $node = PostgreSQL::Test::Cluster->new('main');
-$node->init;
+$node->init(extra => ['--locale-provider=libc']);
$node->start;
$node->issues_sql_like(
diff --git a/src/interfaces/ecpg/test/Makefile b/src/interfaces/ecpg/test/Makefile
index d7a7d1d1ca..cf841a3a5b 100644
--- a/src/interfaces/ecpg/test/Makefile
+++ b/src/interfaces/ecpg/test/Makefile
@@ -14,9 +14,6 @@ override CPPFLAGS := \
'-DSHELLPROG="$(SHELL)"' \
$(CPPFLAGS)
-# default encoding for regression tests
-ENCODING = SQL_ASCII
-
ifneq ($(build_os),mingw32)
abs_builddir := $(shell pwd)
else
diff --git a/src/interfaces/ecpg/test/connect/test5.pgc b/src/interfaces/ecpg/test/connect/test5.pgc
index de29160089..d512553677 100644
--- a/src/interfaces/ecpg/test/connect/test5.pgc
+++ b/src/interfaces/ecpg/test/connect/test5.pgc
@@ -55,7 +55,7 @@ exec sql end declare section;
exec sql connect to 'unix:postgresql://localhost/ecpg2_regression' as main user :user USING "connectpw";
exec sql disconnect main;
- exec sql connect to unix:postgresql://localhost/ecpg2_regression?connect_timeout=180&client_encoding=latin1 as main user regress_ecpg_user1/connectpw;
+ exec sql connect to unix:postgresql://localhost/ecpg2_regression?connect_timeout=180&client_encoding=sql_ascii as main user regress_ecpg_user1/connectpw;
exec sql disconnect main;
exec sql connect to "unix:postgresql://200.46.204.71/ecpg2_regression" as main user regress_ecpg_user1/connectpw;
diff --git a/src/interfaces/ecpg/test/expected/connect-test5.c b/src/interfaces/ecpg/test/expected/connect-test5.c
index c1124c627f..ec1514ed9a 100644
--- a/src/interfaces/ecpg/test/expected/connect-test5.c
+++ b/src/interfaces/ecpg/test/expected/connect-test5.c
@@ -117,7 +117,7 @@ main(void)
#line 56 "test5.pgc"
- { ECPGconnect(__LINE__, 0, "unix:postgresql://localhost/ecpg2_regression?connect_timeout=180 & client_encoding=latin1" , "regress_ecpg_user1" , "connectpw" , "main", 0); }
+ { ECPGconnect(__LINE__, 0, "unix:postgresql://localhost/ecpg2_regression?connect_timeout=180 & client_encoding=sql_ascii" , "regress_ecpg_user1" , "connectpw" , "main", 0); }
#line 58 "test5.pgc"
{ ECPGdisconnect(__LINE__, "main");}
diff --git a/src/interfaces/ecpg/test/expected/connect-test5.stderr b/src/interfaces/ecpg/test/expected/connect-test5.stderr
index 01a6a0a13b..51cc18916a 100644
--- a/src/interfaces/ecpg/test/expected/connect-test5.stderr
+++ b/src/interfaces/ecpg/test/expected/connect-test5.stderr
@@ -50,7 +50,7 @@
[NO_PID]: sqlca: code: 0, state: 00000
[NO_PID]: ecpg_finish: connection main closed
[NO_PID]: sqlca: code: 0, state: 00000
-[NO_PID]: ECPGconnect: opening database ecpg2_regression on <DEFAULT> port <DEFAULT> with options connect_timeout=180 & client_encoding=latin1 for user regress_ecpg_user1
+[NO_PID]: ECPGconnect: opening database ecpg2_regression on <DEFAULT> port <DEFAULT> with options connect_timeout=180 & client_encoding=sql_ascii for user regress_ecpg_user1
[NO_PID]: sqlca: code: 0, state: 00000
[NO_PID]: ecpg_finish: connection main closed
[NO_PID]: sqlca: code: 0, state: 00000
diff --git a/src/interfaces/ecpg/test/meson.build b/src/interfaces/ecpg/test/meson.build
index d0be73ccf9..04c6819a79 100644
--- a/src/interfaces/ecpg/test/meson.build
+++ b/src/interfaces/ecpg/test/meson.build
@@ -69,7 +69,6 @@ ecpg_test_files = files(
ecpg_regress_args = [
'--dbname=ecpg1_regression,ecpg2_regression',
'--create-role=regress_ecpg_user1,regress_ecpg_user2',
- '--encoding=SQL_ASCII',
]
tests += {
diff --git a/src/test/icu/t/010_database.pl b/src/test/icu/t/010_database.pl
index 80ab1c7789..45d77c319a 100644
--- a/src/test/icu/t/010_database.pl
+++ b/src/test/icu/t/010_database.pl
@@ -12,7 +12,7 @@ if ($ENV{with_icu} ne 'yes')
}
my $node1 = PostgreSQL::Test::Cluster->new('node1');
-$node1->init;
+$node1->init(extra => ['--locale-provider=libc']);
$node1->start;
$node1->safe_psql('postgres',
--
2.34.1
^ permalink raw reply [nested|flat] 12+ messages in thread
* Re: Move defaults toward ICU in 16?
2023-02-02 13:44 Re: Move defaults toward ICU in 16? Robert Haas <[email protected]>
2023-02-02 16:31 ` Re: Move defaults toward ICU in 16? Jeff Davis <[email protected]>
2023-02-02 22:14 ` Re: Move defaults toward ICU in 16? Thomas Munro <[email protected]>
2023-02-02 23:10 ` Re: Move defaults toward ICU in 16? Tom Lane <[email protected]>
2023-02-08 20:16 ` Re: Move defaults toward ICU in 16? Jeff Davis <[email protected]>
2023-02-09 02:22 ` Re: Move defaults toward ICU in 16? Andres Freund <[email protected]>
2023-02-11 00:17 ` Re: Move defaults toward ICU in 16? Jeff Davis <[email protected]>
@ 2023-02-11 02:00 ` Andres Freund <[email protected]>
2023-02-14 17:48 ` Re: Move defaults toward ICU in 16? Jeff Davis <[email protected]>
0 siblings, 1 reply; 12+ messages in thread
From: Andres Freund @ 2023-02-11 02:00 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: Tom Lane <[email protected]>; Thomas Munro <[email protected]>; Robert Haas <[email protected]>; pgsql-hackers
Hi,
On 2023-02-10 16:17:00 -0800, Jeff Davis wrote:
> One CI test is failing: "Windows - Server 2019, VS 2019 - Meson &
> ninja"; if I apply Andres patch (
> https://github.com/anarazel/postgres/commit/dde7c68 ), then it works.
Until something like my patch above is done more generally applicable, I think
your patch should disable ICU on windows. Can't just fail to build.
Perhaps we don't need to force ICU use to on with the meson build, given that
it defaults to auto-detection?
> I ran into one annoyance with pg_upgrade, which is that a v15 cluster
> initialized with the defaults requires that the v16 cluster is
> initialized with --locale-provider=libc, because otherwise the old and
> new cluster will have mismatching template databases. Simple to fix
> once you see the error, but I wonder how many initdb scripts might be
> broken? I suppose it's just the cost of changing a default? Would an
> environment variable help for cases where it's difficult to pass that
> extra option down through a script?
That seems problematic to me.
But, shouldn't pg_upgrade be able to deal with this? As long as the databases
are created with template0, we can create the collations at that point?
> @@ -15323,7 +15311,7 @@ else
> We can't simply define LARGE_OFF_T to be 9223372036854775807,
> since some C++ compilers masquerading as C compilers
> incorrectly reject 9223372036854775807. */
> -#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
> +#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
> int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
> && LARGE_OFF_T % 2147483647 == 1)
> ? 1 : -1];
This stuff shouldn't be in here, it's due to a debian patched autoconf.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 12+ messages in thread
* Re: Move defaults toward ICU in 16?
2023-02-02 13:44 Re: Move defaults toward ICU in 16? Robert Haas <[email protected]>
2023-02-02 16:31 ` Re: Move defaults toward ICU in 16? Jeff Davis <[email protected]>
2023-02-02 22:14 ` Re: Move defaults toward ICU in 16? Thomas Munro <[email protected]>
2023-02-02 23:10 ` Re: Move defaults toward ICU in 16? Tom Lane <[email protected]>
2023-02-08 20:16 ` Re: Move defaults toward ICU in 16? Jeff Davis <[email protected]>
2023-02-09 02:22 ` Re: Move defaults toward ICU in 16? Andres Freund <[email protected]>
2023-02-11 00:17 ` Re: Move defaults toward ICU in 16? Jeff Davis <[email protected]>
2023-02-11 02:00 ` Re: Move defaults toward ICU in 16? Andres Freund <[email protected]>
@ 2023-02-14 17:48 ` Jeff Davis <[email protected]>
2023-02-14 17:59 ` Re: Move defaults toward ICU in 16? Andres Freund <[email protected]>
0 siblings, 1 reply; 12+ messages in thread
From: Jeff Davis @ 2023-02-14 17:48 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Tom Lane <[email protected]>; Thomas Munro <[email protected]>; Robert Haas <[email protected]>; pgsql-hackers
On Fri, 2023-02-10 at 18:00 -0800, Andres Freund wrote:
> Until something like my patch above is done more generally
> applicable, I think
> your patch should disable ICU on windows. Can't just fail to build.
>
> Perhaps we don't need to force ICU use to on with the meson build,
> given that
> it defaults to auto-detection?
Done. I changed it back to 'auto', and tests pass.
>
> But, shouldn't pg_upgrade be able to deal with this? As long as the
> databases
> are created with template0, we can create the collations at that
> point?
Are you saying that the upgraded cluster could have a different default
collation for the template databases than the original cluster?
That would be wrong to do, at least by default, but I could see it
being a useful option.
Or maybe I misunderstand what you're saying?
>
> This stuff shouldn't be in here, it's due to a debian patched
> autoconf.
Removed, thank you.
--
Jeff Davis
PostgreSQL Contributor Team - AWS
Attachments:
[text/x-patch] v3-0001-Build-ICU-support-by-default.patch (8.6K, ../../[email protected]/2-v3-0001-Build-ICU-support-by-default.patch)
download | inline diff:
From 0a691bdc1952871b2ec8d1c5086c90c8943d99cb Mon Sep 17 00:00:00 2001
From: Jeff Davis <[email protected]>
Date: Fri, 10 Feb 2023 12:08:11 -0800
Subject: [PATCH v3 1/2] Build ICU support by default.
Discussion: https://postgr.es/m/[email protected]
---
.cirrus.yml | 1 +
configure | 36 ++++++----------
configure.ac | 8 +++-
doc/src/sgml/installation.sgml | 76 +++++++++++++++++++---------------
4 files changed, 61 insertions(+), 60 deletions(-)
diff --git a/.cirrus.yml b/.cirrus.yml
index f212978752..34450a9c7b 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -751,6 +751,7 @@ task:
time ./configure \
--host=x86_64-w64-mingw32 \
--enable-cassert \
+ --without-icu \
CC="ccache x86_64-w64-mingw32-gcc" \
CXX="ccache x86_64-w64-mingw32-g++"
make -s -j${BUILD_JOBS} clean
diff --git a/configure b/configure
index 5d07fd0bb9..507ca3c983 100755
--- a/configure
+++ b/configure
@@ -1558,7 +1558,7 @@ Optional Packages:
set WAL block size in kB [8]
--with-CC=CMD set compiler (deprecated)
--with-llvm build with LLVM based JIT support
- --with-icu build with ICU support
+ --without-icu build without ICU support
--with-tcl build Tcl modules (PL/Tcl)
--with-tclconfig=DIR tclConfig.sh is in DIR
--with-perl build Perl modules (PL/Perl)
@@ -8401,7 +8401,9 @@ $as_echo "#define USE_ICU 1" >>confdefs.h
esac
else
- with_icu=no
+ with_icu=yes
+
+$as_echo "#define USE_ICU 1" >>confdefs.h
fi
@@ -8470,31 +8472,17 @@ fi
# Put the nasty error message in config.log where it belongs
echo "$ICU_PKG_ERRORS" >&5
- as_fn_error $? "Package requirements (icu-uc icu-i18n) were not met:
-
-$ICU_PKG_ERRORS
-
-Consider adjusting the PKG_CONFIG_PATH environment variable if you
-installed software in a non-standard prefix.
-
-Alternatively, you may set the environment variables ICU_CFLAGS
-and ICU_LIBS to avoid the need to call pkg-config.
-See the pkg-config man page for more details." "$LINENO" 5
+ as_fn_error $? "ICU library not found
+If you have ICU already installed, see config.log for details on the
+failure. It is possible the compiler isn't looking in the proper directory.
+Use --without-icu to disable ICU support." "$LINENO" 5
elif test $pkg_failed = untried; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
- { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
-$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
-as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
-is in your PATH or set the PKG_CONFIG environment variable to the full
-path to pkg-config.
-
-Alternatively, you may set the environment variables ICU_CFLAGS
-and ICU_LIBS to avoid the need to call pkg-config.
-See the pkg-config man page for more details.
-
-To get pkg-config, see <http://pkg-config.freedesktop.org/>.
-See \`config.log' for more details" "$LINENO" 5; }
+ as_fn_error $? "ICU library not found
+If you have ICU already installed, see config.log for details on the
+failure. It is possible the compiler isn't looking in the proper directory.
+Use --without-icu to disable ICU support." "$LINENO" 5
else
ICU_CFLAGS=$pkg_cv_ICU_CFLAGS
ICU_LIBS=$pkg_cv_ICU_LIBS
diff --git a/configure.ac b/configure.ac
index e9b74ced6c..909f5dba3c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -853,13 +853,17 @@ AC_SUBST(enable_thread_safety)
# ICU
#
AC_MSG_CHECKING([whether to build with ICU support])
-PGAC_ARG_BOOL(with, icu, no, [build with ICU support],
+PGAC_ARG_BOOL(with, icu, yes, [build without ICU support],
[AC_DEFINE([USE_ICU], 1, [Define to build with ICU support. (--with-icu)])])
AC_MSG_RESULT([$with_icu])
AC_SUBST(with_icu)
if test "$with_icu" = yes; then
- PKG_CHECK_MODULES(ICU, icu-uc icu-i18n)
+ PKG_CHECK_MODULES(ICU, icu-uc icu-i18n, [],
+ [AC_MSG_ERROR([ICU library not found
+If you have ICU already installed, see config.log for details on the
+failure. It is possible the compiler isn't looking in the proper directory.
+Use --without-icu to disable ICU support.])])
fi
#
diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml
index 6619e69462..bad6bc6e5e 100644
--- a/doc/src/sgml/installation.sgml
+++ b/doc/src/sgml/installation.sgml
@@ -146,6 +146,35 @@ documentation. See standalone-profile.xsl for details.
<application>pg_restore</application>.
</para>
</listitem>
+
+ <listitem>
+ <para>
+ The ICU locale provider (see <xref linkend="locale-providers"/>) is used by default. If you don't want to use it then you must specify the <option>--without-icu</option> option to <filename>configure</filename>. Using this option disables support for ICU collation features (see <xref linkend="collation"/>).
+ </para>
+ <para>
+ ICU support requires the <productname>ICU4C</productname> package to be
+ installed. The minimum required version of
+ <productname>ICU4C</productname> is currently 4.2.
+ </para>
+
+ <para>
+ By default,
+ <productname>pkg-config</productname><indexterm><primary>pkg-config</primary></indexterm>
+ will be used to find the required compilation options. This is
+ supported for <productname>ICU4C</productname> version 4.6 and later.
+ For older versions, or if <productname>pkg-config</productname> is not
+ available, the variables <envar>ICU_CFLAGS</envar> and
+ <envar>ICU_LIBS</envar> can be specified to
+ <filename>configure</filename>, like in this example:
+<programlisting>
+./configure ... ICU_CFLAGS='-I/some/where/include' ICU_LIBS='-L/some/where/lib -licui18n -licuuc -licudata'
+</programlisting>
+ (If <productname>ICU4C</productname> is in the default search path
+ for the compiler, then you still need to specify nonempty strings in
+ order to avoid use of <productname>pkg-config</productname>, for
+ example, <literal>ICU_CFLAGS=' '</literal>.)
+ </para>
+ </listitem>
</itemizedlist>
</para>
@@ -926,40 +955,6 @@ build-postgresql:
</listitem>
</varlistentry>
- <varlistentry id="configure-option-with-icu">
- <term><option>--with-icu</option></term>
- <listitem>
- <para>
- Build with support for
- the <productname>ICU</productname><indexterm><primary>ICU</primary></indexterm>
- library, enabling use of ICU collation
- features<phrase condition="standalone-ignore"> (see
- <xref linkend="collation"/>)</phrase>.
- This requires the <productname>ICU4C</productname> package
- to be installed. The minimum required version
- of <productname>ICU4C</productname> is currently 4.2.
- </para>
-
- <para>
- By default,
- <productname>pkg-config</productname><indexterm><primary>pkg-config</primary></indexterm>
- will be used to find the required compilation options. This is
- supported for <productname>ICU4C</productname> version 4.6 and later.
- For older versions, or if <productname>pkg-config</productname> is
- not available, the variables <envar>ICU_CFLAGS</envar>
- and <envar>ICU_LIBS</envar> can be specified
- to <filename>configure</filename>, like in this example:
-<programlisting>
-./configure ... --with-icu ICU_CFLAGS='-I/some/where/include' ICU_LIBS='-L/some/where/lib -licui18n -licuuc -licudata'
-</programlisting>
- (If <productname>ICU4C</productname> is in the default search path
- for the compiler, then you still need to specify nonempty strings in
- order to avoid use of <productname>pkg-config</productname>, for
- example, <literal>ICU_CFLAGS=' '</literal>.)
- </para>
- </listitem>
- </varlistentry>
-
<varlistentry id="configure-with-llvm">
<term><option>--with-llvm</option></term>
<listitem>
@@ -1231,6 +1226,19 @@ build-postgresql:
<variablelist>
+ <varlistentry id="configure-option-without-icu">
+ <term><option>--without-icu</option></term>
+ <listitem>
+ <para>
+ Build without support for the
+ <productname>ICU</productname><indexterm><primary>ICU</primary></indexterm>
+ library, disabling the use of ICU collation features<phrase
+ condition="standalone-ignore"> (see <xref
+ linkend="collation"/>)</phrase>.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry id="configure-option-without-readline">
<term><option>--without-readline</option></term>
<listitem>
--
2.34.1
[text/x-patch] v3-0002-Use-ICU-by-default-at-initdb-time.patch (17.6K, ../../[email protected]/3-v3-0002-Use-ICU-by-default-at-initdb-time.patch)
download | inline diff:
From 9818b4b8c5ec15aa6cbf0e6db6ae888173885dca Mon Sep 17 00:00:00 2001
From: Jeff Davis <[email protected]>
Date: Wed, 8 Feb 2023 12:06:26 -0800
Subject: [PATCH v3 2/2] Use ICU by default at initdb time.
If the ICU locale is not specified, initialize the default collator
and retrieve the locale name from that.
Discussion: https://postgr.es/m/[email protected]
---
contrib/citext/expected/citext_utf8.out | 4 +-
contrib/citext/expected/citext_utf8_1.out | 4 +-
contrib/citext/sql/citext_utf8.sql | 4 +-
contrib/unaccent/expected/unaccent.out | 9 +++
contrib/unaccent/expected/unaccent_1.out | 8 ++
contrib/unaccent/sql/unaccent.sql | 11 +++
doc/src/sgml/ref/initdb.sgml | 48 +++++++-----
src/bin/initdb/Makefile | 4 +-
src/bin/initdb/initdb.c | 76 ++++++++++++++++++-
src/bin/initdb/t/001_initdb.pl | 7 +-
src/bin/pg_dump/t/002_pg_dump.pl | 2 +-
src/bin/scripts/t/020_createdb.pl | 2 +-
src/interfaces/ecpg/test/Makefile | 3 -
src/interfaces/ecpg/test/connect/test5.pgc | 2 +-
.../ecpg/test/expected/connect-test5.c | 2 +-
.../ecpg/test/expected/connect-test5.stderr | 2 +-
src/interfaces/ecpg/test/meson.build | 1 -
src/test/icu/t/010_database.pl | 2 +-
18 files changed, 149 insertions(+), 42 deletions(-)
create mode 100644 contrib/unaccent/expected/unaccent_1.out
diff --git a/contrib/citext/expected/citext_utf8.out b/contrib/citext/expected/citext_utf8.out
index 666b07ccec..85ce9c3b64 100644
--- a/contrib/citext/expected/citext_utf8.out
+++ b/contrib/citext/expected/citext_utf8.out
@@ -3,7 +3,9 @@
* and a Unicode-aware locale.
*/
SELECT getdatabaseencoding() <> 'UTF8' OR
- current_setting('lc_ctype') = 'C'
+ current_setting('lc_ctype') = 'C' OR
+ (SELECT datlocprovider='i' FROM pg_database
+ WHERE datname=current_database())
AS skip_test \gset
\if :skip_test
\quit
diff --git a/contrib/citext/expected/citext_utf8_1.out b/contrib/citext/expected/citext_utf8_1.out
index 433e985349..60ddebd841 100644
--- a/contrib/citext/expected/citext_utf8_1.out
+++ b/contrib/citext/expected/citext_utf8_1.out
@@ -3,7 +3,9 @@
* and a Unicode-aware locale.
*/
SELECT getdatabaseencoding() <> 'UTF8' OR
- current_setting('lc_ctype') = 'C'
+ current_setting('lc_ctype') = 'C' OR
+ (SELECT datlocprovider='i' FROM pg_database
+ WHERE datname=current_database())
AS skip_test \gset
\if :skip_test
\quit
diff --git a/contrib/citext/sql/citext_utf8.sql b/contrib/citext/sql/citext_utf8.sql
index d068000b42..e9c504e764 100644
--- a/contrib/citext/sql/citext_utf8.sql
+++ b/contrib/citext/sql/citext_utf8.sql
@@ -4,7 +4,9 @@
*/
SELECT getdatabaseencoding() <> 'UTF8' OR
- current_setting('lc_ctype') = 'C'
+ current_setting('lc_ctype') = 'C' OR
+ (SELECT datlocprovider='i' FROM pg_database
+ WHERE datname=current_database())
AS skip_test \gset
\if :skip_test
\quit
diff --git a/contrib/unaccent/expected/unaccent.out b/contrib/unaccent/expected/unaccent.out
index ee0ac71a1c..cef98ee60c 100644
--- a/contrib/unaccent/expected/unaccent.out
+++ b/contrib/unaccent/expected/unaccent.out
@@ -1,3 +1,12 @@
+-- unaccent is broken if the default collation is provided by ICU and
+-- LC_CTYPE=C
+SELECT current_setting('lc_ctype') = 'C' AND
+ (SELECT datlocprovider='i' FROM pg_database
+ WHERE datname=current_database())
+ AS skip_test \gset
+\if :skip_test
+\quit
+\endif
CREATE EXTENSION unaccent;
-- must have a UTF8 database
SELECT getdatabaseencoding();
diff --git a/contrib/unaccent/expected/unaccent_1.out b/contrib/unaccent/expected/unaccent_1.out
new file mode 100644
index 0000000000..0a4a3838ab
--- /dev/null
+++ b/contrib/unaccent/expected/unaccent_1.out
@@ -0,0 +1,8 @@
+-- unaccent is broken if the default collation is provided by ICU and
+-- LC_CTYPE=C
+SELECT current_setting('lc_ctype') = 'C' AND
+ (SELECT datlocprovider='i' FROM pg_database
+ WHERE datname=current_database())
+ AS skip_test \gset
+\if :skip_test
+\quit
diff --git a/contrib/unaccent/sql/unaccent.sql b/contrib/unaccent/sql/unaccent.sql
index 3fc0c706be..027dfb964a 100644
--- a/contrib/unaccent/sql/unaccent.sql
+++ b/contrib/unaccent/sql/unaccent.sql
@@ -1,3 +1,14 @@
+
+-- unaccent is broken if the default collation is provided by ICU and
+-- LC_CTYPE=C
+SELECT current_setting('lc_ctype') = 'C' AND
+ (SELECT datlocprovider='i' FROM pg_database
+ WHERE datname=current_database())
+ AS skip_test \gset
+\if :skip_test
+\quit
+\endif
+
CREATE EXTENSION unaccent;
-- must have a UTF8 database
diff --git a/doc/src/sgml/ref/initdb.sgml b/doc/src/sgml/ref/initdb.sgml
index 5b2bdac101..4f37386ea3 100644
--- a/doc/src/sgml/ref/initdb.sgml
+++ b/doc/src/sgml/ref/initdb.sgml
@@ -89,10 +89,28 @@ PostgreSQL documentation
and character set encoding. These can also be set separately for each
database when it is created. <command>initdb</command> determines those
settings for the template databases, which will serve as the default for
- all other databases. By default, <command>initdb</command> uses the
- locale provider <literal>libc</literal>, takes the locale settings from
- the environment, and determines the encoding from the locale settings.
- This is almost always sufficient, unless there are special requirements.
+ all other databases.
+ </para>
+
+ <para>
+ By default, <command>initdb</command> uses the ICU library to provide
+ locale services if the server was built with ICU support; otherwise it uses
+ the <literal>libc</literal> locale provider (see <xref
+ linkend="locale-providers"/>). To choose the specific ICU locale ID to
+ apply, use the option <option>--icu-locale</option>. Note that for
+ implementation reasons and to support legacy code,
+ <command>initdb</command> will still select and initialize libc locale
+ settings when the ICU locale provider is used.
+ </para>
+
+ <para>
+ Alternatively, <command>initdb</command> can use the locale provider
+ <literal>libc</literal>. To select this option, specify
+ <literal>--locale-provider=libc</literal>, or build the server without ICU
+ support. The <literal>libc</literal> locale provider takes the locale
+ settings from the environment, and determines the encoding from the locale
+ settings. This is almost always sufficient, unless there are special
+ requirements.
</para>
<para>
@@ -103,17 +121,6 @@ PostgreSQL documentation
categories can give nonsensical results, so this should be used with care.
</para>
- <para>
- Alternatively, the ICU library can be used to provide locale services.
- (Again, this only sets the default for subsequently created databases.) To
- select this option, specify <literal>--locale-provider=icu</literal>.
- To choose the specific ICU locale ID to apply, use the option
- <option>--icu-locale</option>. Note that
- for implementation reasons and to support legacy code,
- <command>initdb</command> will still select and initialize libc locale
- settings when the ICU locale provider is used.
- </para>
-
<para>
When <command>initdb</command> runs, it will print out the locale settings
it has chosen. If you have complex requirements or specified multiple
@@ -234,7 +241,8 @@ PostgreSQL documentation
<term><option>--icu-locale=<replaceable>locale</replaceable></option></term>
<listitem>
<para>
- Specifies the ICU locale ID, if the ICU locale provider is used.
+ Specifies the ICU locale ID, if the ICU locale provider is used. By
+ default, ICU obtains the ICU locale from the ICU default collator.
</para>
</listitem>
</varlistentry>
@@ -297,10 +305,12 @@ PostgreSQL documentation
<term><option>--locale-provider={<literal>libc</literal>|<literal>icu</literal>}</option></term>
<listitem>
<para>
- This option sets the locale provider for databases created in the
- new cluster. It can be overridden in the <command>CREATE
+ This option sets the locale provider for databases created in the new
+ cluster. It can be overridden in the <command>CREATE
DATABASE</command> command when new databases are subsequently
- created. The default is <literal>libc</literal>.
+ created. The default is <literal>icu</literal> if the server was
+ built with ICU support; otherwise the default is
+ <literal>libc</literal> (see <xref linkend="locale-providers"/>).
</para>
</listitem>
</varlistentry>
diff --git a/src/bin/initdb/Makefile b/src/bin/initdb/Makefile
index eab89c5501..d69bd89572 100644
--- a/src/bin/initdb/Makefile
+++ b/src/bin/initdb/Makefile
@@ -16,7 +16,7 @@ subdir = src/bin/initdb
top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
-override CPPFLAGS := -I$(libpq_srcdir) -I$(top_srcdir)/src/timezone $(CPPFLAGS)
+override CPPFLAGS := -I$(libpq_srcdir) -I$(top_srcdir)/src/timezone $(ICU_CFLAGS) $(CPPFLAGS)
# Note: it's important that we link to encnames.o from libpgcommon, not
# from libpq, else we have risks of version skew if we run with a libpq
@@ -24,7 +24,7 @@ override CPPFLAGS := -I$(libpq_srcdir) -I$(top_srcdir)/src/timezone $(CPPFLAGS)
# should ensure that that happens.
#
# We need libpq only because fe_utils does.
-LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
+LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport) $(ICU_LIBS)
# use system timezone data?
ifneq (,$(with_system_tzdata))
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index 7a58c33ace..0776294499 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -53,6 +53,9 @@
#include <netdb.h>
#include <sys/socket.h>
#include <sys/stat.h>
+#ifdef USE_ICU
+#include <unicode/ucol.h>
+#endif
#include <unistd.h>
#include <signal.h>
#include <time.h>
@@ -133,7 +136,11 @@ static char *lc_monetary = NULL;
static char *lc_numeric = NULL;
static char *lc_time = NULL;
static char *lc_messages = NULL;
+#ifdef USE_ICU
+static char locale_provider = COLLPROVIDER_ICU;
+#else
static char locale_provider = COLLPROVIDER_LIBC;
+#endif
static char *icu_locale = NULL;
static const char *default_text_search_config = NULL;
static char *username = NULL;
@@ -2024,6 +2031,72 @@ check_icu_locale_encoding(int user_enc)
return true;
}
+/*
+ * Check that ICU accepts the locale name; or if not specified, retrieve the
+ * default ICU locale.
+ */
+static void
+check_icu_locale()
+{
+#ifdef USE_ICU
+ UCollator *collator;
+ UErrorCode status;
+
+ status = U_ZERO_ERROR;
+ collator = ucol_open(icu_locale, &status);
+ if (U_FAILURE(status))
+ {
+ if (icu_locale)
+ pg_fatal("could not open collator for locale \"%s\": %s",
+ icu_locale, u_errorName(status));
+ else
+ pg_fatal("could not open collator for default locale: %s",
+ u_errorName(status));
+ }
+
+ /* if not specified, get locale from default collator */
+ if (icu_locale == NULL)
+ {
+ const char *default_locale;
+
+ status = U_ZERO_ERROR;
+ default_locale = ucol_getLocaleByType(collator, ULOC_VALID_LOCALE,
+ &status);
+ if (U_FAILURE(status))
+ {
+ ucol_close(collator);
+ pg_fatal("could not determine default ICU locale");
+ }
+
+ if (U_ICU_VERSION_MAJOR_NUM >= 54)
+ {
+ const bool strict = true;
+ char *langtag;
+ int len;
+
+ len = uloc_toLanguageTag(default_locale, NULL, 0, strict, &status);
+ langtag = pg_malloc(len + 1);
+ status = U_ZERO_ERROR;
+ uloc_toLanguageTag(default_locale, langtag, len + 1, strict,
+ &status);
+
+ if (U_FAILURE(status))
+ {
+ ucol_close(collator);
+ pg_fatal("could not determine language tag for default locale \"%s\": %s",
+ default_locale, u_errorName(status));
+ }
+
+ icu_locale = langtag;
+ }
+ else
+ icu_locale = pg_strdup(default_locale);
+ }
+
+ ucol_close(collator);
+#endif
+}
+
/*
* set up the locale variables
*
@@ -2077,8 +2150,7 @@ setlocales(void)
if (locale_provider == COLLPROVIDER_ICU)
{
- if (!icu_locale)
- pg_fatal("ICU locale must be specified");
+ check_icu_locale();
/*
* In supported builds, the ICU locale ID will be checked by the
diff --git a/src/bin/initdb/t/001_initdb.pl b/src/bin/initdb/t/001_initdb.pl
index 772769acab..e5d214e09c 100644
--- a/src/bin/initdb/t/001_initdb.pl
+++ b/src/bin/initdb/t/001_initdb.pl
@@ -97,11 +97,6 @@ SKIP:
if ($ENV{with_icu} eq 'yes')
{
- command_fails_like(
- [ 'initdb', '--no-sync', '--locale-provider=icu', "$tempdir/data2" ],
- qr/initdb: error: ICU locale must be specified/,
- 'locale provider ICU requires --icu-locale');
-
command_ok(
[
'initdb', '--no-sync',
@@ -116,7 +111,7 @@ if ($ENV{with_icu} eq 'yes')
'--locale-provider=icu', '--icu-locale=@colNumeric=lower',
"$tempdir/dataX"
],
- qr/FATAL: could not open collator for locale/,
+ qr/error: could not open collator for locale/,
'fails for invalid ICU locale');
command_fails_like(
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index d92247c915..30294f381c 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -1684,7 +1684,7 @@ my %tests = (
create_sql =>
"CREATE DATABASE dump_test2 LOCALE = 'C' TEMPLATE = template0;",
regexp => qr/^
- \QCREATE DATABASE dump_test2 \E.*\QLOCALE = 'C';\E
+ \QCREATE DATABASE dump_test2 \E.*\QLOCALE = 'C'\E
/xm,
like => { pg_dumpall_dbprivs => 1, },
},
diff --git a/src/bin/scripts/t/020_createdb.pl b/src/bin/scripts/t/020_createdb.pl
index 3ad4fbb00c..8ec58cdd64 100644
--- a/src/bin/scripts/t/020_createdb.pl
+++ b/src/bin/scripts/t/020_createdb.pl
@@ -13,7 +13,7 @@ program_version_ok('createdb');
program_options_handling_ok('createdb');
my $node = PostgreSQL::Test::Cluster->new('main');
-$node->init;
+$node->init(extra => ['--locale-provider=libc']);
$node->start;
$node->issues_sql_like(
diff --git a/src/interfaces/ecpg/test/Makefile b/src/interfaces/ecpg/test/Makefile
index d7a7d1d1ca..cf841a3a5b 100644
--- a/src/interfaces/ecpg/test/Makefile
+++ b/src/interfaces/ecpg/test/Makefile
@@ -14,9 +14,6 @@ override CPPFLAGS := \
'-DSHELLPROG="$(SHELL)"' \
$(CPPFLAGS)
-# default encoding for regression tests
-ENCODING = SQL_ASCII
-
ifneq ($(build_os),mingw32)
abs_builddir := $(shell pwd)
else
diff --git a/src/interfaces/ecpg/test/connect/test5.pgc b/src/interfaces/ecpg/test/connect/test5.pgc
index de29160089..d512553677 100644
--- a/src/interfaces/ecpg/test/connect/test5.pgc
+++ b/src/interfaces/ecpg/test/connect/test5.pgc
@@ -55,7 +55,7 @@ exec sql end declare section;
exec sql connect to 'unix:postgresql://localhost/ecpg2_regression' as main user :user USING "connectpw";
exec sql disconnect main;
- exec sql connect to unix:postgresql://localhost/ecpg2_regression?connect_timeout=180&client_encoding=latin1 as main user regress_ecpg_user1/connectpw;
+ exec sql connect to unix:postgresql://localhost/ecpg2_regression?connect_timeout=180&client_encoding=sql_ascii as main user regress_ecpg_user1/connectpw;
exec sql disconnect main;
exec sql connect to "unix:postgresql://200.46.204.71/ecpg2_regression" as main user regress_ecpg_user1/connectpw;
diff --git a/src/interfaces/ecpg/test/expected/connect-test5.c b/src/interfaces/ecpg/test/expected/connect-test5.c
index c1124c627f..ec1514ed9a 100644
--- a/src/interfaces/ecpg/test/expected/connect-test5.c
+++ b/src/interfaces/ecpg/test/expected/connect-test5.c
@@ -117,7 +117,7 @@ main(void)
#line 56 "test5.pgc"
- { ECPGconnect(__LINE__, 0, "unix:postgresql://localhost/ecpg2_regression?connect_timeout=180 & client_encoding=latin1" , "regress_ecpg_user1" , "connectpw" , "main", 0); }
+ { ECPGconnect(__LINE__, 0, "unix:postgresql://localhost/ecpg2_regression?connect_timeout=180 & client_encoding=sql_ascii" , "regress_ecpg_user1" , "connectpw" , "main", 0); }
#line 58 "test5.pgc"
{ ECPGdisconnect(__LINE__, "main");}
diff --git a/src/interfaces/ecpg/test/expected/connect-test5.stderr b/src/interfaces/ecpg/test/expected/connect-test5.stderr
index 01a6a0a13b..51cc18916a 100644
--- a/src/interfaces/ecpg/test/expected/connect-test5.stderr
+++ b/src/interfaces/ecpg/test/expected/connect-test5.stderr
@@ -50,7 +50,7 @@
[NO_PID]: sqlca: code: 0, state: 00000
[NO_PID]: ecpg_finish: connection main closed
[NO_PID]: sqlca: code: 0, state: 00000
-[NO_PID]: ECPGconnect: opening database ecpg2_regression on <DEFAULT> port <DEFAULT> with options connect_timeout=180 & client_encoding=latin1 for user regress_ecpg_user1
+[NO_PID]: ECPGconnect: opening database ecpg2_regression on <DEFAULT> port <DEFAULT> with options connect_timeout=180 & client_encoding=sql_ascii for user regress_ecpg_user1
[NO_PID]: sqlca: code: 0, state: 00000
[NO_PID]: ecpg_finish: connection main closed
[NO_PID]: sqlca: code: 0, state: 00000
diff --git a/src/interfaces/ecpg/test/meson.build b/src/interfaces/ecpg/test/meson.build
index d0be73ccf9..04c6819a79 100644
--- a/src/interfaces/ecpg/test/meson.build
+++ b/src/interfaces/ecpg/test/meson.build
@@ -69,7 +69,6 @@ ecpg_test_files = files(
ecpg_regress_args = [
'--dbname=ecpg1_regression,ecpg2_regression',
'--create-role=regress_ecpg_user1,regress_ecpg_user2',
- '--encoding=SQL_ASCII',
]
tests += {
diff --git a/src/test/icu/t/010_database.pl b/src/test/icu/t/010_database.pl
index 80ab1c7789..45d77c319a 100644
--- a/src/test/icu/t/010_database.pl
+++ b/src/test/icu/t/010_database.pl
@@ -12,7 +12,7 @@ if ($ENV{with_icu} ne 'yes')
}
my $node1 = PostgreSQL::Test::Cluster->new('node1');
-$node1->init;
+$node1->init(extra => ['--locale-provider=libc']);
$node1->start;
$node1->safe_psql('postgres',
--
2.34.1
^ permalink raw reply [nested|flat] 12+ messages in thread
* Re: Move defaults toward ICU in 16?
2023-02-02 13:44 Re: Move defaults toward ICU in 16? Robert Haas <[email protected]>
2023-02-02 16:31 ` Re: Move defaults toward ICU in 16? Jeff Davis <[email protected]>
2023-02-02 22:14 ` Re: Move defaults toward ICU in 16? Thomas Munro <[email protected]>
2023-02-02 23:10 ` Re: Move defaults toward ICU in 16? Tom Lane <[email protected]>
2023-02-08 20:16 ` Re: Move defaults toward ICU in 16? Jeff Davis <[email protected]>
2023-02-09 02:22 ` Re: Move defaults toward ICU in 16? Andres Freund <[email protected]>
2023-02-11 00:17 ` Re: Move defaults toward ICU in 16? Jeff Davis <[email protected]>
2023-02-11 02:00 ` Re: Move defaults toward ICU in 16? Andres Freund <[email protected]>
2023-02-14 17:48 ` Re: Move defaults toward ICU in 16? Jeff Davis <[email protected]>
@ 2023-02-14 17:59 ` Andres Freund <[email protected]>
0 siblings, 0 replies; 12+ messages in thread
From: Andres Freund @ 2023-02-14 17:59 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: Tom Lane <[email protected]>; Thomas Munro <[email protected]>; Robert Haas <[email protected]>; pgsql-hackers
Hi,
On 2023-02-14 09:48:08 -0800, Jeff Davis wrote:
> On Fri, 2023-02-10 at 18:00 -0800, Andres Freund wrote:
> > But, shouldn't pg_upgrade be able to deal with this? As long as the
> > databases
> > are created with template0, we can create the collations at that
> > point?
>
> Are you saying that the upgraded cluster could have a different default
> collation for the template databases than the original cluster?
> That would be wrong to do, at least by default, but I could see it
> being a useful option.
>
> Or maybe I misunderstand what you're saying?
I am saying that pg_upgrade should be able to deal with the difference. The
details of how to implement that, don't matter that much.
FWIW, I don't think it matters much what collation template0 has, since we
allow to change the locale provider when using template0 as the template.
We could easily update template0, if we think that's necessary. But I don't
think it really is. As long as the newly created databases have the right
provider, I'd lean towards not touching template0. But whatever...
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 12+ messages in thread
end of thread, other threads:[~2023-02-14 17:59 UTC | newest]
Thread overview: 12+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2018-05-15 11:21 [PATCH v7 2/3] Infrastructure for asynchronous execution Kyotaro Horiguchi <[email protected]>
2023-02-02 13:44 Re: Move defaults toward ICU in 16? Robert Haas <[email protected]>
2023-02-02 16:31 ` Re: Move defaults toward ICU in 16? Jeff Davis <[email protected]>
2023-02-02 22:14 ` Re: Move defaults toward ICU in 16? Thomas Munro <[email protected]>
2023-02-02 22:48 ` Re: Move defaults toward ICU in 16? Peter Geoghegan <[email protected]>
2023-02-02 23:10 ` Re: Move defaults toward ICU in 16? Tom Lane <[email protected]>
2023-02-08 20:16 ` Re: Move defaults toward ICU in 16? Jeff Davis <[email protected]>
2023-02-09 02:22 ` Re: Move defaults toward ICU in 16? Andres Freund <[email protected]>
2023-02-11 00:17 ` Re: Move defaults toward ICU in 16? Jeff Davis <[email protected]>
2023-02-11 02:00 ` Re: Move defaults toward ICU in 16? Andres Freund <[email protected]>
2023-02-14 17:48 ` Re: Move defaults toward ICU in 16? Jeff Davis <[email protected]>
2023-02-14 17:59 ` Re: Move defaults toward ICU in 16? Andres Freund <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox