public inbox for [email protected]
help / color / mirror / Atom feed[PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table
10+ messages / 5 participants
[nested] [flat]
* [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table
@ 2020-06-06 22:42 Justin Pryzby <[email protected]>
0 siblings, 0 replies; 10+ messages in thread
From: Justin Pryzby @ 2020-06-06 22:42 UTC (permalink / raw)
Note, this effectively reverts 050098b14, so take care to not reintroduce the
bug it fixed.
---
doc/src/sgml/ref/create_index.sgml | 9 --
src/backend/commands/indexcmds.c | 143 ++++++++++++++++++-------
src/test/regress/expected/indexing.out | 60 ++++++++++-
src/test/regress/sql/indexing.sql | 18 +++-
4 files changed, 176 insertions(+), 54 deletions(-)
diff --git a/doc/src/sgml/ref/create_index.sgml b/doc/src/sgml/ref/create_index.sgml
index a5271a9f8f..6869a18968 100644
--- a/doc/src/sgml/ref/create_index.sgml
+++ b/doc/src/sgml/ref/create_index.sgml
@@ -686,15 +686,6 @@ Indexes:
cannot.
</para>
- <para>
- Concurrent builds for indexes on partitioned tables are currently not
- supported. However, you may concurrently build the index on each
- partition individually and then finally create the partitioned index
- non-concurrently in order to reduce the time where writes to the
- partitioned table will be locked out. In this case, building the
- partitioned index is a metadata only operation.
- </para>
-
</refsect2>
</refsect1>
diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c
index 127ba7835d..4ac1dacd7d 100644
--- a/src/backend/commands/indexcmds.c
+++ b/src/backend/commands/indexcmds.c
@@ -68,6 +68,7 @@
/* non-export function prototypes */
+static void reindex_invalid_child_indexes(Oid indexRelationId);
static bool CompareOpclassOptions(Datum *opts1, Datum *opts2, int natts);
static void CheckPredicate(Expr *predicate);
static void ComputeIndexAttrs(IndexInfo *indexInfo,
@@ -680,17 +681,6 @@ DefineIndex(Oid relationId,
partitioned = rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE;
if (partitioned)
{
- /*
- * Note: we check 'stmt->concurrent' rather than 'concurrent', so that
- * the error is thrown also for temporary tables. Seems better to be
- * consistent, even though we could do it on temporary table because
- * we're not actually doing it concurrently.
- */
- if (stmt->concurrent)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("cannot create index on partitioned table \"%s\" concurrently",
- RelationGetRelationName(rel))));
if (stmt->excludeOpNames)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
@@ -1128,6 +1118,11 @@ DefineIndex(Oid relationId,
if (pd->nparts != 0)
flags |= INDEX_CREATE_INVALID;
}
+ else if (concurrent && OidIsValid(parentIndexId))
+ {
+ /* If concurrent, initially build index partitions as "invalid" */
+ flags |= INDEX_CREATE_INVALID;
+ }
if (stmt->deferrable)
constr_flags |= INDEX_CONSTR_CREATE_DEFERRABLE;
@@ -1183,6 +1178,14 @@ DefineIndex(Oid relationId,
partdesc = RelationGetPartitionDesc(rel);
if ((!stmt->relation || stmt->relation->inh) && partdesc->nparts > 0)
{
+ /*
+ * Need to close the relation before recursing into children, so
+ * copy needed data into a longlived context.
+ */
+
+ MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
+ ALLOCSET_DEFAULT_SIZES);
+ MemoryContext oldcontext = MemoryContextSwitchTo(ind_context);
int nparts = partdesc->nparts;
Oid *part_oids = palloc(sizeof(Oid) * nparts);
bool invalidate_parent = false;
@@ -1193,8 +1196,10 @@ DefineIndex(Oid relationId,
nparts);
memcpy(part_oids, partdesc->oids, sizeof(Oid) * nparts);
+ parentDesc = CreateTupleDescCopy(RelationGetDescr(rel));
+ table_close(rel, NoLock);
+ MemoryContextSwitchTo(oldcontext);
- parentDesc = RelationGetDescr(rel);
opfamOids = palloc(sizeof(Oid) * numberOfKeyAttributes);
for (i = 0; i < numberOfKeyAttributes; i++)
opfamOids[i] = get_opclass_family(classObjectId[i]);
@@ -1237,10 +1242,12 @@ DefineIndex(Oid relationId,
continue;
}
+ oldcontext = MemoryContextSwitchTo(ind_context);
childidxs = RelationGetIndexList(childrel);
attmap =
build_attrmap_by_name(RelationGetDescr(childrel),
parentDesc);
+ MemoryContextSwitchTo(oldcontext);
foreach(cell, childidxs)
{
@@ -1311,10 +1318,14 @@ DefineIndex(Oid relationId,
*/
if (!found)
{
- IndexStmt *childStmt = copyObject(stmt);
+ IndexStmt *childStmt;
bool found_whole_row;
ListCell *lc;
+ oldcontext = MemoryContextSwitchTo(ind_context);
+ childStmt = copyObject(stmt);
+ MemoryContextSwitchTo(oldcontext);
+
/*
* We can't use the same index name for the child index,
* so clear idxname to let the recursive invocation choose
@@ -1366,10 +1377,18 @@ DefineIndex(Oid relationId,
createdConstraintId,
is_alter_table, check_rights, check_not_in_use,
skip_build, quiet);
+ if (concurrent)
+ {
+ PopActiveSnapshot();
+ PushActiveSnapshot(GetTransactionSnapshot());
+ invalidate_parent = true;
+ }
}
- pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
- i + 1);
+ /* For concurrent build, this is a catalog-only stage */
+ if (!concurrent)
+ pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ i + 1);
free_attrmap(attmap);
}
@@ -1379,41 +1398,33 @@ DefineIndex(Oid relationId,
* invalid, this is incorrect, so update our row to invalid too.
*/
if (invalidate_parent)
- {
- Relation pg_index = table_open(IndexRelationId, RowExclusiveLock);
- HeapTuple tup,
- newtup;
-
- tup = SearchSysCache1(INDEXRELID,
- ObjectIdGetDatum(indexRelationId));
- if (!HeapTupleIsValid(tup))
- elog(ERROR, "cache lookup failed for index %u",
- indexRelationId);
- newtup = heap_copytuple(tup);
- ((Form_pg_index) GETSTRUCT(newtup))->indisvalid = false;
- CatalogTupleUpdate(pg_index, &tup->t_self, newtup);
- ReleaseSysCache(tup);
- table_close(pg_index, RowExclusiveLock);
- heap_freetuple(newtup);
- }
- }
+ index_set_state_flags(indexRelationId, INDEX_DROP_CLEAR_VALID);
+ } else
+ table_close(rel, NoLock);
/*
* Indexes on partitioned tables are not themselves built, so we're
* done here.
*/
- table_close(rel, NoLock);
if (!OidIsValid(parentIndexId))
+ {
+ if (concurrent)
+ reindex_invalid_child_indexes(indexRelationId);
+
pgstat_progress_end_command();
+ }
+
return address;
}
- if (!concurrent)
+ if (!concurrent || OidIsValid(parentIndexId))
{
- /* Close the heap and we're done, in the non-concurrent case */
- table_close(rel, NoLock);
+ /*
+ * We're done if this is the top-level index,
+ * or the catalog-only phase of a partition built concurrently
+ */
- /* If this is the top-level index, we're done. */
+ table_close(rel, NoLock);
if (!OidIsValid(parentIndexId))
pgstat_progress_end_command();
@@ -1617,6 +1628,62 @@ DefineIndex(Oid relationId,
return address;
}
+/* Reindex invalid child indexes created earlier */
+static void
+reindex_invalid_child_indexes(Oid indexRelationId)
+{
+ ListCell *lc;
+ int npart = 0;
+ ReindexParams params = {
+ .options = REINDEXOPT_CONCURRENTLY
+ };
+
+ MemoryContext ind_context = AllocSetContextCreate(PortalContext, "CREATE INDEX",
+ ALLOCSET_DEFAULT_SIZES);
+ MemoryContext oldcontext;
+ List *childs = find_inheritance_children(indexRelationId, ShareLock);
+ List *partitions = NIL;
+
+ PreventInTransactionBlock(true, "REINDEX INDEX");
+
+ foreach (lc, childs)
+ {
+ Oid partoid = lfirst_oid(lc);
+
+ /* XXX: need to retrofit progress reporting into it */
+ // pgstat_progress_update_param(PROGRESS_CREATEIDX_PARTITIONS_DONE,
+ // npart++);
+
+ if (get_index_isvalid(partoid) ||
+ !RELKIND_HAS_STORAGE(get_rel_relkind(partoid)))
+ continue;
+
+ /* Save partition OID */
+ oldcontext = MemoryContextSwitchTo(ind_context);
+ partitions = lappend_oid(partitions, partoid);
+ MemoryContextSwitchTo(oldcontext);
+ }
+
+ /*
+ * Process each partition listed in a separate transaction. Note that
+ * this commits and then starts a new transaction immediately.
+ * XXX: since this is done in 2*N transactions, it could just as well
+ * call ReindexRelationConcurrently directly
+ */
+ ReindexMultipleInternal(partitions, ¶ms);
+
+ /*
+ * CIC needs to mark a partitioned index as VALID, which itself
+ * requires setting READY, which is unset for CIC (even though
+ * it's meaningless for an index without storage).
+ * This must be done only while holding a lock which precludes adding
+ * partitions.
+ * See also: validatePartitionedIndex().
+ */
+ index_set_state_flags(indexRelationId, INDEX_CREATE_SET_READY);
+ CommandCounterIncrement();
+ index_set_state_flags(indexRelationId, INDEX_CREATE_SET_VALID);
+}
/*
* CheckMutability
diff --git a/src/test/regress/expected/indexing.out b/src/test/regress/expected/indexing.out
index c93f4470c9..f04abc6897 100644
--- a/src/test/regress/expected/indexing.out
+++ b/src/test/regress/expected/indexing.out
@@ -50,11 +50,63 @@ select relname, relkind, relhassubclass, inhparent::regclass
(8 rows)
drop table idxpart;
--- Some unsupported features
+-- CIC on partitioned table
create table idxpart (a int, b int, c text) partition by range (a);
-create table idxpart1 partition of idxpart for values from (0) to (10);
-create index concurrently on idxpart (a);
-ERROR: cannot create index on partitioned table "idxpart" concurrently
+create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a);
+create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a);
+create table idxpart111 partition of idxpart11 default partition by range(a);
+create table idxpart1111 partition of idxpart111 default partition by range(a);
+create table idxpart2 partition of idxpart for values from (10) to (20);
+insert into idxpart2 values(10),(10); -- not unique
+create index concurrently on idxpart (a); -- partitioned
+create index concurrently on idxpart1 (a); -- partitioned and partition
+create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves
+create index concurrently on idxpart2 (a); -- leaf
+create unique index concurrently on idxpart (a); -- partitioned, unique failure
+ERROR: could not create unique index "idxpart2_a_idx2_ccnew"
+DETAIL: Key (a)=(10) is duplicated.
+\d idxpart
+ Partitioned table "public.idxpart"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+---------
+ a | integer | | |
+ b | integer | | |
+ c | text | | |
+Partition key: RANGE (a)
+Indexes:
+ "idxpart_a_idx" btree (a)
+ "idxpart_a_idx1" UNIQUE, btree (a) INVALID
+Number of partitions: 2 (Use \d+ to list them.)
+
+\d idxpart1
+ Partitioned table "public.idxpart1"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+---------
+ a | integer | | |
+ b | integer | | |
+ c | text | | |
+Partition of: idxpart FOR VALUES FROM (0) TO (10)
+Partition key: RANGE (a)
+Indexes:
+ "idxpart1_a_idx" btree (a) INVALID
+ "idxpart1_a_idx1" btree (a)
+ "idxpart1_a_idx2" UNIQUE, btree (a) INVALID
+Number of partitions: 1 (Use \d+ to list them.)
+
+\d idxpart2
+ Table "public.idxpart2"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+---------
+ a | integer | | |
+ b | integer | | |
+ c | text | | |
+Partition of: idxpart FOR VALUES FROM (10) TO (20)
+Indexes:
+ "idxpart2_a_idx" btree (a)
+ "idxpart2_a_idx1" btree (a)
+ "idxpart2_a_idx2" UNIQUE, btree (a) INVALID
+ "idxpart2_a_idx2_ccnew" UNIQUE, btree (a) INVALID
+
drop table idxpart;
-- Verify bugfix with query on indexed partitioned table with no partitions
-- https://postgr.es/m/[email protected]
diff --git a/src/test/regress/sql/indexing.sql b/src/test/regress/sql/indexing.sql
index 42f398b67c..3d4b6e9bc9 100644
--- a/src/test/regress/sql/indexing.sql
+++ b/src/test/regress/sql/indexing.sql
@@ -29,10 +29,22 @@ select relname, relkind, relhassubclass, inhparent::regclass
where relname like 'idxpart%' order by relname;
drop table idxpart;
--- Some unsupported features
+-- CIC on partitioned table
create table idxpart (a int, b int, c text) partition by range (a);
-create table idxpart1 partition of idxpart for values from (0) to (10);
-create index concurrently on idxpart (a);
+create table idxpart1 partition of idxpart for values from (0) to (10) partition by range(a);
+create table idxpart11 partition of idxpart1 for values from (0) to (10) partition by range(a);
+create table idxpart111 partition of idxpart11 default partition by range(a);
+create table idxpart1111 partition of idxpart111 default partition by range(a);
+create table idxpart2 partition of idxpart for values from (10) to (20);
+insert into idxpart2 values(10),(10); -- not unique
+create index concurrently on idxpart (a); -- partitioned
+create index concurrently on idxpart1 (a); -- partitioned and partition
+create index concurrently on idxpart11 (a); -- partitioned and partition, with no leaves
+create index concurrently on idxpart2 (a); -- leaf
+create unique index concurrently on idxpart (a); -- partitioned, unique failure
+\d idxpart
+\d idxpart1
+\d idxpart2
drop table idxpart;
-- Verify bugfix with query on indexed partitioned table with no partitions
--
2.17.0
--QTprm0S8XgL7H0Dt
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="v13-0002-f-progress-reporting.patch"
^ permalink raw reply [nested|flat] 10+ messages in thread
* Re: Replace uses of deprecated Python module distutils.sysconfig
@ 2022-01-18 16:47 Peter Eisentraut <[email protected]>
2022-01-18 17:04 ` Re: Replace uses of deprecated Python module distutils.sysconfig Tom Lane <[email protected]>
0 siblings, 1 reply; 10+ messages in thread
From: Peter Eisentraut @ 2022-01-18 16:47 UTC (permalink / raw)
To: Tom Lane <[email protected]>; +Cc: pgsql-hackers
On 18.01.22 16:24, Tom Lane wrote:
> Peter Eisentraut <[email protected]> writes:
>> Also, considering the failure on prairiedog, I do see now on
>> <https://docs.python.org/3/library/sysconfig.html; that the sysconfig
>> module is "New in version 3.2". I had interpreted the fact that it
>> exists in version 2.7 that that includes all higher versions, but
>> obviously there were multiple branches involved, so that was a mistaken
>> assumption.
>
> Hm. I installed 3.1 because we claim support for that. I don't mind
> updating to 3.2 (as long as we adjust the docs to match), but it seems
> kinda moot unless you figure out a solution for the include-path
> issue. I see that platforms as recent as Debian 10 are failing,
> so I don't think we can dismiss that as not needing fixing.
I have reverted this for now.
I don't have a clear idea how to fix this in the long run. We would
perhaps need to determine at which points the various platforms had
fixed this issue in their Python installations and select between the
old and the new approach based on that. Seems messy.
^ permalink raw reply [nested|flat] 10+ messages in thread
* Re: Replace uses of deprecated Python module distutils.sysconfig
2022-01-18 16:47 Re: Replace uses of deprecated Python module distutils.sysconfig Peter Eisentraut <[email protected]>
@ 2022-01-18 17:04 ` Tom Lane <[email protected]>
2022-01-19 00:18 ` Re: Replace uses of deprecated Python module distutils.sysconfig Tom Lane <[email protected]>
0 siblings, 1 reply; 10+ messages in thread
From: Tom Lane @ 2022-01-18 17:04 UTC (permalink / raw)
To: Peter Eisentraut <[email protected]>; +Cc: pgsql-hackers
Peter Eisentraut <[email protected]> writes:
> I don't have a clear idea how to fix this in the long run. We would
> perhaps need to determine at which points the various platforms had
> fixed this issue in their Python installations and select between the
> old and the new approach based on that. Seems messy.
Are we sure it's an issue within Python, rather than something we
could dodge by invoking sysconfig differently? It's hard to believe
that sysconfig could be totally unfit for the purpose of finding out
the include path and would remain so for multiple years.
regards, tom lane
^ permalink raw reply [nested|flat] 10+ messages in thread
* Re: Replace uses of deprecated Python module distutils.sysconfig
2022-01-18 16:47 Re: Replace uses of deprecated Python module distutils.sysconfig Peter Eisentraut <[email protected]>
2022-01-18 17:04 ` Re: Replace uses of deprecated Python module distutils.sysconfig Tom Lane <[email protected]>
@ 2022-01-19 00:18 ` Tom Lane <[email protected]>
2022-01-19 16:21 ` Re: Replace uses of deprecated Python module distutils.sysconfig Tom Lane <[email protected]>
2022-01-19 16:49 ` Re: Replace uses of deprecated Python module distutils.sysconfig Peter Eisentraut <[email protected]>
2022-01-20 15:04 ` Re: Replace uses of deprecated Python module distutils.sysconfig Andrew Dunstan <[email protected]>
0 siblings, 3 replies; 10+ messages in thread
From: Tom Lane @ 2022-01-19 00:18 UTC (permalink / raw)
To: Peter Eisentraut <[email protected]>; +Cc: Andrew Dunstan <[email protected]>; [email protected]
I wrote:
> Are we sure it's an issue within Python, rather than something we
> could dodge by invoking sysconfig differently? It's hard to believe
> that sysconfig could be totally unfit for the purpose of finding out
> the include path and would remain so for multiple years.
I dug up a Debian 9 image and found that I could reproduce the problem
against its python2 (2.7.13) installation, but not its python3 (3.5.3):
$ python2 -m sysconfig | grep include
include = "/usr/local/include/python2.7"
platinclude = "/usr/local/include/python2.7"
...
$ python3 -m sysconfig | grep include
include = "/usr/include/python3.5m"
platinclude = "/usr/include/python3.5m"
...
Looking at the buildfarm animals that failed this way, 10 out of 11
are using python 2.x. The lone exception is Andrew's prion. I wonder
if there is something unusual about its python3 installation.
Anyway, based on these results, we might have better luck switching to
sysconfig after we start forcing python3. I'm tempted to resurrect the
idea of changing configure's probe order to "python3 python python2"
in the meantime, just so we can see how much of the buildfarm is ready
for that.
regards, tom lane
^ permalink raw reply [nested|flat] 10+ messages in thread
* Re: Replace uses of deprecated Python module distutils.sysconfig
2022-01-18 16:47 Re: Replace uses of deprecated Python module distutils.sysconfig Peter Eisentraut <[email protected]>
2022-01-18 17:04 ` Re: Replace uses of deprecated Python module distutils.sysconfig Tom Lane <[email protected]>
2022-01-19 00:18 ` Re: Replace uses of deprecated Python module distutils.sysconfig Tom Lane <[email protected]>
@ 2022-01-19 16:21 ` Tom Lane <[email protected]>
2 siblings, 0 replies; 10+ messages in thread
From: Tom Lane @ 2022-01-19 16:21 UTC (permalink / raw)
To: Peter Eisentraut <[email protected]>; +Cc: Andrew Dunstan <[email protected]>; [email protected]
I wrote:
> Anyway, based on these results, we might have better luck switching to
> sysconfig after we start forcing python3.
On the other hand, that answer is not back-patchable, and we surely
need a back-patchable fix, because people will try to build the
back branches against newer pythons.
Based on the buildfarm results so far, the problem can be described
as "some installations say /usr/local when they should have said /usr".
I experimented with the attached delta patch and it fixes the problem
on my Debian 9 image. (I don't know Python, so there may be a better
way to do this.) We'd have to also bump the minimum 3.x version to
3.2, but that seems very unlikely to bother anyone.
regards, tom lane
Attachments:
[text/x-diff] hack-bad-sysconfig-answer.patch (2.0K, ../../[email protected]/2-hack-bad-sysconfig-answer.patch)
download | inline diff:
diff --git a/config/python.m4 b/config/python.m4
index 8ca1eaa64b..c65356c6ac 100644
--- a/config/python.m4
+++ b/config/python.m4
@@ -56,13 +56,20 @@ AC_MSG_RESULT([$python_configdir])
AC_MSG_CHECKING([Python include directories])
python_includespec=`${PYTHON} -c "
-import sysconfig
-a = '-I' + sysconfig.get_path('include')
-b = '-I' + sysconfig.get_path('platinclude')
+import sysconfig, os.path
+a = sysconfig.get_path('include')
+b = sysconfig.get_path('platinclude')
+# Some versions of sysconfig report '/usr/local/include'
+# when they should have said '/usr/include'
+if not os.path.exists(a + '/Python.h'):
+ aalt = a.replace('/usr/local/', '/usr/', 1)
+ if os.path.exists(aalt + '/Python.h'):
+ a = aalt
+ b = b.replace('/usr/local/', '/usr/', 1)
if a == b:
- print(a)
+ print('-I' + a)
else:
- print(a + ' ' + b)"`
+ print('-I' + a + ' -I' + b)"`
if test "$PORTNAME" = win32 ; then
python_includespec=`echo $python_includespec | sed 's,[[\]],/,g'`
fi
diff --git a/configure b/configure
index 9c856cb1d5..f88db9467d 100755
--- a/configure
+++ b/configure
@@ -10370,13 +10370,20 @@ $as_echo "$python_configdir" >&6; }
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking Python include directories" >&5
$as_echo_n "checking Python include directories... " >&6; }
python_includespec=`${PYTHON} -c "
-import sysconfig
-a = '-I' + sysconfig.get_path('include')
-b = '-I' + sysconfig.get_path('platinclude')
+import sysconfig, os.path
+a = sysconfig.get_path('include')
+b = sysconfig.get_path('platinclude')
+# Some versions of sysconfig report '/usr/local/include'
+# when they should have said '/usr/include'
+if not os.path.exists(a + '/Python.h'):
+ aalt = a.replace('/usr/local/', '/usr/', 1)
+ if os.path.exists(aalt + '/Python.h'):
+ a = aalt
+ b = b.replace('/usr/local/', '/usr/', 1)
if a == b:
- print(a)
+ print('-I' + a)
else:
- print(a + ' ' + b)"`
+ print('-I' + a + ' -I' + b)"`
if test "$PORTNAME" = win32 ; then
python_includespec=`echo $python_includespec | sed 's,[\],/,g'`
fi
^ permalink raw reply [nested|flat] 10+ messages in thread
* Re: Replace uses of deprecated Python module distutils.sysconfig
2022-01-18 16:47 Re: Replace uses of deprecated Python module distutils.sysconfig Peter Eisentraut <[email protected]>
2022-01-18 17:04 ` Re: Replace uses of deprecated Python module distutils.sysconfig Tom Lane <[email protected]>
2022-01-19 00:18 ` Re: Replace uses of deprecated Python module distutils.sysconfig Tom Lane <[email protected]>
@ 2022-01-19 16:49 ` Peter Eisentraut <[email protected]>
2022-01-19 20:40 ` Re: Replace uses of deprecated Python module distutils.sysconfig Tom Lane <[email protected]>
2 siblings, 1 reply; 10+ messages in thread
From: Peter Eisentraut @ 2022-01-19 16:49 UTC (permalink / raw)
To: Tom Lane <[email protected]>; +Cc: Andrew Dunstan <[email protected]>; [email protected]
On 19.01.22 01:18, Tom Lane wrote:
> Anyway, based on these results, we might have better luck switching to
> sysconfig after we start forcing python3. I'm tempted to resurrect the
> idea of changing configure's probe order to "python3 python python2"
> in the meantime, just so we can see how much of the buildfarm is ready
> for that.
This seems sensible in any case, given that we have quasi-committed to
enforcing Python 3 soon.
^ permalink raw reply [nested|flat] 10+ messages in thread
* Re: Replace uses of deprecated Python module distutils.sysconfig
2022-01-18 16:47 Re: Replace uses of deprecated Python module distutils.sysconfig Peter Eisentraut <[email protected]>
2022-01-18 17:04 ` Re: Replace uses of deprecated Python module distutils.sysconfig Tom Lane <[email protected]>
2022-01-19 00:18 ` Re: Replace uses of deprecated Python module distutils.sysconfig Tom Lane <[email protected]>
2022-01-19 16:49 ` Re: Replace uses of deprecated Python module distutils.sysconfig Peter Eisentraut <[email protected]>
@ 2022-01-19 20:40 ` Tom Lane <[email protected]>
2022-01-19 21:20 ` Re: Replace uses of deprecated Python module distutils.sysconfig Juan José Santamaría Flecha <[email protected]>
0 siblings, 1 reply; 10+ messages in thread
From: Tom Lane @ 2022-01-19 20:40 UTC (permalink / raw)
To: Peter Eisentraut <[email protected]>; +Cc: Andrew Dunstan <[email protected]>; [email protected]
Peter Eisentraut <[email protected]> writes:
> On 19.01.22 01:18, Tom Lane wrote:
>> ... I'm tempted to resurrect the
>> idea of changing configure's probe order to "python3 python python2"
>> in the meantime, just so we can see how much of the buildfarm is ready
>> for that.
> This seems sensible in any case, given that we have quasi-committed to
> enforcing Python 3 soon.
Done. (I couldn't find any equivalent logic in the MSVC build scripts
though; is there something I missed?)
regards, tom lane
^ permalink raw reply [nested|flat] 10+ messages in thread
* Re: Replace uses of deprecated Python module distutils.sysconfig
2022-01-18 16:47 Re: Replace uses of deprecated Python module distutils.sysconfig Peter Eisentraut <[email protected]>
2022-01-18 17:04 ` Re: Replace uses of deprecated Python module distutils.sysconfig Tom Lane <[email protected]>
2022-01-19 00:18 ` Re: Replace uses of deprecated Python module distutils.sysconfig Tom Lane <[email protected]>
2022-01-19 16:49 ` Re: Replace uses of deprecated Python module distutils.sysconfig Peter Eisentraut <[email protected]>
2022-01-19 20:40 ` Re: Replace uses of deprecated Python module distutils.sysconfig Tom Lane <[email protected]>
@ 2022-01-19 21:20 ` Juan José Santamaría Flecha <[email protected]>
2022-01-20 01:27 ` Re: Replace uses of deprecated Python module distutils.sysconfig Tom Lane <[email protected]>
0 siblings, 1 reply; 10+ messages in thread
From: Juan José Santamaría Flecha @ 2022-01-19 21:20 UTC (permalink / raw)
To: Tom Lane <[email protected]>; +Cc: Peter Eisentraut <[email protected]>; Andrew Dunstan <[email protected]>; PostgreSQL Hackers <[email protected]>
On Wed, Jan 19, 2022 at 9:40 PM Tom Lane <[email protected]> wrote:
>
> Done. (I couldn't find any equivalent logic in the MSVC build scripts
> though; is there something I missed?)
>
> MSVC will use the path configured in src\tools\msvc\config.pl $config->{"python"},
there is no ambiguity.
Regards,
Juan José Santamaría Flecha
^ permalink raw reply [nested|flat] 10+ messages in thread
* Re: Replace uses of deprecated Python module distutils.sysconfig
2022-01-18 16:47 Re: Replace uses of deprecated Python module distutils.sysconfig Peter Eisentraut <[email protected]>
2022-01-18 17:04 ` Re: Replace uses of deprecated Python module distutils.sysconfig Tom Lane <[email protected]>
2022-01-19 00:18 ` Re: Replace uses of deprecated Python module distutils.sysconfig Tom Lane <[email protected]>
2022-01-19 16:49 ` Re: Replace uses of deprecated Python module distutils.sysconfig Peter Eisentraut <[email protected]>
2022-01-19 20:40 ` Re: Replace uses of deprecated Python module distutils.sysconfig Tom Lane <[email protected]>
2022-01-19 21:20 ` Re: Replace uses of deprecated Python module distutils.sysconfig Juan José Santamaría Flecha <[email protected]>
@ 2022-01-20 01:27 ` Tom Lane <[email protected]>
0 siblings, 0 replies; 10+ messages in thread
From: Tom Lane @ 2022-01-20 01:27 UTC (permalink / raw)
To: Juan José Santamaría Flecha <[email protected]>; +Cc: Peter Eisentraut <[email protected]>; Andrew Dunstan <[email protected]>; PostgreSQL Hackers <[email protected]>
=?UTF-8?Q?Juan_Jos=C3=A9_Santamar=C3=ADa_Flecha?= <[email protected]> writes:
> On Wed, Jan 19, 2022 at 9:40 PM Tom Lane <[email protected]> wrote:
>> Done. (I couldn't find any equivalent logic in the MSVC build scripts
>> though; is there something I missed?)
> MSVC will use the path configured in src\tools\msvc\config.pl $config->{"python"},
> there is no ambiguity.
Ah, right. We have only three active Windows animals that are building
with python, and all three are using 3.something, so that side of the
house seems to be ready to go.
regards, tom lane
^ permalink raw reply [nested|flat] 10+ messages in thread
* Re: Replace uses of deprecated Python module distutils.sysconfig
2022-01-18 16:47 Re: Replace uses of deprecated Python module distutils.sysconfig Peter Eisentraut <[email protected]>
2022-01-18 17:04 ` Re: Replace uses of deprecated Python module distutils.sysconfig Tom Lane <[email protected]>
2022-01-19 00:18 ` Re: Replace uses of deprecated Python module distutils.sysconfig Tom Lane <[email protected]>
@ 2022-01-20 15:04 ` Andrew Dunstan <[email protected]>
2 siblings, 0 replies; 10+ messages in thread
From: Andrew Dunstan @ 2022-01-20 15:04 UTC (permalink / raw)
To: Tom Lane <[email protected]>; Peter Eisentraut <[email protected]>; +Cc: [email protected]
On 1/18/22 19:18, Tom Lane wrote:
> I wrote:
>> Are we sure it's an issue within Python, rather than something we
>> could dodge by invoking sysconfig differently? It's hard to believe
>> that sysconfig could be totally unfit for the purpose of finding out
>> the include path and would remain so for multiple years.
> I dug up a Debian 9 image and found that I could reproduce the problem
> against its python2 (2.7.13) installation, but not its python3 (3.5.3):
>
> $ python2 -m sysconfig | grep include
> include = "/usr/local/include/python2.7"
> platinclude = "/usr/local/include/python2.7"
> ...
> $ python3 -m sysconfig | grep include
> include = "/usr/include/python3.5m"
> platinclude = "/usr/include/python3.5m"
> ...
>
> Looking at the buildfarm animals that failed this way, 10 out of 11
> are using python 2.x. The lone exception is Andrew's prion. I wonder
> if there is something unusual about its python3 installation.
It's an Amazon Linux instance, and using their packages, which seem a
bit odd (there's nothing in /usr/local/include). Maybe we should be
looking at INCLUEPY?
[ec2-user@ip-172-31-22-42 bf]$ python3 -m sysconfig | grep include
include = "/usr/local/include/python3.6m"
platinclude = "/usr/local/include/python3.6m"
CONFIG_ARGS = "'--build=x86_64-redhat-linux-gnu'
'--host=x86_64-redhat-linux-gnu' '--target=x86_64-amazon-linux-gnu'
'--program-prefix=' '--prefix=/usr' '--exec-prefix=/usr'
'--bindir=/usr/bin' '--sbindir=/usr/sbin' '--sysconfdir=/etc'
'--datadir=/usr/share' '--includedir=/usr/include' '--libdir=/usr/lib64'
'--libexecdir=/usr/libexec' '--localstatedir=/var'
'--sharedstatedir=/var/lib' '--mandir=/usr/share/man'
'--infodir=/usr/share/info' '--enable-ipv6' '--enable-shared'
'--with-computed-gotos=yes' '--with-dbmliborder=gdbm:ndbm:bdb'
'--with-system-expat' '--with-system-ffi'
'--enable-loadable-sqlite-extensions' '--with-dtrace' '--with-valgrind'
'--without-ensurepip' '--enable-optimizations'
'build_alias=x86_64-redhat-linux-gnu'
'host_alias=x86_64-redhat-linux-gnu'
'target_alias=x86_64-amazon-linux-gnu' 'CFLAGS=-O2 -g -pipe -Wall
-Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector
--param=ssp-buffer-size=4 -m64 -mtune=generic -D_GNU_SOURCE -fPIC
-fwrapv ' 'LDFLAGS= -g ' 'CPPFLAGS= '
'PKG_CONFIG_PATH=:/usr/lib64/pkgconfig:/usr/share/pkgconfig'"
CONFINCLUDEDIR = "/usr/include"
CONFINCLUDEPY = "/usr/include/python3.6m"
INCLDIRSTOMAKE = "/usr/include /usr/include/python3.6m"
INCLUDEDIR = "/usr/include"
INCLUDEPY = "/usr/include/python3.6m"
I have upgraded it to python 3.8, but got similar results.
cheers
andrew
--
Andrew Dunstan
EDB: https://www.enterprisedb.com
^ permalink raw reply [nested|flat] 10+ messages in thread
end of thread, other threads:[~2022-01-20 15:04 UTC | newest]
Thread overview: 10+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2020-06-06 22:42 [PATCH v13 01/18] Allow CREATE INDEX CONCURRENTLY on partitioned table Justin Pryzby <[email protected]>
2022-01-18 16:47 Re: Replace uses of deprecated Python module distutils.sysconfig Peter Eisentraut <[email protected]>
2022-01-18 17:04 ` Re: Replace uses of deprecated Python module distutils.sysconfig Tom Lane <[email protected]>
2022-01-19 00:18 ` Re: Replace uses of deprecated Python module distutils.sysconfig Tom Lane <[email protected]>
2022-01-19 16:21 ` Re: Replace uses of deprecated Python module distutils.sysconfig Tom Lane <[email protected]>
2022-01-19 16:49 ` Re: Replace uses of deprecated Python module distutils.sysconfig Peter Eisentraut <[email protected]>
2022-01-19 20:40 ` Re: Replace uses of deprecated Python module distutils.sysconfig Tom Lane <[email protected]>
2022-01-19 21:20 ` Re: Replace uses of deprecated Python module distutils.sysconfig Juan José Santamaría Flecha <[email protected]>
2022-01-20 01:27 ` Re: Replace uses of deprecated Python module distutils.sysconfig Tom Lane <[email protected]>
2022-01-20 15:04 ` Re: Replace uses of deprecated Python module distutils.sysconfig Andrew Dunstan <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox