public inbox for [email protected]
help / color / mirror / Atom feedFrom: Peter Eisentraut <[email protected]>
To: Tristan Partin <[email protected]>
Cc: pgsql-hackers <[email protected]>
Subject: Re: make dist using git archive
Date: Fri, 26 Jan 2024 07:28:15 +0100
Message-ID: <[email protected]> (raw)
In-Reply-To: <[email protected]>
References: <[email protected]>
<[email protected]>
<[email protected]>
<[email protected]>
On 25.01.24 17:25, Tristan Partin wrote:
> The way that this currently works is that you will fail at configure
> time if bz2 doesn't exist on the system. Meson will try to resolve a
> .path() method on a NotFoundProgram. You might want to define the bz2
> target to just call `exit 1` in this case.
>
> if bzip2.found()
> # do your current target
> else
> bz2 = run_target('tar.bz2', command: ['exit', 1])
> endif
>
> This should cause pgdist to appropriately fail at run time when
> generating the bz2 tarball.
Ok, done that way.
> For what it's worth, I run Meson 1.3, and the behavior of generating the
> tarballs even though it is a dirty tree still occurred. In the new patch
> you seem to say it was fixed in 0.60.
The problem I'm referring to is that before 0.60, alias_target cannot
depend on run_target (only "build target"). This is AFAICT not
documented and might not have been an intentional change, but you can
trace it in the meson source code, and it shows in the PostgreSQL CI.
That's also why for the above bzip2 issue I have to use custom_target in
place of your run_target.
From 48ddd079c1530baaabf92a5650ff9a7bfa7ac3d6 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <[email protected]>
Date: Thu, 25 Jan 2024 12:28:28 +0100
Subject: [PATCH v2 1/2] make dist uses git archive
Discussion: https://www.postgresql.org/message-id/flat/40e80f77-a294-4f29-a16f-e21bc7bc75fc%40eisentraut.org
---
GNUmakefile.in | 34 ++++++++++++----------------------
meson.build | 47 +++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 59 insertions(+), 22 deletions(-)
diff --git a/GNUmakefile.in b/GNUmakefile.in
index eba569e930e..680c765dd73 100644
--- a/GNUmakefile.in
+++ b/GNUmakefile.in
@@ -87,29 +87,19 @@ update-unicode: | submake-generated-headers submake-libpgport
distdir = postgresql-$(VERSION)
dummy = =install=
+GIT = git
+
dist: $(distdir).tar.gz $(distdir).tar.bz2
- rm -rf $(distdir)
-
-$(distdir).tar: distdir
- $(TAR) chf $@ $(distdir)
-
-.INTERMEDIATE: $(distdir).tar
-
-distdir-location:
- @echo $(distdir)
-
-distdir:
- rm -rf $(distdir)* $(dummy)
- for x in `cd $(top_srcdir) && find . \( -name CVS -prune \) -o \( -name .git -prune \) -o -print`; do \
- file=`expr X$$x : 'X\./\(.*\)'`; \
- if test -d "$(top_srcdir)/$$file" ; then \
- mkdir "$(distdir)/$$file" && chmod 777 "$(distdir)/$$file"; \
- else \
- ln "$(top_srcdir)/$$file" "$(distdir)/$$file" >/dev/null 2>&1 \
- || cp "$(top_srcdir)/$$file" "$(distdir)/$$file"; \
- fi || exit; \
- done
- $(MAKE) -C $(distdir) distclean
+
+.PHONY: check-dirty-index
+check-dirty-index:
+ $(GIT) -C $(srcdir) diff-index --quiet HEAD
+
+$(distdir).tar.gz: check-dirty-index
+ $(GIT) -C $(srcdir) archive --format tar.gz --prefix $(distdir)/ HEAD -o $(abs_top_builddir)/$@
+
+$(distdir).tar.bz2: check-dirty-index
+ $(GIT) -C $(srcdir) -c tar.tar.bz2.command='$(BZIP2) -c' archive --format tar.bz2 --prefix $(distdir)/ HEAD -o $(abs_top_builddir)/$@
distcheck: dist
rm -rf $(dummy)
diff --git a/meson.build b/meson.build
index 55184db2488..c245a1818e0 100644
--- a/meson.build
+++ b/meson.build
@@ -3348,6 +3348,53 @@ run_target('help',
+###############################################################
+# Distribution archive
+###############################################################
+
+git = find_program('git', required: false, native: true)
+bzip2 = find_program('bzip2', required: false, native: true)
+
+distdir = meson.project_name() + '-' + meson.project_version()
+
+check_dirty_index = run_target('check-dirty-index',
+ command: [git, '-C', '@SOURCE_ROOT@', 'diff-index', '--quiet', 'HEAD'])
+
+tar_gz = custom_target('tar.gz',
+ build_always_stale: true,
+ command: [git, '-C', '@SOURCE_ROOT@', 'archive',
+ '--format', 'tar.gz',
+ '--prefix', distdir + '/',
+ '-o', join_paths(meson.build_root(), '@OUTPUT@'),
+ 'HEAD', '.'],
+ install: false,
+ output: distdir + '.tar.gz',
+)
+
+if bzip2.found()
+ tar_bz2 = custom_target('tar.bz2',
+ build_always_stale: true,
+ command: [git, '-C', '@SOURCE_ROOT@', '-c', 'tar.tar.bz2.command="' + bzip2.path() + '" -c', 'archive',
+ '--format', 'tar.bz2',
+ '--prefix', distdir + '/',
+ '-o', join_paths(meson.build_root(), '@OUTPUT@'),
+ 'HEAD', '.'],
+ install: false,
+ output: distdir + '.tar.bz2',
+ )
+else
+ tar_bz2 = custom_target('tar.bz2',
+ command: ['sh', '-c', 'exit 1'],
+ output: distdir + '.tar.bz2',
+ )
+endif
+
+# FIXME: would like to add check_dirty_index here, broken(?) before
+# meson 0.60.0
+alias_target('pgdist', [tar_gz, tar_bz2])
+
+
+
###############################################################
# The End, The End, My Friend
###############################################################
base-commit: 46d8587b504170ca14f064890bc7ccbbd7523f81
--
2.43.0
From 65055901d7476ecef67ff96f8218ca8fa344838f Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <[email protected]>
Date: Thu, 25 Jan 2024 15:35:48 +0100
Subject: [PATCH v2 2/2] WIP: Add dist building to CI
Note: freebsd repartition script didn't copy .git directory
---
.cirrus.tasks.yml | 28 +++++++++++++++++++++++++
src/tools/ci/gcp_freebsd_repartition.sh | 2 +-
2 files changed, 29 insertions(+), 1 deletion(-)
diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index e4e1bcfeb99..6eba3073a35 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -202,6 +202,11 @@ task:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
EOF
+ dist_script: |
+ su postgres -c 'meson compile -C build -v pgdist'
+ tar_artifacts:
+ path: "build*/*.tar.*"
+
on_failure:
# if the server continues running, it often causes cirrus-ci to fail
# during upload, as it doesn't expect artifacts to change size
@@ -345,6 +350,10 @@ task:
make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
EOF
+ dist_script: su postgres -c "make dist -j${BUILD_JOBS}"
+ tar_artifacts:
+ path: "build*/*.tar.*"
+
on_failure:
<<: *on_failure_ac
@@ -403,6 +412,10 @@ task:
PYTHONCOERCECLOCALE=0 LANG=C meson test $MTEST_ARGS -C build-32 --num-processes ${TEST_JOBS}
EOF
+ dist_script: su postgres -c 'meson compile -C build-32 -v pgdist'
+ tar_artifacts:
+ path: "build*/*.tar.*"
+
on_failure:
<<: *on_failure_meson
@@ -496,6 +509,10 @@ task:
ulimit -n 1024 # default is 256, pretty low
meson test $MTEST_ARGS --num-processes ${TEST_JOBS}
+ dist_script: meson compile -C build -v pgdist
+ tar_artifacts:
+ path: "build*/*.tar.*"
+
on_failure:
<<: *on_failure_meson
cores_script: src/tools/ci/cores_backtrace.sh macos "${HOME}/cores"
@@ -567,6 +584,12 @@ task:
vcvarsall x64
meson test %MTEST_ARGS% --num-processes %TEST_JOBS%
+ dist_script: |
+ vcvarsall x64
+ meson compile -C build -v pgdist
+ tar_artifacts:
+ path: "build*/*.tar.*"
+
on_failure:
<<: *on_failure_meson
crashlog_artifacts:
@@ -627,6 +650,11 @@ task:
test_world_script: |
%BASH% -c "meson test %MTEST_ARGS% --num-processes %TEST_JOBS%"
+ dist_script: |
+ %BASH% -c "meson compile -C build -v pgdist"
+ tar_artifacts:
+ path: "build*/*.tar.*"
+
on_failure:
<<: *on_failure_meson
crashlog_artifacts:
diff --git a/src/tools/ci/gcp_freebsd_repartition.sh b/src/tools/ci/gcp_freebsd_repartition.sh
index 2d5e1738998..edf38e17575 100755
--- a/src/tools/ci/gcp_freebsd_repartition.sh
+++ b/src/tools/ci/gcp_freebsd_repartition.sh
@@ -25,4 +25,4 @@ du -hs $CIRRUS_WORKING_DIR
mv $CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR.orig
mkdir $CIRRUS_WORKING_DIR
mount -o noatime /dev/da0p3 $CIRRUS_WORKING_DIR
-cp -r $CIRRUS_WORKING_DIR.orig/* $CIRRUS_WORKING_DIR/
+cp -a $CIRRUS_WORKING_DIR.orig/ $CIRRUS_WORKING_DIR/
--
2.43.0
Attachments:
[text/plain] v2-0001-make-dist-uses-git-archive.patch (3.7K, ../[email protected]/2-v2-0001-make-dist-uses-git-archive.patch)
download | inline diff:
From 48ddd079c1530baaabf92a5650ff9a7bfa7ac3d6 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <[email protected]>
Date: Thu, 25 Jan 2024 12:28:28 +0100
Subject: [PATCH v2 1/2] make dist uses git archive
Discussion: https://www.postgresql.org/message-id/flat/40e80f77-a294-4f29-a16f-e21bc7bc75fc%40eisentraut.org
---
GNUmakefile.in | 34 ++++++++++++----------------------
meson.build | 47 +++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 59 insertions(+), 22 deletions(-)
diff --git a/GNUmakefile.in b/GNUmakefile.in
index eba569e930e..680c765dd73 100644
--- a/GNUmakefile.in
+++ b/GNUmakefile.in
@@ -87,29 +87,19 @@ update-unicode: | submake-generated-headers submake-libpgport
distdir = postgresql-$(VERSION)
dummy = =install=
+GIT = git
+
dist: $(distdir).tar.gz $(distdir).tar.bz2
- rm -rf $(distdir)
-
-$(distdir).tar: distdir
- $(TAR) chf $@ $(distdir)
-
-.INTERMEDIATE: $(distdir).tar
-
-distdir-location:
- @echo $(distdir)
-
-distdir:
- rm -rf $(distdir)* $(dummy)
- for x in `cd $(top_srcdir) && find . \( -name CVS -prune \) -o \( -name .git -prune \) -o -print`; do \
- file=`expr X$$x : 'X\./\(.*\)'`; \
- if test -d "$(top_srcdir)/$$file" ; then \
- mkdir "$(distdir)/$$file" && chmod 777 "$(distdir)/$$file"; \
- else \
- ln "$(top_srcdir)/$$file" "$(distdir)/$$file" >/dev/null 2>&1 \
- || cp "$(top_srcdir)/$$file" "$(distdir)/$$file"; \
- fi || exit; \
- done
- $(MAKE) -C $(distdir) distclean
+
+.PHONY: check-dirty-index
+check-dirty-index:
+ $(GIT) -C $(srcdir) diff-index --quiet HEAD
+
+$(distdir).tar.gz: check-dirty-index
+ $(GIT) -C $(srcdir) archive --format tar.gz --prefix $(distdir)/ HEAD -o $(abs_top_builddir)/$@
+
+$(distdir).tar.bz2: check-dirty-index
+ $(GIT) -C $(srcdir) -c tar.tar.bz2.command='$(BZIP2) -c' archive --format tar.bz2 --prefix $(distdir)/ HEAD -o $(abs_top_builddir)/$@
distcheck: dist
rm -rf $(dummy)
diff --git a/meson.build b/meson.build
index 55184db2488..c245a1818e0 100644
--- a/meson.build
+++ b/meson.build
@@ -3348,6 +3348,53 @@ run_target('help',
+###############################################################
+# Distribution archive
+###############################################################
+
+git = find_program('git', required: false, native: true)
+bzip2 = find_program('bzip2', required: false, native: true)
+
+distdir = meson.project_name() + '-' + meson.project_version()
+
+check_dirty_index = run_target('check-dirty-index',
+ command: [git, '-C', '@SOURCE_ROOT@', 'diff-index', '--quiet', 'HEAD'])
+
+tar_gz = custom_target('tar.gz',
+ build_always_stale: true,
+ command: [git, '-C', '@SOURCE_ROOT@', 'archive',
+ '--format', 'tar.gz',
+ '--prefix', distdir + '/',
+ '-o', join_paths(meson.build_root(), '@OUTPUT@'),
+ 'HEAD', '.'],
+ install: false,
+ output: distdir + '.tar.gz',
+)
+
+if bzip2.found()
+ tar_bz2 = custom_target('tar.bz2',
+ build_always_stale: true,
+ command: [git, '-C', '@SOURCE_ROOT@', '-c', 'tar.tar.bz2.command="' + bzip2.path() + '" -c', 'archive',
+ '--format', 'tar.bz2',
+ '--prefix', distdir + '/',
+ '-o', join_paths(meson.build_root(), '@OUTPUT@'),
+ 'HEAD', '.'],
+ install: false,
+ output: distdir + '.tar.bz2',
+ )
+else
+ tar_bz2 = custom_target('tar.bz2',
+ command: ['sh', '-c', 'exit 1'],
+ output: distdir + '.tar.bz2',
+ )
+endif
+
+# FIXME: would like to add check_dirty_index here, broken(?) before
+# meson 0.60.0
+alias_target('pgdist', [tar_gz, tar_bz2])
+
+
+
###############################################################
# The End, The End, My Friend
###############################################################
base-commit: 46d8587b504170ca14f064890bc7ccbbd7523f81
--
2.43.0
[text/plain] v2-0002-WIP-Add-dist-building-to-CI.patch (3.0K, ../[email protected]/3-v2-0002-WIP-Add-dist-building-to-CI.patch)
download | inline diff:
From 65055901d7476ecef67ff96f8218ca8fa344838f Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <[email protected]>
Date: Thu, 25 Jan 2024 15:35:48 +0100
Subject: [PATCH v2 2/2] WIP: Add dist building to CI
Note: freebsd repartition script didn't copy .git directory
---
.cirrus.tasks.yml | 28 +++++++++++++++++++++++++
src/tools/ci/gcp_freebsd_repartition.sh | 2 +-
2 files changed, 29 insertions(+), 1 deletion(-)
diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml
index e4e1bcfeb99..6eba3073a35 100644
--- a/.cirrus.tasks.yml
+++ b/.cirrus.tasks.yml
@@ -202,6 +202,11 @@ task:
build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
EOF
+ dist_script: |
+ su postgres -c 'meson compile -C build -v pgdist'
+ tar_artifacts:
+ path: "build*/*.tar.*"
+
on_failure:
# if the server continues running, it often causes cirrus-ci to fail
# during upload, as it doesn't expect artifacts to change size
@@ -345,6 +350,10 @@ task:
make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
EOF
+ dist_script: su postgres -c "make dist -j${BUILD_JOBS}"
+ tar_artifacts:
+ path: "build*/*.tar.*"
+
on_failure:
<<: *on_failure_ac
@@ -403,6 +412,10 @@ task:
PYTHONCOERCECLOCALE=0 LANG=C meson test $MTEST_ARGS -C build-32 --num-processes ${TEST_JOBS}
EOF
+ dist_script: su postgres -c 'meson compile -C build-32 -v pgdist'
+ tar_artifacts:
+ path: "build*/*.tar.*"
+
on_failure:
<<: *on_failure_meson
@@ -496,6 +509,10 @@ task:
ulimit -n 1024 # default is 256, pretty low
meson test $MTEST_ARGS --num-processes ${TEST_JOBS}
+ dist_script: meson compile -C build -v pgdist
+ tar_artifacts:
+ path: "build*/*.tar.*"
+
on_failure:
<<: *on_failure_meson
cores_script: src/tools/ci/cores_backtrace.sh macos "${HOME}/cores"
@@ -567,6 +584,12 @@ task:
vcvarsall x64
meson test %MTEST_ARGS% --num-processes %TEST_JOBS%
+ dist_script: |
+ vcvarsall x64
+ meson compile -C build -v pgdist
+ tar_artifacts:
+ path: "build*/*.tar.*"
+
on_failure:
<<: *on_failure_meson
crashlog_artifacts:
@@ -627,6 +650,11 @@ task:
test_world_script: |
%BASH% -c "meson test %MTEST_ARGS% --num-processes %TEST_JOBS%"
+ dist_script: |
+ %BASH% -c "meson compile -C build -v pgdist"
+ tar_artifacts:
+ path: "build*/*.tar.*"
+
on_failure:
<<: *on_failure_meson
crashlog_artifacts:
diff --git a/src/tools/ci/gcp_freebsd_repartition.sh b/src/tools/ci/gcp_freebsd_repartition.sh
index 2d5e1738998..edf38e17575 100755
--- a/src/tools/ci/gcp_freebsd_repartition.sh
+++ b/src/tools/ci/gcp_freebsd_repartition.sh
@@ -25,4 +25,4 @@ du -hs $CIRRUS_WORKING_DIR
mv $CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR.orig
mkdir $CIRRUS_WORKING_DIR
mount -o noatime /dev/da0p3 $CIRRUS_WORKING_DIR
-cp -r $CIRRUS_WORKING_DIR.orig/* $CIRRUS_WORKING_DIR/
+cp -a $CIRRUS_WORKING_DIR.orig/ $CIRRUS_WORKING_DIR/
--
2.43.0
view thread (16+ messages) latest in thread
reply
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: [email protected]
Cc: [email protected], [email protected]
Subject: Re: make dist using git archive
In-Reply-To: <[email protected]>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox