From: Andrew Dunstan <andrew.dunstan@2ndquadrant.com>
To: Peter Eisentraut <peter.eisentraut@2ndquadrant.com>
To: Michael Paquier <michael@paquier.xyz>
Cc: Juan José Santamaría Flecha <juanjo.santamaria@gmail.com>
Cc: Dagfinn Ilmari Mannsåker <ilmari@ilmari.org>
Cc: pgsql-hackers <pgsql-hackers@postgresql.org>
Subject: Re: TAP tests and symlinks on Windows
Date: Fri, 10 Jul 2020 07:58:02 -0400
Message-ID: <ea68701d-0225-b457-e729-67e9fe4f41e6@2ndQuadrant.com> (raw)
In-Reply-To: <19483ac1-0f31-cfd3-8ef7-5da0ac5a5f46@2ndQuadrant.com>
References: <87sgf4pmb8.fsf@wibble.ilmari.org>
<20200612065957.GA3362@paquier.xyz>
<CAC+AXB0VJ9A4f6Ob9b3wV8V_VGND5X3MBLKQRd0+8L6t-gMiCQ@mail.gmail.com>
<20200613060054.GM3362@paquier.xyz>
<20200615062319.GF52676@paquier.xyz>
<11c4ee09-3706-5912-ffd7-dc3657de7a5e@2ndQuadrant.com>
<20200617074434.GB118592@paquier.xyz>
<20200623105527.GA1974@paquier.xyz>
<3ebcde2b-3b78-2e5f-9249-03c709dc23dd@2ndquadrant.com>
<20200629075616.GC330401@paquier.xyz>
<20200630121328.GB67289@paquier.xyz>
<504f09d7-d600-44ca-a32b-0ed209cc2273@2ndquadrant.com>
<19483ac1-0f31-cfd3-8ef7-5da0ac5a5f46@2ndQuadrant.com>
On 7/8/20 9:54 AM, Andrew Dunstan wrote:
>
>
>
> Then, with a little more sprinkling of perl2host the pg_basebackup tests
> can be made to work on msys2.
>
>
> I'm going to prepare patches along these lines.
>
>
After much frustration and gnashing of teeth here's a patch that allows
almost all the TAP tests involving symlinks to work as expected on all
Windows build environments, without requiring an additional Perl module.
I have tested this on a system that is very similar to that running
drongo and fairywren, with both msys2 and MSVC builds.
I didn't change the name of perl2host - Sufficient unto the day is the
evil thereof. But I did modify it a) to allow use of cygpath if
available and b) to allow it to succeed if the grandparent directory
exists when cygpath isn't available.
cheers
andrew
--
Andrew Dunstan https://www.2ndQuadrant.com
PostgreSQL Development, 24x7 Support, Remote DBA, Training & Services
Attachments:
[text/x-patch] win-tap-symlink-2.patch (16.0K, ../ea68701d-0225-b457-e729-67e9fe4f41e6@2ndQuadrant.com/2-win-tap-symlink-2.patch)
download | inline diff:diff --git a/doc/src/sgml/regress.sgml b/doc/src/sgml/regress.sgmlindex d98187c970..e6e2b2762d 100644--- a/doc/src/sgml/regress.sgml+++ b/doc/src/sgml/regress.sgml@@ -775,6 +775,7 @@ make check PROVE_TESTS='t/001_test1.pl t/003_test3.pl'
<para>
The TAP tests require the Perl module <literal>IPC::Run</literal>.
This module is available from CPAN or an operating system package.
+ On Windows, <literal>Win32API::File</literal> is also required .
</para>
<para>
diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.plindex 208df557b8..38b2a35c3b 100644--- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl+++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl@@ -211,154 +211,168 @@ $node->command_fails(
'pg_basebackup tar with long name fails');
unlink "$pgdata/$superlongname";
-# The following tests test symlinks. Windows doesn't have symlinks, so-# skip on Windows.+# The following tests are for symlinks.++# Move pg_replslot out of $pgdata and create a symlink to it.+$node->stop;++# Set umask so test directories and files are created with group permissions+umask(0027);++# Enable group permissions on PGDATA+chmod_recursive("$pgdata", 0750, 0640);++rename("$pgdata/pg_replslot", "$tempdir/pg_replslot")+ or BAIL_OUT "could not move $pgdata/pg_replslot";+dir_symlink("$tempdir/pg_replslot", "$pgdata/pg_replslot")+ or BAIL_OUT "could not symlink to $pgdata/pg_replslot";++$node->start;++# Create a temporary directory in the system location and symlink it+# to our physical temp location. That way we can use shorter names+# for the tablespace directories, which hopefully won't run afoul of+# the 99 character length limit.+my $shorter_tempdir = TestLib::tempdir_short . "/tempdir";+dir_symlink "$tempdir", $shorter_tempdir;++mkdir "$tempdir/tblspc1";+my $realTsDir = TestLib::perl2host("$shorter_tempdir/tblspc1");+my $real_tempdir = TestLib::perl2host($tempdir);+$node->safe_psql('postgres',+ "CREATE TABLESPACE tblspc1 LOCATION '$realTsDir';");+$node->safe_psql('postgres',+ "CREATE TABLE test1 (a int) TABLESPACE tblspc1;");+$node->command_ok([ 'pg_basebackup', '-D', "$real_tempdir/tarbackup2", '-Ft' ],+ 'tar format with tablespaces');+ok(-f "$tempdir/tarbackup2/base.tar", 'backup tar was created');+my @tblspc_tars = glob "$tempdir/tarbackup2/[0-9]*.tar";+is(scalar(@tblspc_tars), 1, 'one tablespace tar was created');+rmtree("$tempdir/tarbackup2");++# Create an unlogged table to test that forks other than init are not copied.+$node->safe_psql('postgres',+ 'CREATE UNLOGGED TABLE tblspc1_unlogged (id int) TABLESPACE tblspc1;'+ );++my $tblspc1UnloggedPath = $node->safe_psql('postgres',+ q{select pg_relation_filepath('tblspc1_unlogged')});++# Make sure main and init forks exist+ok( -f "$pgdata/${tblspc1UnloggedPath}_init",+ 'unlogged init fork in tablespace');+ok(-f "$pgdata/$tblspc1UnloggedPath", 'unlogged main fork in tablespace');++# Create files that look like temporary relations to ensure they are ignored+# in a tablespace.+my @tempRelationFiles = qw(t888_888 t888888_888888_vm.1);+my $tblSpc1Id = basename(+ dirname(+ dirname(+ $node->safe_psql(+ 'postgres', q{select pg_relation_filepath('test1')}))));++foreach my $filename (@tempRelationFiles)+{+ append_to_file(+ "$shorter_tempdir/tblspc1/$tblSpc1Id/$postgresOid/$filename",+ 'TEMP_RELATION');+}++$node->command_fails(+ [ 'pg_basebackup', '-D', "$tempdir/backup1", '-Fp' ],+ 'plain format with tablespaces fails without tablespace mapping');++$node->command_ok(+ [+ 'pg_basebackup', '-D', "$tempdir/backup1", '-Fp',+ "-T$realTsDir=$real_tempdir/tbackup/tblspc1"+ ],+ 'plain format with tablespaces succeeds with tablespace mapping');+ok(-d "$tempdir/tbackup/tblspc1", 'tablespace was relocated');++# This symlink check is not supported on Windows as -l+# doesn't work with junctions
SKIP:
{
- skip "symlinks not supported on Windows", 18 if ($windows_os);-- # Move pg_replslot out of $pgdata and create a symlink to it.- $node->stop;-- # Set umask so test directories and files are created with group permissions- umask(0027);-- # Enable group permissions on PGDATA- chmod_recursive("$pgdata", 0750, 0640);-- rename("$pgdata/pg_replslot", "$tempdir/pg_replslot")- or BAIL_OUT "could not move $pgdata/pg_replslot";- symlink("$tempdir/pg_replslot", "$pgdata/pg_replslot")- or BAIL_OUT "could not symlink to $pgdata/pg_replslot";-- $node->start;-- # Create a temporary directory in the system location and symlink it- # to our physical temp location. That way we can use shorter names- # for the tablespace directories, which hopefully won't run afoul of- # the 99 character length limit.- my $shorter_tempdir = TestLib::tempdir_short . "/tempdir";- symlink "$tempdir", $shorter_tempdir;-- mkdir "$tempdir/tblspc1";- $node->safe_psql('postgres',- "CREATE TABLESPACE tblspc1 LOCATION '$shorter_tempdir/tblspc1';");- $node->safe_psql('postgres',- "CREATE TABLE test1 (a int) TABLESPACE tblspc1;");- $node->command_ok([ 'pg_basebackup', '-D', "$tempdir/tarbackup2", '-Ft' ],- 'tar format with tablespaces');- ok(-f "$tempdir/tarbackup2/base.tar", 'backup tar was created');- my @tblspc_tars = glob "$tempdir/tarbackup2/[0-9]*.tar";- is(scalar(@tblspc_tars), 1, 'one tablespace tar was created');- rmtree("$tempdir/tarbackup2");-- # Create an unlogged table to test that forks other than init are not copied.- $node->safe_psql('postgres',- 'CREATE UNLOGGED TABLE tblspc1_unlogged (id int) TABLESPACE tblspc1;'- );-- my $tblspc1UnloggedPath = $node->safe_psql('postgres',- q{select pg_relation_filepath('tblspc1_unlogged')});-- # Make sure main and init forks exist- ok( -f "$pgdata/${tblspc1UnloggedPath}_init",- 'unlogged init fork in tablespace');- ok(-f "$pgdata/$tblspc1UnloggedPath", 'unlogged main fork in tablespace');-- # Create files that look like temporary relations to ensure they are ignored- # in a tablespace.- my @tempRelationFiles = qw(t888_888 t888888_888888_vm.1);- my $tblSpc1Id = basename(- dirname(- dirname(- $node->safe_psql(- 'postgres', q{select pg_relation_filepath('test1')}))));-- foreach my $filename (@tempRelationFiles)- {- append_to_file(- "$shorter_tempdir/tblspc1/$tblSpc1Id/$postgresOid/$filename",- 'TEMP_RELATION');- }-- $node->command_fails(- [ 'pg_basebackup', '-D', "$tempdir/backup1", '-Fp' ],- 'plain format with tablespaces fails without tablespace mapping');-- $node->command_ok(- [- 'pg_basebackup', '-D', "$tempdir/backup1", '-Fp',- "-T$shorter_tempdir/tblspc1=$tempdir/tbackup/tblspc1"- ],- 'plain format with tablespaces succeeds with tablespace mapping');- ok(-d "$tempdir/tbackup/tblspc1", 'tablespace was relocated');+ skip "symlink check not implemented on Windows", 1+ if ($windows_os);
opendir(my $dh, "$pgdata/pg_tblspc") or die;
ok( ( grep {
- -l "$tempdir/backup1/pg_tblspc/$_"- and readlink "$tempdir/backup1/pg_tblspc/$_" eq- "$tempdir/tbackup/tblspc1"- } readdir($dh)),+ -l "$tempdir/backup1/pg_tblspc/$_"+ and readlink "$tempdir/backup1/pg_tblspc/$_" eq+ "$tempdir/tbackup/tblspc1"+ } readdir($dh)),
"tablespace symlink was updated");
closedir $dh;
+}++# Group access should be enabled on all backup files+SKIP:+{+ skip "unix-style permissions not supported on Windows", 1+ if ($windows_os);- # Group access should be enabled on all backup files
ok(check_mode_recursive("$tempdir/backup1", 0750, 0640),
- "check backup dir permissions");+ "check backup dir permissions");+}++# Unlogged relation forks other than init should not be copied+my ($tblspc1UnloggedBackupPath) =+ $tblspc1UnloggedPath =~ /[^\/]*\/[^\/]*\/[^\/]*$/g;++ok(-f "$tempdir/tbackup/tblspc1/${tblspc1UnloggedBackupPath}_init",+ 'unlogged init fork in tablespace backup');+ok(!-f "$tempdir/tbackup/tblspc1/$tblspc1UnloggedBackupPath",+ 'unlogged main fork not in tablespace backup');- # Unlogged relation forks other than init should not be copied- my ($tblspc1UnloggedBackupPath) =- $tblspc1UnloggedPath =~ /[^\/]*\/[^\/]*\/[^\/]*$/g;-- ok(-f "$tempdir/tbackup/tblspc1/${tblspc1UnloggedBackupPath}_init",- 'unlogged init fork in tablespace backup');- ok(!-f "$tempdir/tbackup/tblspc1/$tblspc1UnloggedBackupPath",- 'unlogged main fork not in tablespace backup');-- # Temp relations should not be copied.- foreach my $filename (@tempRelationFiles)- {- ok( !-f "$tempdir/tbackup/tblspc1/$tblSpc1Id/$postgresOid/$filename",- "[tblspc1]/$postgresOid/$filename not copied");-- # Also remove temp relation files or tablespace drop will fail.- my $filepath =- "$shorter_tempdir/tblspc1/$tblSpc1Id/$postgresOid/$filename";-- unlink($filepath)- or BAIL_OUT("unable to unlink $filepath");- }-- ok( -d "$tempdir/backup1/pg_replslot",- 'pg_replslot symlink copied as directory');- rmtree("$tempdir/backup1");-- mkdir "$tempdir/tbl=spc2";- $node->safe_psql('postgres', "DROP TABLE test1;");- $node->safe_psql('postgres', "DROP TABLE tblspc1_unlogged;");- $node->safe_psql('postgres', "DROP TABLESPACE tblspc1;");- $node->safe_psql('postgres',- "CREATE TABLESPACE tblspc2 LOCATION '$shorter_tempdir/tbl=spc2';");- $node->command_ok(- [- 'pg_basebackup', '-D', "$tempdir/backup3", '-Fp',- "-T$shorter_tempdir/tbl\\=spc2=$tempdir/tbackup/tbl\\=spc2"- ],- 'mapping tablespace with = sign in path');- ok(-d "$tempdir/tbackup/tbl=spc2",- 'tablespace with = sign was relocated');- $node->safe_psql('postgres', "DROP TABLESPACE tblspc2;");- rmtree("$tempdir/backup3");-- mkdir "$tempdir/$superlongname";- $node->safe_psql('postgres',- "CREATE TABLESPACE tblspc3 LOCATION '$tempdir/$superlongname';");- $node->command_ok(- [ 'pg_basebackup', '-D', "$tempdir/tarbackup_l3", '-Ft' ],- 'pg_basebackup tar with long symlink target');- $node->safe_psql('postgres', "DROP TABLESPACE tblspc3;");- rmtree("$tempdir/tarbackup_l3");+# Temp relations should not be copied.+foreach my $filename (@tempRelationFiles)+{+ ok( !-f "$tempdir/tbackup/tblspc1/$tblSpc1Id/$postgresOid/$filename",+ "[tblspc1]/$postgresOid/$filename not copied");++ # Also remove temp relation files or tablespace drop will fail.+ my $filepath =+ "$shorter_tempdir/tblspc1/$tblSpc1Id/$postgresOid/$filename";++ unlink($filepath)+ or BAIL_OUT("unable to unlink $filepath");
}
+ok( -d "$tempdir/backup1/pg_replslot",+ 'pg_replslot symlink copied as directory');+rmtree("$tempdir/backup1");++mkdir "$tempdir/tbl=spc2";+$realTsDir = TestLib::perl2host("$shorter_tempdir/tbl=spc2");+$node->safe_psql('postgres', "DROP TABLE test1;");+$node->safe_psql('postgres', "DROP TABLE tblspc1_unlogged;");+$node->safe_psql('postgres', "DROP TABLESPACE tblspc1;");+$node->safe_psql('postgres',+ "CREATE TABLESPACE tblspc2 LOCATION '$realTsDir';");+$realTsDir =~ s/=/\\=/;+$node->command_ok(+ [+ 'pg_basebackup', '-D', "$tempdir/backup3", '-Fp',+ "-T$realTsDir=$real_tempdir/tbackup/tbl\\=spc2"+ ],+ 'mapping tablespace with = sign in path');+ok(-d "$tempdir/tbackup/tbl=spc2",+ 'tablespace with = sign was relocated');+$node->safe_psql('postgres', "DROP TABLESPACE tblspc2;");+rmtree("$tempdir/backup3");++mkdir "$tempdir/$superlongname";+$realTsDir = TestLib::perl2host("$shorter_tempdir/$superlongname");+$node->safe_psql('postgres',+ "CREATE TABLESPACE tblspc3 LOCATION '$realTsDir';");+$node->command_ok(+ [ 'pg_basebackup', '-D', "$tempdir/tarbackup_l3", '-Ft' ],+ 'pg_basebackup tar with long symlink target');+$node->safe_psql('postgres', "DROP TABLESPACE tblspc3;");+rmtree("$tempdir/tarbackup_l3");+
$node->command_ok([ 'pg_basebackup', '-D', "$tempdir/backupR", '-R' ],
'pg_basebackup -R runs');
ok(-f "$tempdir/backupR/postgresql.auto.conf", 'postgresql.auto.conf exists');
diff --git a/src/bin/pg_dump/t/010_dump_connstr.pl b/src/bin/pg_dump/t/010_dump_connstr.plindex abdb07c558..1fd8a78abc 100644--- a/src/bin/pg_dump/t/010_dump_connstr.pl+++ b/src/bin/pg_dump/t/010_dump_connstr.pl@@ -5,7 +5,7 @@ use PostgresNode;
use TestLib;
use Test::More;
-if ($^O eq 'msys' && `uname -or` =~ /^[2-9].*Msys/)+if ($TestLib::is_msys2)
{
plan skip_all => 'High bit name tests fail on Msys2';
}
diff --git a/src/bin/pg_rewind/t/004_pg_xlog_symlink.pl b/src/bin/pg_rewind/t/004_pg_xlog_symlink.plindex 3813543ee1..fff4758508 100644--- a/src/bin/pg_rewind/t/004_pg_xlog_symlink.pl+++ b/src/bin/pg_rewind/t/004_pg_xlog_symlink.pl@@ -6,16 +6,7 @@ use warnings;
use File::Copy;
use File::Path qw(rmtree);
use TestLib;
-use Test::More;-if ($windows_os)-{- plan skip_all => 'symlinks not supported on Windows';- exit;-}-else-{- plan tests => 5;-}+use Test::More tests => 5;
use FindBin;
use lib $FindBin::RealBin;
@@ -36,7 +27,7 @@ sub run_test
# turn pg_wal into a symlink
print("moving $test_primary_datadir/pg_wal to $primary_xlogdir\n");
move("$test_primary_datadir/pg_wal", $primary_xlogdir) or die;
- symlink($primary_xlogdir, "$test_primary_datadir/pg_wal") or die;+ dir_symlink($primary_xlogdir, "$test_primary_datadir/pg_wal") or die;
RewindTest::start_primary();
diff --git a/src/test/perl/TestLib.pm b/src/test/perl/TestLib.pmindex a7490d2ce7..207e92fdc0 100644--- a/src/test/perl/TestLib.pm+++ b/src/test/perl/TestLib.pm@@ -67,6 +67,7 @@ our @EXPORT = qw(
check_mode_recursive
chmod_recursive
check_pg_config
+ dir_symlink
system_or_bail
system_log
run_log
@@ -84,10 +85,11 @@ our @EXPORT = qw(
command_checks_all
$windows_os
+ $is_msys2
$use_unix_sockets
);
-our ($windows_os, $use_unix_sockets, $tmp_check, $log_path, $test_logfile);+our ($windows_os, $is_msys2, $use_unix_sockets, $tmp_check, $log_path, $test_logfile);
BEGIN
{
@@ -114,6 +116,9 @@ BEGIN
# Must be set early
$windows_os = $Config{osname} eq 'MSWin32' || $Config{osname} eq 'msys';
+ # Check if this environment is MSYS2.+ $is_msys2 = $^O eq 'msys' && `uname -or` =~ /^[2-9].*Msys/;+
if ($windows_os)
{
require Win32API::File;
@@ -137,6 +142,10 @@ BEGIN
Set to true when running under Windows, except on Cygwin.
+=item C<$is_msys2>++Set to true when running under MSYS2.+
=back
=cut
@@ -263,9 +272,10 @@ sub tempdir_short
=item perl2host()
-Translate a Perl file name to a host file name. Currently, this is a no-op+Translate a virtual file name to a host file name. Currently, this is a no-op
except for the case of Perl=msys and host=mingw32. The subject need not
-exist, but its parent directory must exist.+exist, but its parent or grandparent directory must exist unless cygpath is+available.
=cut
@@ -273,6 +283,17 @@ sub perl2host
{
my ($subject) = @_;
return $subject unless $Config{osname} eq 'msys';
+ if ($is_msys2)+ {+ # get absolute, windows type path+ my $path = qx{cygpath -a -w "$subject"};+ if (! $?)+ {+ chomp $path;+ return $path if $path;+ }+ # fall through if this didn't work.+ }
my $here = cwd;
my $leaf;
if (chdir $subject)
@@ -283,7 +304,12 @@ sub perl2host
{
$leaf = '/' . basename $subject;
my $parent = dirname $subject;
- chdir $parent or die "could not chdir \"$parent\": $!";+ if (! chdir $parent)+ {+ $leaf = '/' . basename ($parent) . $leaf;+ $parent = dirname $parent;+ chdir $parent or die "could not chdir \"$parent\": $!";+ }
}
# this odd way of calling 'pwd -W' is the only way that seems to work.
@@ -602,6 +628,41 @@ sub check_pg_config
=pod
+=item dir_symlink(oldname, newname)++Portably create a symlink for a director. On Windows this creates a junction.+Elsewhere it just calls perl's builtin symlink.++=cut++sub dir_symlink+{+ my $oldname = shift;+ my $newname = shift;+ if ($windows_os)+ {+ $oldname = perl2host($oldname);+ $newname = perl2host($newname);+ $oldname =~ s,/,\\,g;+ $newname =~ s,/,\\,g;+ my $cmd = qq{mklink /j "$newname" "$oldname"};+ if ($Config{osname} eq 'msys')+ {+ # need some indirection on msys+ $cmd = qq{echo '$cmd' | \$COMSPEC /Q};+ }+ note("dir_symlink cmd: $cmd");+ system($cmd);+ }+ else+ {+ symlink $oldname, $newname;+ }+ die "No $newname" unless -e $newname;+}++=pod+
=back
=head1 Test::More-LIKE METHODS
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: pgsql-hackers@postgresql.org
Cc: andrew.dunstan@2ndquadrant.com, peter.eisentraut@2ndquadrant.com, michael@paquier.xyz, juanjo.santamaria@gmail.com, ilmari@ilmari.org
Subject: Re: TAP tests and symlinks on Windows
In-Reply-To: <ea68701d-0225-b457-e729-67e9fe4f41e6@2ndQuadrant.com>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by DDX for PostgreSQL; see mirroring instructions
for how to clone and mirror all data and code used for this inbox