pgbackrest.git / summary / log / commit / refs
commit e1829bb2d269192fa5af4effcf74e53b293d4ea0
Author: David Steele <github@thelabyrinth.net>
Date: Mon Jun 23 22:54:00 2014 +0000
First fully working backup in a while.
bin/pg_backrest.pl | 2 +-
lib/BackRest/Backup.pm | 33 +++++++++++-----
lib/BackRest/File.pm | 76 +++++++++++++++++++++++--------------
lib/BackRest/Utility.pm | 2 +-
test/lib/BackRestTest/BackupTest.pm | 14 +++----
test/lib/BackRestTest/CommonTest.pm | 16 ++++----
test/lib/BackRestTest/FileTest.pm | 18 +++------
7 files changed, 93 insertions(+), 68 deletions(-)
diff --git a/bin/pg_backrest.pl b/bin/pg_backrest.pl
index 1e963d60d..b93c660c1 100755
--- a/bin/pg_backrest.pl
+++ b/bin/pg_backrest.pl
@@ -356,7 +356,7 @@ if ($strOperation eq OP_ARCHIVE_PUSH || $strOperation eq OP_ARCHIVE_PULL)
&log(INFO, "pushing archive log " . $ARGV[1] . ($bCompressAsync ? " asynchronously" : ""));
- archive_push($ARGV[1]);
+ archive_push(config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_PATH), $ARGV[1]);
# Only continue if we are archiving local and a backup server is defined
if (!($strSection eq CONFIG_SECTION_ARCHIVE && defined(config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST))))
diff --git a/lib/BackRest/Backup.pm b/lib/BackRest/Backup.pm
index dd9617921..a025c438c 100644
--- a/lib/BackRest/Backup.pm
+++ b/lib/BackRest/Backup.pm
@@ -258,8 +258,20 @@ sub archive_get
####################################################################################################################################
sub archive_push
{
+ my $strDbClusterPath = shift;
my $strSourceFile = shift;
-
+
+ # If the source file path is not absolute then it is relative to the data path
+ if (index($strSourceFile, '/',) != 0)
+ {
+ if (!defined($strDbClusterPath))
+ {
+ confess &log(ERROR, "database path must be set if relative xlog paths are used");
+ }
+
+ $strSourceFile = "${strDbClusterPath}/${strSourceFile}";
+ }
+
# Get the destination file
my $strDestinationFile = basename($strSourceFile);
@@ -273,12 +285,12 @@ sub archive_push
}
# Copy the archive file
- $oFile->file_copy(PATH_DB_ABSOLUTE, $strSourceFile, # Source file
- PATH_BACKUP_ARCHIVE, $strDestinationFile, # Destination file
- false, # Source is not compressed
- false, # !!! For now, compress destination
- undef, undef, undef, # Unused params
- true); # Create path if it does not exist
+ $oFile->copy(PATH_DB_ABSOLUTE, $strSourceFile, # Source file
+ PATH_BACKUP_ARCHIVE, $strDestinationFile, # Destination file
+ false, # Source is not compressed
+ false, # !!! For now, compress destination
+ undef, undef, undef, # Unused params
+ true); # Create path if it does not exist
}
####################################################################################################################################
@@ -1377,7 +1389,10 @@ sub backup
&log(DEBUG, "archiving: ${strArchive} (${stryArchiveFile[0]})");
- $oFile->file_copy(PATH_BACKUP_ARCHIVE, $stryArchiveFile[0], PATH_BACKUP_TMP, "base/pg_xlog/${strArchive}");
+ $oFile->copy(PATH_BACKUP_ARCHIVE, $stryArchiveFile[0],
+ PATH_BACKUP_TMP, "base/pg_xlog/${strArchive}",
+ $stryArchiveFile[0] =~ "^.*\.$oFile->{strCompressExtension}\$",
+ false); # !!! THIS NEEDS TO BE FIXED
}
}
@@ -1389,7 +1404,7 @@ sub backup
# Rename the backup tmp path to complete the backup
&log(DEBUG, "moving ${strBackupTmpPath} to " . $oFile->path_get(PATH_BACKUP_CLUSTER, $strBackupPath));
- $oFile->file_move(PATH_BACKUP_TMP, undef, PATH_BACKUP_CLUSTER, $strBackupPath);
+ $oFile->move(PATH_BACKUP_TMP, undef, PATH_BACKUP_CLUSTER, $strBackupPath);
}
####################################################################################################################################
diff --git a/lib/BackRest/File.pm b/lib/BackRest/File.pm
index bf89ace1b..fc716f2ea 100644
--- a/lib/BackRest/File.pm
+++ b/lib/BackRest/File.pm
@@ -442,44 +442,33 @@ sub move
# Run locally
else
{
- # If the destination path does not exist, create it or error out
- if (!$self->exists($strDestinationPathType, dirname($strDestinationFile)))
+ if (!rename($strPathOpSource, $strPathOpDestination))
{
- if ($bDestinationPathCreate)
+ my $strError = "${strPathOpDestination} could not be moved: " . $!;
+ my $iErrorCode = COMMAND_ERR_FILE_READ;
+
+ if (!$self->exists(PATH_ABSOLUTE, dirname($strPathOpDestination)))
{
- $self->path_create($strDestinationPathType, dirname($strDestinationFile));
+ $strError = "${strPathOpDestination} does not exist";
+ $iErrorCode = COMMAND_ERR_FILE_MISSING;
}
- else
- {
- my $strError = "destination " . dirname($strPathOpDestination) . " does not exist";
+ if (!($bDestinationPathCreate && $iErrorCode == COMMAND_ERR_FILE_MISSING))
+ {
if ($strSourcePathType eq PATH_ABSOLUTE)
{
- print $strError;
- exit (COMMAND_ERR_PATH_MISSING);
+ confess &log(ERROR, $strError, $iErrorCode);
}
confess &log(ERROR, "${strDebug}: " . $strError);
}
- }
-
- if (!rename($strPathOpSource, $strPathOpDestination))
- {
- my $strError = "${strPathOpSource} could not be moved: " . $!;
- my $iErrorCode = COMMAND_ERR_FILE_MOVE;
- if (!$self->exists($strSourcePathType, dirname($strSourceFile)))
- {
- $strError = "${strPathOpSource} does not exist";
- $iErrorCode = COMMAND_ERR_FILE_MISSING;
- }
+ $self->path_create(PATH_ABSOLUTE, dirname($strPathOpDestination));
- if ($strSourcePathType eq PATH_ABSOLUTE)
+ if (!rename($strPathOpSource, $strPathOpDestination))
{
- confess &log(ERROR, $strError, $iErrorCode);
+ confess &log(ERROR, "unable to open move file ${strPathOpSource}: " . $!);
}
-
- confess &log(ERROR, "${strDebug}: " . $strError);
}
}
}
@@ -1110,12 +1099,13 @@ sub copy
my $bIgnoreMissingSource = shift;
my $lModificationTime = shift;
my $strPermission = shift;
- my $bPathCreate = shift;
+ my $bDestinationPathCreate = shift;
# Set defaults
$bSourceCompressed = defined($bSourceCompressed) ? $bSourceCompressed : false;
$bDestinationCompress = defined($bDestinationCompress) ? $bDestinationCompress : false;
$bIgnoreMissingSource = defined($bIgnoreMissingSource) ? $bIgnoreMissingSource : false;
+ $bDestinationPathCreate = defined($bDestinationPathCreate) ? $bDestinationPathCreate : false;
# Set working variables
my $bSourceRemote = $self->is_remote($strSourcePathType) || $strSourcePathType eq PIPE_STDIN;
@@ -1134,7 +1124,8 @@ sub copy
(defined($strDestinationOp) ? ":${strDestinationFile}" : "") .
", source_compressed = " . ($bSourceCompressed ? "true" : "false") .
", destination_compress = " . ($bDestinationCompress ? "true" : "false") .
- ", ignore_missing_source = " . ($bIgnoreMissingSource ? "true" : "false");
+ ", ignore_missing_source = " . ($bIgnoreMissingSource ? "true" : "false") .
+ ", destination_path_create = " . ($bDestinationPathCreate ? "true" : "false");
&log(DEBUG, OP_FILE_COPY . ": ${strDebug}");
# Open the source and destination files (if needed)
@@ -1177,8 +1168,35 @@ sub copy
if (!$bDestinationRemote)
{
- open($hDestinationFile, ">", $strDestinationTmpOp)
- or confess &log(ERROR, "cannot open ${strDestinationTmpOp}: " . $!);
+ # Open the destination temp file
+ if (!open($hDestinationFile, ">", $strDestinationTmpOp))
+ {
+ my $strError = "${strDestinationTmpOp} could not be opened: " . $!;
+ my $iErrorCode = COMMAND_ERR_FILE_READ;
+
+ if (!$self->exists(PATH_ABSOLUTE, dirname($strDestinationOp)))
+ {
+ $strError = "${strDestinationOp} does not exist";
+ $iErrorCode = COMMAND_ERR_FILE_MISSING;
+ }
+
+ if (!($bDestinationPathCreate && $iErrorCode == COMMAND_ERR_FILE_MISSING))
+ {
+ if ($strSourcePathType eq PATH_ABSOLUTE)
+ {
+ confess &log(ERROR, $strError, $iErrorCode);
+ }
+
+ confess &log(ERROR, "${strDebug}: " . $strError);
+ }
+
+ $self->path_create(PATH_ABSOLUTE, dirname($strDestinationOp));
+
+ if (!open($hDestinationFile, ">", $strDestinationTmpOp))
+ {
+ confess &log(ERROR, "unable to open destination file ${strDestinationOp}: " . $!);
+ }
+ }
}
# If source or destination are remote
@@ -1357,7 +1375,7 @@ sub copy
}
# Move the file from tmp to final destination
- $self->move(PATH_ABSOLUTE, $strDestinationTmpOp, PATH_ABSOLUTE, $strDestinationOp, $bPathCreate);
+ $self->move(PATH_ABSOLUTE, $strDestinationTmpOp, PATH_ABSOLUTE, $strDestinationOp, false);
}
return true;
diff --git a/lib/BackRest/Utility.pm b/lib/BackRest/Utility.pm
index 552f01955..6d7e83fa9 100644
--- a/lib/BackRest/Utility.pm
+++ b/lib/BackRest/Utility.pm
@@ -293,7 +293,7 @@ sub date_string_get
sub log_file_set
{
my $strFile = shift;
-
+
unless (-e dirname($strFile))
{
mkdir(dirname($strFile)) or die "unable to create directory for log file ${strFile}";
diff --git a/test/lib/BackRestTest/BackupTest.pm b/test/lib/BackRestTest/BackupTest.pm
index 80226fbe4..4284d3070 100755
--- a/test/lib/BackRestTest/BackupTest.pm
+++ b/test/lib/BackRestTest/BackupTest.pm
@@ -38,7 +38,7 @@ my $strUserBackRest;
sub BackRestTestBackup_ClusterDrop
{
my $strPath = shift;
-
+
# If the db directory already exists, stop the cluster and remove the directory
if (-e $strPath . "/postmaster.pid")
{
@@ -53,13 +53,13 @@ sub BackRestTestBackup_ClusterCreate
{
my $strPath = shift;
my $iPort = shift;
-
- my $strArchive = BackRestTestCommon_CommandMainGet() . " --stanza=" . BackRestTestCommon_StanzaGet() .
+
+ my $strArchive = BackRestTestCommon_CommandMainGet() . " --stanza=" . BackRestTestCommon_StanzaGet() .
" --config=" . BackRestTestCommon_DbPathGet() . "/pg_backrest.conf archive-push %p";
BackRestTestCommon_Execute("initdb -D $strPath -A trust");
- BackRestTestCommon_Execute("/Library/PostgreSQL/9.3/bin/pg_ctl start -o \"-c port=$iPort -c checkpoint_segments=1 -c wal_level=archive " .
- "-c archive_mode=on -c archive_command='$strArchive'\" " .
+ BackRestTestCommon_Execute("/Library/PostgreSQL/9.3/bin/pg_ctl start -o \"-c port=$iPort -c checkpoint_segments=1 -c wal_level=archive " .
+ "-c archive_mode=on -c archive_command='$strArchive'\" " .
"-D $strPath -l $strPath/postgresql.log -w -s");
}
@@ -92,7 +92,7 @@ sub BackRestTestBackup_Setup
# Create the db/common directory
mkdir($strTestPath . "/db/common") or confess "Unable to create ${strTestPath}/db/common path";
-
+
# Create the cluster
BackRestTestBackup_ClusterCreate($strTestPath . "/db/common", BackRestTestCommon_DbPortGet);
@@ -131,7 +131,7 @@ sub BackRestTestBackup_Test
BackRestTestCommon_ConfigCreate(BackRestTestCommon_DbPathGet() . '/pg_backrest.conf', REMOTE_DB);#, REMOTE_BACKUP);
BackRestTestCommon_ConfigCreate(BackRestTestCommon_BackupPathGet() . '/pg_backrest.conf', REMOTE_BACKUP);#, REMOTE_DB);
-
+
BackRestTestCommon_Execute(BackRestTestCommon_CommandMainGet() . ' --config=' . BackRestTestCommon_BackupPathGet() .
"/pg_backrest.conf --type=incr --stanza=${strStanza} backup");
diff --git a/test/lib/BackRestTest/CommonTest.pm b/test/lib/BackRestTest/CommonTest.pm
index 21f1e2934..6f8d7db1a 100755
--- a/test/lib/BackRestTest/CommonTest.pm
+++ b/test/lib/BackRestTest/CommonTest.pm
@@ -49,7 +49,7 @@ sub BackRestTestCommon_Execute
{
my $strCommand = shift;
my $bSuppressError = shift;
-
+
if (system($strCommand) != 0)
{
if (!defined($bSuppressError) || !$bSuppressError)
@@ -66,9 +66,9 @@ sub BackRestTestCommon_ExecuteBackRest
{
my $strCommand = shift;
my $bSuppressError = shift;
-
+
$strCommand = "ssh ${strCommonUserBackRest}\@${strCommonHost} '${strCommand}'";
-
+
BackRestTestCommon_Execute($strCommand, $bSuppressError);
}
@@ -101,7 +101,7 @@ sub BackRestTestCommon_ConfigCreate
my $strLocal = shift;
my $strRemote = shift;
my $oParamHashRef = shift;
-
+
my %oParamHash;
tie %oParamHash, 'Config::IniFiles';
@@ -118,10 +118,9 @@ sub BackRestTestCommon_ConfigCreate
$oParamHash{$strCommonStanza}{'host'} = $strCommonHost;
$oParamHash{$strCommonStanza}{'user'} = $strCommonUser;
}
-
+
if ($strLocal eq REMOTE_BACKUP)
{
- $oParamHash{$strCommonStanza}{'path'} = $strCommonDbCommonPath;
$oParamHash{'db:command:option'}{'psql'} = "--port=${iCommonDbPort}";
}
elsif ($strLocal eq REMOTE_DB)
@@ -132,13 +131,14 @@ sub BackRestTestCommon_ConfigCreate
confess "invalid local type ${strLocal}";
}
+ $oParamHash{$strCommonStanza}{'path'} = $strCommonDbCommonPath;
$oParamHash{'global:backup'}{'path'} = $strCommonBackupPath;
-
+
$oParamHash{'global:log'}{'level-console'} = 'debug';
$oParamHash{'global:log'}{'level-file'} = 'trace';
tied(%oParamHash)->WriteConfig($strFile) or die "could not write config file ${strFile}";
-
+
chmod(0770, $strFile) or die "unable to set permissions for ${strFile}";
}
diff --git a/test/lib/BackRestTest/FileTest.pm b/test/lib/BackRestTest/FileTest.pm
index fd37b7d96..d52a892c3 100755
--- a/test/lib/BackRestTest/FileTest.pm
+++ b/test/lib/BackRestTest/FileTest.pm
@@ -119,14 +119,11 @@ sub BackRestTestFile_Test
oRemote => $bRemote ? $oRemote : undef
))->clone();
- # Loop through exists (does the paren path exist?)
- for (my $bExists = 0; $bExists <= 1; $bExists++)
- {
- # Loop through exists (does the paren path exist?)
+ # Loop through error
for (my $bError = 0; $bError <= 1; $bError++)
{
# Loop through permission (permission will be set on true)
- for (my $bPermission = 0; $bPermission <= $bExists; $bPermission++)
+ for (my $bPermission = 0; $bPermission <= 1; $bPermission++)
{
my $strPathType = PATH_BACKUP_CLUSTER;
@@ -138,7 +135,7 @@ sub BackRestTestFile_Test
}
&log(INFO, "run ${iRun} - " .
- "remote ${bRemote}, exists ${bExists}, error ${bError}, permission ${bPermission}");
+ "remote ${bRemote}, error ${bError}, permission ${bPermission}");
# Setup test directory
BackRestTestFile_Setup($bError);
@@ -161,13 +158,9 @@ sub BackRestTestFile_Test
$strPath = "${strTestPath}/private/path";
$strPathType = PATH_BACKUP_ABSOLUTE;
}
- elsif (!$bExists)
- {
- $strPath = "error/path";
- }
# Execute in eval to catch errors
- my $bErrorExpected = !$bExists || $bError;
+ my $bErrorExpected = $bError;
eval
{
@@ -216,7 +209,6 @@ sub BackRestTestFile_Test
}
}
}
- }
}
}
@@ -960,7 +952,7 @@ sub BackRestTestFile_Test
$strMessage = $oMessage;
}
- if ($bError && defined($iCode) && $iCode == COMMAND_ERR_FILE_READ)
+ if ($bError)
{
next;
}
[parent: 88bbbb4222a5]