pgbackrest.git / summary / log / commit / refs
commit f047cd0dfd3de01ff4ea678f394af8302e89e7c0
Author: David Steele <david@pgmasters.net>
Date: Sat Mar 10 04:00:20 2018 +0000
Improve code generation performance.
build/lib/pgBackRestBuild/Build.pm | 21 +-----
doc/xml/release.xml | 2 +-
libc/build/lib/pgBackRestLibC/Build.pm | 20 +----
test/lib/pgBackRestTest/Common/BuildTest.pm | 57 +++++++++++++++
test/test.pl | 109 +++++++++++++---------------
5 files changed, 114 insertions(+), 95 deletions(-)
diff --git a/build/lib/pgBackRestBuild/Build.pm b/build/lib/pgBackRestBuild/Build.pm
index a59d86be4..776923bbf 100644
--- a/build/lib/pgBackRestBuild/Build.pm
+++ b/build/lib/pgBackRestBuild/Build.pm
@@ -162,25 +162,8 @@ sub buildAll
$strExt = $strFileType eq BLD_C ? $strFileExt : "${strFileExt}h";
}
- # Only save the file if the content has changed
- my $strFileName = "${strPath}/${strFile}.auto.${strExt}";
- my $strContent = trim($rhSource->{$strFileType}) . "\n";
- my $bChanged = true;
-
- if ($oStorage->exists($strFileName))
- {
- my $strOldContent = ${$oStorage->get($strFileName)};
-
- if ($strContent eq $strOldContent)
- {
- $bChanged = false;
- }
- }
-
- if ($bChanged)
- {
- $oStorage->put($strFileName, $strContent);
- }
+ # Save the file
+ $oStorage->put("${strPath}/${strFile}.auto.${strExt}", trim($rhSource->{$strFileType}) . "\n");
}
}
}
diff --git a/doc/xml/release.xml b/doc/xml/release.xml
index 393902c60..67c882a08 100644
--- a/doc/xml/release.xml
+++ b/doc/xml/release.xml
@@ -110,7 +110,7 @@
<release-development-list>
<release-item>
- <p>Improve bin and libc build performance.</p>
+ <p>Buld performance improvements. Improve bin and libc build performance. Improve code generation performance.</p>
</release-item>
<release-item>
diff --git a/libc/build/lib/pgBackRestLibC/Build.pm b/libc/build/lib/pgBackRestLibC/Build.pm
index c96eb841e..5ef0b3647 100644
--- a/libc/build/lib/pgBackRestLibC/Build.pm
+++ b/libc/build/lib/pgBackRestLibC/Build.pm
@@ -297,24 +297,8 @@ sub buildXsAll
"\n" .
"1;\n";
- # Only save the file if it has changed
- my $strLibFile = 'lib/' . BACKREST_NAME . '/' . LIB_AUTO_NAME . '.pm';
- my $bChanged = true;
-
- if ($oStorage->exists($strLibFile))
- {
- my $strContentOld = ${$oStorage->get($strLibFile)};
-
- if ($strContent eq $strContentOld)
- {
- $bChanged = false;
- }
- }
-
- if ($bChanged)
- {
- $oStorage->put($strLibFile, $strContent);
- }
+ # Save the file
+ $oStorage->put('lib/' . BACKREST_NAME . '/' . LIB_AUTO_NAME . '.pm', $strContent);
# Build error file
#-------------------------------------------------------------------------------------------------------------------------------
diff --git a/test/lib/pgBackRestTest/Common/BuildTest.pm b/test/lib/pgBackRestTest/Common/BuildTest.pm
new file mode 100644
index 000000000..1dd4ecdcc
--- /dev/null
+++ b/test/lib/pgBackRestTest/Common/BuildTest.pm
@@ -0,0 +1,57 @@
+####################################################################################################################################
+# Build Binaries and Auto-Generate Code
+####################################################################################################################################
+package pgBackRestTest::Common::BuildTest;
+
+####################################################################################################################################
+# Perl includes
+####################################################################################################################################
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+use English '-no_match_vars';
+
+# use Cwd qw(abs_path);
+use Exporter qw(import);
+ our @EXPORT = qw();
+
+use pgBackRest::Common::Log;
+
+####################################################################################################################################
+# Find last modification time in a list of directories, with optional filters
+####################################################################################################################################
+sub buildLastModTime
+{
+ my $oStorage = shift;
+ my $strBasePath = shift;
+ my $rstrySubPath = shift;
+ my $strPattern = shift;
+
+ my $lTimestampLast = 0;
+
+ foreach my $strSubPath (defined($rstrySubPath) ? @{$rstrySubPath} : (''))
+ {
+ my $hManifest = $oStorage->manifest($strBasePath . ($strSubPath eq '' ? '' : "/${strSubPath}"));
+
+ foreach my $strFile (sort(keys(%{$hManifest})))
+ {
+ next if (defined($strPattern) && $strFile !~ /$strPattern/);
+
+ if ($hManifest->{$strFile}{type} eq 'f' && $hManifest->{$strFile}{modification_time} > $lTimestampLast)
+ {
+ $lTimestampLast = $hManifest->{$strFile}{modification_time};
+ }
+ }
+ }
+
+ if ($lTimestampLast == 0)
+ {
+ confess &log(ERROR, "no files found");
+ }
+
+ return $lTimestampLast;
+}
+
+push @EXPORT, qw(buildLastModTime);
+
+1;
diff --git a/test/test.pl b/test/test.pl
index a5e7ed20a..05b52c76b 100755
--- a/test/test.pl
+++ b/test/test.pl
@@ -47,6 +47,7 @@ use pgBackRestBuild::Error::Data;
use BackRestDoc::Custom::DocCustomRelease;
+use pgBackRestTest::Common::BuildTest;
use pgBackRestTest::Common::ContainerTest;
use pgBackRestTest::Common::CiTest;
use pgBackRestTest::Common::DefineTest;
@@ -238,6 +239,7 @@ eval
}
logLevelSet(uc($strLogLevel), uc($strLogLevel), OFF);
+ &log(INFO, "test begin - log level ${strLogLevel}");
if (@stryModuleTest != 0 && @stryModule != 1)
{
@@ -303,47 +305,58 @@ eval
{
# Auto-generate C files
#---------------------------------------------------------------------------------------------------------------------------
- &log(INFO, "autogenerate C code");
+ &log(INFO, "check code autogenerate");
+
errorDefineLoad(${$oStorageBackRest->get("build/error.yaml")});
- my $rhBuild =
+ if (buildLastModTime($oStorageBackRest, "${strBackRestBase}", ['build']) >
+ buildLastModTime($oStorageBackRest, "${strBackRestBase}", ['src'], '\.auto\.c$'))
{
- 'config' =>
- {
- &BLD_DATA => buildConfig(),
- &BLD_PATH => 'config',
- },
+ &log(INFO, " autogenerate C code");
- 'configDefine' =>
+ my $rhBuild =
{
- &BLD_DATA => buildConfigDefine(),
- &BLD_PATH => 'config',
- },
+ 'config' =>
+ {
+ &BLD_DATA => buildConfig(),
+ &BLD_PATH => 'config',
+ },
- 'configParse' =>
- {
- &BLD_DATA => buildConfigParse(),
- &BLD_PATH => 'config',
- },
+ 'configDefine' =>
+ {
+ &BLD_DATA => buildConfigDefine(),
+ &BLD_PATH => 'config',
+ },
- 'error' =>
- {
- &BLD_DATA => buildError(),
- &BLD_PATH => 'common',
- },
- };
+ 'configParse' =>
+ {
+ &BLD_DATA => buildConfigParse(),
+ &BLD_PATH => 'config',
+ },
+
+ 'error' =>
+ {
+ &BLD_DATA => buildError(),
+ &BLD_PATH => 'common',
+ },
+ };
- buildAll("${strBackRestBase}/src", $rhBuild);
+ buildAll("${strBackRestBase}/src", $rhBuild);
+ }
# Auto-generate XS files
#
# Use statements are put here so this will be easy to get rid of someday.
#---------------------------------------------------------------------------------------------------------------------------
- &log(INFO, "autogenerate Perl code");
- use lib dirname(dirname($0)) . '/libc/build/lib';
- use pgBackRestLibC::Build; ## no critic (Modules::ProhibitConditionalUseStatements)
+ if (buildLastModTime($oStorageBackRest, "${strBackRestBase}", ['libc/build']) >
+ buildLastModTime($oStorageBackRest, "${strBackRestBase}", ['libc', 'lib'], '(\.auto\.pm|Auto\.pm)$'))
+ {
+ &log(INFO, " autogenerate Perl code");
+ use lib dirname(dirname($0)) . '/libc/build/lib';
+ use pgBackRestLibC::Build; ## no critic (Modules::ProhibitConditionalUseStatements)
- buildXsAll("${strBackRestBase}/libc");
+ buildXsAll("${strBackRestBase}/libc");
+ }
if ($bGenOnly)
{
@@ -351,6 +364,7 @@ eval
}
# Sync time to prevent build failures when running on VirtualBox.
+ &log(INFO, "sync vbox time");
my $strVBoxService = '/usr/sbin/VBoxService';
if ($oStorageTest->exists($strVBoxService))
@@ -365,6 +379,8 @@ eval
(new pgBackRestTest::Common::CiTest($oStorageBackRest))->process();
}
+ &log(INFO, "check version info");
+
# Load the doc module dynamically since it is not supported on all systems
require BackRestDoc::Common::Doc;
BackRestDoc::Common::Doc->import();
@@ -421,6 +437,7 @@ eval
if (!$bDryRun || $bVmOut)
{
+ &log(INFO, "cleanup old data and containers");
containerRemove('test-([0-9]+|build)');
for (my $iVmIdx = 0; $iVmIdx < 8; $iVmIdx++)
@@ -464,6 +481,8 @@ eval
#---------------------------------------------------------------------------------------------------------------------------
if (!$bDryRun && $bBuildRequired)
{
+ &log(INFO, "check bin builds");
+
my $oVm = vmGet();
my $strVagrantPath = "${strBackRestBase}/test/.vagrant";
@@ -475,27 +494,14 @@ eval
my @stryBinSrcPath = ('src');
# Find the lastest modified time for dirs that affect the bin build
- my $lTimestampLast = $oStorageBackRest->exists($strBinSmart) ? $oStorageBackRest->info($strBinSmart)->mtime : 0;
-
- foreach my $strBinSrcPath (@stryBinSrcPath)
- {
- my $hManifest = $oStorageBackRest->manifest($strBinSrcPath);
-
- foreach my $strFile (sort(keys(%{$hManifest})))
- {
- if ($hManifest->{$strFile}{type} eq 'f' && $hManifest->{$strFile}{modification_time} > $lTimestampLast)
- {
- $lTimestampLast = $hManifest->{$strFile}{modification_time};
- }
- }
- }
+ my $lTimestampLast = buildLastModTime($oStorageBackRest, $strBackRestBase, \@stryBinSrcPath);
# Rebuild if the modification time of the smart file does equal the last changes in source paths
if ($bSmart)
{
if (!$oStorageBackRest->exists($strBinSmart) || $oStorageBackRest->info($strBinSmart)->mtime < $lTimestampLast)
{
- &log(INFO, 'bin dependencies have changed, rebuilding...');
+ &log(INFO, ' bin dependencies have changed, rebuilding...');
$bRebuild = true;
}
@@ -515,7 +521,7 @@ eval
if ($bRebuild)
{
- &log(INFO, "build bin for ${strBuildVM} (${strBuildPath})");
+ &log(INFO, " build bin for ${strBuildVM} (${strBuildPath})");
executeTest(
"docker run -itd -h test-build --name=test-build" .
@@ -554,25 +560,14 @@ eval
my @stryLibCSrcPath = ('libc', 'src');
# Find the lastest modified time for dirs that affect the libc build
- foreach my $strLibCSrcPath (@stryLibCSrcPath)
- {
- my $hManifest = $oStorageBackRest->manifest($strLibCSrcPath);
-
- foreach my $strFile (sort(keys(%{$hManifest})))
- {
- if ($hManifest->{$strFile}{type} eq 'f' && $hManifest->{$strFile}{modification_time} > $lTimestampLast)
- {
- $lTimestampLast = $hManifest->{$strFile}{modification_time};
- }
- }
- }
+ $lTimestampLast = buildLastModTime($oStorageBackRest, $strBackRestBase, \@stryLibCSrcPath);
# Rebuild if the modification time of the smart file does equal the last changes in source paths
if ($bSmart)
{
if (!$oStorageBackRest->exists($strLibCSmart) || $oStorageBackRest->info($strLibCSmart)->mtime < $lTimestampLast)
{
- &log(INFO, 'libc dependencies have changed, rebuilding...');
+ &log(INFO, ' libc dependencies have changed, rebuilding...');
$bRebuild = true;
}
@@ -600,7 +595,7 @@ eval
if ($bRebuild)
{
- &log(INFO, "build C library for ${strBuildVM} (${strBuildPath})");
+ &log(INFO, " build C library for ${strBuildVM} (${strBuildPath})");
# It's very expensive to rebuild the Makefile so make sure it has actually changed
my $bMakeRebuild =
[parent: b987f05f3529]