pgbackrest.git / summary / log / commit / refs

commit    07f38f584a6014e10888fe0f8016febab79ce67b
Author:   David Steele <david@pgmasters.net>
Date:     Tue Mar 20 03:33:28 2018 +0000

    Use lcov for C unit test coverage reporting.
    
    Switch from Devel::Cover because it would not report on branch coverage for reports converted from gcov.
    
    Branch coverage is not complete, so for the time being errors will only be generated when statement coverage is not complete. Coverage of unit tests is not displayed in the report unless they are incomplete for either statement or branch coverage.


.travis.yml | 2 +- doc/xml/release.xml | 10 +++ src/common/regExp.c | 4 +- test/Vagrantfile | 2 +- test/lib/pgBackRestTest/Common/CiTest.pm | 2 +- test/lib/pgBackRestTest/Common/ContainerTest.pm | 2 +- test/lib/pgBackRestTest/Common/JobTest.pm | 112 ++++++++++++++---------- test/src/lcov.conf | 3 + test/src/module/common/memContextTest.c | 15 ++-- test/src/module/common/typeBufferTest.c | 6 +- test/test.pl | 110 +++++++++++++++-------- 11 files changed, 166 insertions(+), 102 deletions(-) diff --git a/.travis.yml b/.travis.yml index c7506f6c1..2117da972 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ env: - PGB_CI="doc" before_install: - - sudo apt-get -qq update && sudo apt-get install libxml-checker-perl libdbd-pg-perl libperl-critic-perl libtemplate-perl libpod-coverage-perl libtest-differences-perl libhtml-parser-perl lintian debhelper txt2man devscripts libjson-perl libio-socket-ssl-perl libxml-libxml-perl libyaml-perl python-pip + - sudo apt-get -qq update && sudo apt-get install libxml-checker-perl libdbd-pg-perl libperl-critic-perl libtemplate-perl libpod-coverage-perl libtest-differences-perl libhtml-parser-perl lintian debhelper txt2man devscripts libjson-perl libio-socket-ssl-perl libxml-libxml-perl libyaml-perl python-pip lcov - | # Install & Configure AWS CLI pip install --upgrade --user awscli diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 215e6c2f2..1d8de75ca 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -12,6 +12,16 @@ </intro> <release-list> + <release date="XXXX-XX-XX" version="2.02dev" title="UNDER DEVELOPMENT"> + <release-test-list> + <release-feature-list> + <release-item> + <p>Use <proper>lcov</proper> for C unit test coverage reporting. Switch from <proper>Devel::Cover</proper> because it would not report on branch coverage for reports converted from <proper>gcov</proper>. Branch coverage is not complete, so for the time being errors will only be generated when statement coverage is not complete. Coverage of unit tests is not displayed in the report unless they are incomplete for either statement or branch coverage.</p> + </release-item> + </release-feature-list> + </release-test-list> + </release> + <release date="2018-03-19" version="2.01" title="Minor Bug Fixes and Improvements"> <release-core-list> <release-bug-list> diff --git a/src/common/regExp.c b/src/common/regExp.c index c39d577e2..9018c4fec 100644 --- a/src/common/regExp.c +++ b/src/common/regExp.c @@ -68,8 +68,8 @@ regExpMatch(RegExp *this, const String *string) int result = regexec(&this->regExp, strPtr(string), 0, NULL, 0); // Check for an error - if (result != 0 && result != REG_NOMATCH) - regExpError(result); // {uncoverable - no error condition known} + if (result != 0 && result != REG_NOMATCH) // {uncoverable - no error condition known} + regExpError(result); // {+uncoverable} return result == 0; } diff --git a/test/Vagrantfile b/test/Vagrantfile index 3a51a34f0..397c0211f 100644 --- a/test/Vagrantfile +++ b/test/Vagrantfile @@ -52,7 +52,7 @@ Vagrant.configure(2) do |config| #--------------------------------------------------------------------------------------------------------------------------- echo 'Install Build Tools' && date - apt-get install -y devscripts build-essential lintian git txt2man debhelper libssl-dev + apt-get install -y devscripts build-essential lintian git txt2man debhelper libssl-dev lcov #--------------------------------------------------------------------------------------------------------------------------- echo 'Install AWS CLI' && date diff --git a/test/lib/pgBackRestTest/Common/CiTest.pm b/test/lib/pgBackRestTest/Common/CiTest.pm index 781d7ff6b..524ccccf8 100644 --- a/test/lib/pgBackRestTest/Common/CiTest.pm +++ b/test/lib/pgBackRestTest/Common/CiTest.pm @@ -101,7 +101,7 @@ sub process "before_install:\n" . " - sudo apt-get -qq update && sudo apt-get install libxml-checker-perl libdbd-pg-perl libperl-critic-perl" . " libtemplate-perl libpod-coverage-perl libtest-differences-perl libhtml-parser-perl lintian debhelper txt2man" . - " devscripts libjson-perl libio-socket-ssl-perl libxml-libxml-perl libyaml-perl python-pip\n" . + " devscripts libjson-perl libio-socket-ssl-perl libxml-libxml-perl libyaml-perl python-pip lcov\n" . " - |\n" . " # Install & Configure AWS CLI\n" . " pip install --upgrade --user awscli\n" . diff --git a/test/lib/pgBackRestTest/Common/ContainerTest.pm b/test/lib/pgBackRestTest/Common/ContainerTest.pm index 9e03fed80..3ac8cf3d3 100755 --- a/test/lib/pgBackRestTest/Common/ContainerTest.pm +++ b/test/lib/pgBackRestTest/Common/ContainerTest.pm @@ -361,7 +361,7 @@ sub containerBuild } elsif ($strOS eq VM_U16) { - $strScript .= ' clang-5.0'; + $strScript .= ' clang-5.0 lcov'; } } diff --git a/test/lib/pgBackRestTest/Common/JobTest.pm b/test/lib/pgBackRestTest/Common/JobTest.pm index b44d8d3d7..8f8299560 100644 --- a/test/lib/pgBackRestTest/Common/JobTest.pm +++ b/test/lib/pgBackRestTest/Common/JobTest.pm @@ -445,73 +445,91 @@ sub end } # Generate coverage reports for the modules + my $strLCovExe = "lcov --config-file=$self->{strBackRestBase}/test/src/lcov.conf"; + my $strLCovOut = $self->{strGCovPath} . '/test.lcov'; + executeTest( - 'docker exec -i -u ' . TEST_USER . " ${strImage} bash -l -c '" . - "cd $self->{strGCovPath} && " . - "gcov test.c'", {bSuppressStdErr => true}); + 'docker exec -i -u ' . TEST_USER . " ${strImage} " . + "${strLCovExe} --capture --directory=$self->{strGCovPath} --o=${strLCovOut}"); - # Mark uncoverable statements as successful + # Generate coverage report for each module foreach my $strModule (sort(keys(%{$hTestCoverage}))) { - # File that contains coverage info for the module - my $strCoverageFile = "$self->{strGCovPath}/" . testRunName(basename($strModule), false) . ".c.gcov"; + my $strModuleName = testRunName($strModule, false); + my $strModuleOutName = $strModuleName; + my $bTest = false; + + if ($strModuleOutName =~ /^module/mg) + { + $strModuleOutName =~ s/^module/test/mg; + $bTest = true; + } + + # Generate lcov reports + my $strModulePath = $self->{strBackRestBase} . "/test/.vagrant/code/${strModuleOutName}"; + my $strLCovFile = "${strModulePath}.lcov"; + my $strLCovTotal = $self->{strBackRestBase} . "/test/.vagrant/code/all.lcov"; + + executeTest( + 'docker exec -i -u ' . TEST_USER . " ${strImage} " . + "${strLCovExe} --extract=${strLCovOut} */${strModuleName}.c --o=${strLCovFile}"); - # If marked as no code then error if there is code - if ($hTestCoverage->{$strModule} eq TESTDEF_COVERAGE_NOCODE) + # Update source file + my $strCoverage = ${$self->{oStorageTest}->get($strLCovFile)}; + + if (defined($strCoverage)) { - if ($self->{oStorageTest}->exists($strCoverageFile)) + if ($hTestCoverage->{$strModule} eq TESTDEF_COVERAGE_NOCODE) { confess &log(ERROR, "module '${strModule}' is marked 'no code' but has code"); } - next; - } - - # Load the coverage file - my $strCoverage = ${$self->{oStorageTest}->get($strCoverageFile)}; + # Get coverage info + my $iTotalLines = (split(':', ($strCoverage =~ m/^LF:.*$/mg)[0]))[1] + 0; + my $iCoveredLines = (split(':', ($strCoverage =~ m/^LH:.*$/mg)[0]))[1] + 0; - # Go line by line and update uncovered statements - my $strUpdatedCoverage; - my $bFirst = true; + my $iTotalBranches = 0; + my $iCoveredBranches = 0; - foreach my $strLine (split("\n", trim($strCoverage))) - { - # Rewrite source line to the original source path - if ($bFirst) + if ($strCoverage =~ /^BRF\:$/mg && $strCoverage =~ /^BRH\:$/mg) { - $strLine = - " -: 0:Source:" . - $self->{strBackRestBase} . ($strModule =~ /Test$/ ? '/test' : '') . "/src/" . - testRunName(${strModule}, false) . ".c"; - $bFirst = false; + $iTotalBranches = (split(':', ($strCoverage =~ m/^BRF:.*$/mg)[0]))[1] + 0; + $iCoveredBranches = (split(':', ($strCoverage =~ m/^BRH:.*$/mg)[0]))[1] + 0; } - # Else if the statement is marked uncoverable - elsif ($strLine =~ /\/\/ \{(uncoverable - [^\}]+|\+uncoverable|uncovered - [^\}]+|\+uncovered)\}\s*$/) + + # Report coverage if this is not a test or if the test does not have complete coverage + if (!$bTest || $iTotalLines != $iCoveredLines || $iTotalBranches != $iCoveredBranches) { - # Error if the statement is marked uncoverable but is in fact covered - if ($strLine !~ /^ \#\#\#\#\#/) + # Fix source file name + $strCoverage =~ s/^SF\:.*$/SF:$strModulePath\.c/mg; + + # Save coverage file + $self->{oStorageTest}->put($strLCovFile, $strCoverage); + + if ($self->{oStorageTest}->exists(${strLCovTotal})) { - &log(ERROR, "line in ${strModule}.c is marked as uncoverable but is covered:\n${strLine}"); + executeTest( + 'docker exec -i -u ' . TEST_USER . " ${strImage} " . + "${strLCovExe} --add-tracefile=${strLCovFile} --add-tracefile=${strLCovTotal} --o=${strLCovTotal}"); + } + else + { + $self->{oStorageTest}->copy($strLCovFile, $strLCovTotal) } - - # Mark the statement as covered with 0 executions - $strLine =~ s/^ \#\#\#\#\#/^ 0/; } - - # Add to updated file - $strUpdatedCoverage .= "${strLine}\n"; + else + { + $self->{oStorageTest}->remove($strLCovFile); + } + } + else + { + if ($hTestCoverage->{$strModule} ne TESTDEF_COVERAGE_NOCODE) + { + confess &log(ERROR, "module '${strModule}' is marked 'code' but has no code"); + } } - - # Store the updated file - $self->{oStorageTest}->put($strCoverageFile, $strUpdatedCoverage); } - - executeTest( - 'docker exec -i -u ' . TEST_USER . " ${strImage} bash -l -c '" . - "cd $self->{strGCovPath} && " . - # Only generate coverage files for VMs that support coverage - (vmCoverage($self->{oTest}->{&TEST_VM}) ? - "gcov2perl -db ../cover_db " . join(' ', @stryCoveredModule) : '') . "'"); } # Record elapsed time diff --git a/test/src/lcov.conf b/test/src/lcov.conf new file mode 100644 index 000000000..ed5355a7a --- /dev/null +++ b/test/src/lcov.conf @@ -0,0 +1,3 @@ +lcov_branch_coverage=1 +lcov_excl_br_line=[A-Z_]+\(|assert\(|testBegin\( +lcov_excl_line=\{\+*uncovered|\{\+*uncoverable diff --git a/test/src/module/common/memContextTest.c b/test/src/module/common/memContextTest.c index 62a959003..9a741ecd5 100644 --- a/test/src/module/common/memContextTest.c +++ b/test/src/module/common/memContextTest.c @@ -58,8 +58,7 @@ testRun() int expectedTotal = 0; for (unsigned int charIdx = 0; charIdx < sizeof(size_t); charIdx++) - if (buffer2[charIdx] == 0) - expectedTotal++; + expectedTotal += buffer2[charIdx] == 0; TEST_RESULT_INT(expectedTotal, sizeof(size_t), "all bytes are 0"); @@ -71,16 +70,14 @@ testRun() expectedTotal = 0; for (unsigned int charIdx = 0; charIdx < sizeof(size_t); charIdx++) - if (buffer2[charIdx] == 0xC7) - expectedTotal++; + expectedTotal += buffer2[charIdx] == 0xC7; TEST_RESULT_INT(expectedTotal, sizeof(size_t), "all old bytes are filled"); expectedTotal = 0; for (unsigned int charIdx = 0; charIdx < sizeof(size_t); charIdx++) - if ((buffer2 + sizeof(size_t))[charIdx] == 0) - expectedTotal++; + expectedTotal += (buffer2 + sizeof(size_t))[charIdx] == 0; TEST_RESULT_INT(expectedTotal, sizeof(size_t), "all new bytes are 0"); memFreeInternal(buffer2); @@ -178,8 +175,7 @@ testRun() int expectedTotal = 0; for (unsigned int charIdx = 0; charIdx < sizeof(size_t); charIdx++) - if (buffer[charIdx] == 0) - expectedTotal++; + expectedTotal += buffer[charIdx] == 0; TEST_RESULT_INT(expectedTotal, sizeof(size_t), "all bytes are 0"); @@ -200,8 +196,7 @@ testRun() int expectedTotal = 0; for (unsigned int charIdx = 0; charIdx < sizeof(size_t); charIdx++) - if (buffer[charIdx] == 0xFE) - expectedTotal++; + expectedTotal += buffer[charIdx] == 0xFE; TEST_RESULT_INT(expectedTotal, sizeof(size_t), "all bytes are 0xFE in original portion"); diff --git a/test/src/module/common/typeBufferTest.c b/test/src/module/common/typeBufferTest.c index df8d8c552..7a441a039 100644 --- a/test/src/module/common/typeBufferTest.c +++ b/test/src/module/common/typeBufferTest.c @@ -48,8 +48,7 @@ testRun() unsigned int sameTotal = 0; for (unsigned int bufferIdx = 0; bufferIdx < 256; bufferIdx++) - if (bufferPtr[bufferIdx] == bufferIdx) - sameTotal++; + sameTotal += bufferPtr[bufferIdx] == bufferIdx; TEST_RESULT_INT(sameTotal, 256, "original bytes match"); @@ -61,8 +60,7 @@ testRun() sameTotal = 0; for (unsigned int bufferIdx = 0; bufferIdx < bufSize(buffer); bufferIdx++) - if (bufferPtr[bufferIdx] == bufferIdx) - sameTotal++; + sameTotal += bufferPtr[bufferIdx] == bufferIdx; TEST_RESULT_INT(sameTotal, 128, "original bytes match"); diff --git a/test/test.pl b/test/test.pl index 2c06c17be..ef296b80a 100755 --- a/test/test.pl +++ b/test/test.pl @@ -433,6 +433,7 @@ eval my $iTestRetry = 0; my $oyProcess = []; my $strCoveragePath = "${strTestPath}/cover_db"; + my $strCodePath = "${strBackRestBase}/test/.vagrant/code"; if (!$bDryRun || $bVmOut) { @@ -444,12 +445,24 @@ eval push(@{$oyProcess}, undef); } - executeTest("sudo umount ${strTestPath}", {bSuppressError => true}); executeTest("sudo rm -rf ${strTestPath}/*"); $oStorageTest->pathCreate($strTestPath, {strMode => '0770', bIgnoreExists => true}); executeTest("sudo mount -t tmpfs -o size=2560M test ${strTestPath}"); $oStorageTest->pathCreate($strCoveragePath, {strMode => '0770', bIgnoreMissing => true, bCreateParent => true}); + + # Remove old coverage dirs -- do it this way so the dirs stay open in finder/explorer, etc. + executeTest("rm -rf ${strBackRestBase}/test/coverage/c/*"); + executeTest("rm -rf ${strBackRestBase}/test/coverage/perl/*"); + + # Copy C code for coverage tests + if (vmCoverage($strVm) && !$bDryRun) + { + $oStorageTest->pathCreate("${strCodePath}/test", {strMode => '0770', bIgnoreExists => true, bCreateParent => true}); + + executeTest("rsync -rt --delete --exclude=test ${strBackRestBase}/src/ ${strCodePath}"); + executeTest("rsync -rt --delete ${strBackRestBase}/test/src/module/ ${strCodePath}/test"); + } } # Determine which tests to run @@ -991,17 +1004,16 @@ eval if (vmCoverage($strVm) && !$bDryRun) { &log(INFO, 'writing coverage report'); - executeTest("rm -rf ${strBackRestBase}/test/coverage"); executeTest("cp -rp ${strCoveragePath} ${strCoveragePath}_temp"); executeTest( "cd ${strCoveragePath}_temp && " . - LIB_COVER_EXE . " -report json -outputdir ${strBackRestBase}/test/coverage ${strCoveragePath}_temp", + LIB_COVER_EXE . " -report json -outputdir ${strBackRestBase}/test/coverage/perl ${strCoveragePath}_temp", {bSuppressStdErr => true}); executeTest("sudo rm -rf ${strCoveragePath}_temp"); executeTest("sudo cp -rp ${strCoveragePath} ${strCoveragePath}_temp"); executeTest( "cd ${strCoveragePath}_temp && " . - LIB_COVER_EXE . " -outputdir ${strBackRestBase}/test/coverage ${strCoveragePath}_temp", + LIB_COVER_EXE . " -outputdir ${strBackRestBase}/test/coverage/perl ${strCoveragePath}_temp", {bSuppressStdErr => true}); executeTest("sudo rm -rf ${strCoveragePath}_temp"); @@ -1031,7 +1043,7 @@ eval # Load the results of coverage testing from JSON my $oJSON = JSON::PP->new()->allow_nonref(); - my $hCoverageResult = $oJSON->decode(${$oStorageBackRest->get('test/coverage/cover.json')}); + my $hCoverageResult = $oJSON->decode(${$oStorageBackRest->get('test/coverage/perl/cover.json')}); # Now compare against code modules that should have full coverage my $hCoverageList = testDefCoverageList(); @@ -1074,27 +1086,15 @@ eval foreach my $strCodeModule (sort(keys(%{$hCoverageActual}))) { - my $strCodeModulePath = "${strBackRestBase}/"; - - # If the first char of the module is lower case when this is a c module + # If the first char of the module is lower case then it's a c module if (substr($strCodeModule, 0, 1) eq lc(substr($strCodeModule, 0, 1))) { - # If it ends with Test then it is a test modile - if ($strCodeModule =~ /Test$/) - { - $strCodeModulePath .= "test/src/${strCodeModule}.c"; - } - else - { - $strCodeModulePath .= "src/${strCodeModule}.c"; - } - } - # Else a Perl module - else - { - $strCodeModulePath .= "lib/" . BACKREST_NAME . "/${strCodeModule}.pm" + next; } + # Create code module path -- where the file is located on disk + my $strCodeModulePath = "${strBackRestBase}/lib/" . BACKREST_NAME . "/${strCodeModule}.pm"; + # Get summary results my $hCoverageResultAll = $hCoverageResult->{'summary'}{$strCodeModulePath}{total}; @@ -1111,7 +1111,7 @@ eval # Error if it really does have coverage if ($hCoverageResultAll) { - confess &log(ERROR, "module ${strCodeModule} is marked 'no code' but has code"); + confess &log(ERROR, "perl module ${strCodeModule} is marked 'no code' but has code"); } # Skip to next module @@ -1134,18 +1134,15 @@ eval if ($iUncoveredLines != 0) { - &log(ERROR, "code module ${strCodeModule} is not fully covered"); + &log(ERROR, "perl module ${strCodeModule} is not fully covered"); $iUncoveredCodeModuleTotal++; - if ($strCodeModulePath !~ /\.c$/) - { - &log(ERROR, ('-' x 80)); - executeTest( - "/usr/bin/cover -report text ${strCoveragePath} --select ${strBackRestBase}/lib/" . - BACKREST_NAME . "/${strCodeModule}.pm", - {bShowOutputAsync => true}); - &log(ERROR, ('-' x 80)); - } + &log(ERROR, ('-' x 80)); + executeTest( + "/usr/bin/cover -report text ${strCoveragePath} --select ${strBackRestBase}/lib/" . + BACKREST_NAME . "/${strCodeModule}.pm", + {bShowOutputAsync => true}); + &log(ERROR, ('-' x 80)); } } # Else test how much partial coverage there was @@ -1156,7 +1153,7 @@ eval if ($iCoveragePercent == 100) { # This has been changed to a warning for now so archive/async tests will pass - &log(WARN, "code module ${strCodeModule} has 100% coverage but is not marked fully covered"); + &log(WARN, "perl module ${strCodeModule} has 100% coverage but is not marked fully covered"); # $iUncoveredCodeModuleTotal++; } else @@ -1170,7 +1167,50 @@ eval # If any modules had partial coverage then display them if (defined($strPartialCoverage)) { - &log(INFO, "module (expected) partial coverage: ${strPartialCoverage}"); + &log(INFO, "perl module (expected) partial coverage: ${strPartialCoverage}"); + } + + # Generate C coverage with lcov + #--------------------------------------------------------------------------------------------------------------------------- + my $strLCovFile = "${strBackRestBase}/test/.vagrant/code/all.lcov"; + + if ($oStorageBackRest->exists($strLCovFile)) + { + executeTest( + "genhtml ${strLCovFile} --branch-coverage --show-details --output-directory" . + " ${strBackRestBase}/test/coverage/c"); + + foreach my $strCodeModule (sort(keys(%{$hCoverageActual}))) + { + # If the first char of the module is upper case then it's a Perl module + if (substr($strCodeModule, 0, 1) eq uc(substr($strCodeModule, 0, 1))) + { + next; + } + + my $strCoverageFile = $strCodeModule; + $strCoverageFile =~ s/^module/test/mg; + $strCoverageFile = "${strBackRestBase}/test/.vagrant/code/${strCoverageFile}.lcov"; + + my $strCoverage = $oStorageBackRest->get( + $oStorageBackRest->openRead($strCoverageFile, {bIgnoreMissing => true})); + + if (defined($strCoverage) && defined($$strCoverage)) + { + my $iTotalLines = (split(':', ($$strCoverage =~ m/^LF:.*$/mg)[0]))[1] + 0; + my $iCoveredLines = (split(':', ($$strCoverage =~ m/^LH:.*$/mg)[0]))[1] + 0; + + if ($iCoveredLines != $iTotalLines) + { + &log(ERROR, "c module ${strCodeModule} is not fully covered ($iCoveredLines/$iTotalLines)"); + $iUncoveredCodeModuleTotal++; + } + } + } + } + else + { + executeTest("rm -rf ${strBackRestBase}/test/coverage/c"); } } } [parent: 2a3d6ecde8a8]