[Michael Banck <michael.banck@credativ.de>]

Hi,

my colleague Bernd Helmle recently added progress reporting to our
pg_checksums application[1]. I have now forward ported this to
pg_verify_checksums for the September commitfest, please see the
attached patch.

Here's the description:

This optionally prints the progress of pg_verify_checksums via read
kilobytes to the terminal with the new command line argument -P.

If -P was forgotten and pg_verify_checksums operates on a large cluster,
the caller can send SIGUSR1 to pg_verify_checksums to turn progress
status reporting on during runtime.


Michael

[1] https://github.com/credativ/pg_checksums/commit/2b691
-- 
Michael Banck
Projektleiter / Senior Berater
Tel.: +49 2166 9901-171
Fax:  +49 2166 9901-100
Email: michael.banck@credativ.de

credativ GmbH, HRB Mönchengladbach 12080
USt-ID-Nummer: DE204566209
Trompeterallee 108, 41189 Mönchengladbach
Geschäftsführung: Dr. Michael Meskes, Jörg Folz, Sascha Heuer

Unser Umgang mit personenbezogenen Daten unterliegt
folgenden Bestimmungen: https://www.credativ.de/datenschutz

Attachments:

  [text/x-patch] pg_verify_checksums_progress.patch (7.4K, ../1535719851.1286.17.camel@credativ.de/2-pg_verify_checksums_progress.patch)
  download | inline diff:
commit ad69b43a24a6eef8a7deef4c9810ee312f3496fb
Author: Michael Banck <michael.banck@credativ.de>
Date:   Fri Aug 31 13:04:59 2018 +0200

    Add progress reporting to pg_verify_checksums.
    
    This optionally prints the progress of pg_verify_checksums via read kilobytes
    to the terminal with the new command line argument -P.
    
    If -P was forgotten and pg_verify_checksums operates on a large cluster, the
    caller can send SIGUSR1 to pg_verify_checksums to turn progress status
    reporting on during runtime.
    
    Author: Bernd Helmle

diff --git a/src/bin/pg_verify_checksums/pg_verify_checksums.c b/src/bin/pg_verify_checksums/pg_verify_checksums.c
index a941236563..27b1103902 100644
--- a/src/bin/pg_verify_checksums/pg_verify_checksums.c
+++ b/src/bin/pg_verify_checksums/pg_verify_checksums.c
@@ -21,6 +21,8 @@
 #include <sys/stat.h>
 #include <dirent.h>
 #include <unistd.h>
+#include <signal.h>
+#include <time.h>
 
 #include "pg_getopt.h"
 
@@ -32,9 +34,18 @@ static ControlFileData *ControlFile;
 
 static char *only_relfilenode = NULL;
 static bool verbose = false;
+static bool show_progress = false;
 
 static const char *progname;
 
+/*
+ * Progress status information.
+ */
+int64		total_size = 0;
+int64		current_size = 0;
+pg_time_t	last_progress_update;
+pg_time_t	scan_started;
+
 static void
 usage()
 {
@@ -45,6 +56,7 @@ usage()
 	printf(_(" [-D, --pgdata=]DATADIR  data directory\n"));
 	printf(_("  -v, --verbose          output verbose messages\n"));
 	printf(_("  -r RELFILENODE         check only relation with specified relfilenode\n"));
+	printf(_("  -P                     show progress information\n"));
 	printf(_("  -V, --version          output version information, then exit\n"));
 	printf(_("  -?, --help             show this help, then exit\n"));
 	printf(_("\nIf no data directory (DATADIR) is specified, "
@@ -60,6 +72,69 @@ static const char *skip[] = {
 	NULL,
 };
 
+static void
+enable_progress_report(int signo,
+					   siginfo_t * siginfo,
+					   void *context)
+{
+
+	/* we handle SIGUSR1 only */
+	if (signo == SIGUSR1)
+	{
+		show_progress = true;
+	}
+
+}
+
+/*
+ * Report current progress status. Parts borrowed from
+ * PostgreSQLs' src/bin/pg_basebackup.c
+ */
+static void
+report_progress(bool force)
+{
+	pg_time_t	now = time(NULL);
+	int			total_percent = 0;
+
+	char		totalstr[32];
+	char		currentstr[32];
+	char		currspeedstr[32];
+
+	/* Make sure we just report at least once a second */
+	if ((now == last_progress_update) && !force)
+		return;
+
+	/* Save current time */
+	last_progress_update = now;
+
+	/*
+	 * Calculate current percent done, based on KiB...
+	 */
+	total_percent = total_size ? (int64) ((current_size / 1024) * 100 / (total_size / 1024)) : 0;
+
+	/* Don't display larger than 100% */
+	if (total_percent > 100)
+		total_percent = 100;
+
+	/* The same for total size */
+	if (current_size > total_size)
+		total_size = current_size / 1024;
+
+	snprintf(totalstr, sizeof(totalstr), INT64_FORMAT,
+			 total_size / 1024);
+	snprintf(currentstr, sizeof(currentstr), INT64_FORMAT,
+			 current_size / 1024);
+	snprintf(currspeedstr, sizeof(currspeedstr), INT64_FORMAT,
+			 (current_size / 1024) / (((time(NULL) - scan_started) == 0) ? 1 : (time(NULL) - scan_started)));
+	fprintf(stderr, "%s/%s kB (%d%%, %s kB/s)",
+			currentstr, totalstr, total_percent, currspeedstr);
+
+	if (isatty(fileno(stderr)))
+		fprintf(stderr, "\r");
+	else
+		fprintf(stderr, "\n");
+}
+
 static bool
 skipfile(char *fn)
 {
@@ -76,7 +151,7 @@ skipfile(char *fn)
 }
 
 static void
-scan_file(char *fn, int segmentno)
+scan_file(char *fn, int segmentno, bool sizeonly)
 {
 	char		buf[BLCKSZ];
 	PageHeader	header = (PageHeader) buf;
@@ -113,6 +188,7 @@ scan_file(char *fn, int segmentno)
 			continue;
 
 		csum = pg_checksum_page(buf, blockno + segmentno * RELSEG_SIZE);
+		current_size += r;
 		if (csum != header->pd_checksum)
 		{
 			if (ControlFile->data_checksum_version == PG_DATA_CHECKSUM_VERSION)
@@ -120,6 +196,9 @@ scan_file(char *fn, int segmentno)
 						progname, fn, blockno, csum, header->pd_checksum);
 			badblocks++;
 		}
+
+		if (show_progress)
+			report_progress(false);
 	}
 
 	if (verbose)
@@ -129,9 +208,10 @@ scan_file(char *fn, int segmentno)
 	close(f);
 }
 
-static void
-scan_directory(char *basedir, char *subdir)
+static int64
+scan_directory(char *basedir, char *subdir, bool sizeonly)
 {
+	int64		dirsize = 0;
 	char		path[MAXPGPATH];
 	DIR		   *dir;
 	struct dirent *de;
@@ -192,16 +272,22 @@ scan_directory(char *basedir, char *subdir)
 				/* Relfilenode not to be included */
 				continue;
 
-			scan_file(fn, segmentno);
+			dirsize += st.st_size;
+
+			if (!sizeonly)
+			{
+				scan_file(fn, segmentno, sizeonly);
+			}
 		}
 #ifndef WIN32
 		else if (S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))
 #else
 		else if (S_ISDIR(st.st_mode) || pgwin32_is_junction(fn))
 #endif
-			scan_directory(path, de->d_name);
+			dirsize += scan_directory(path, de->d_name, sizeonly);
 	}
 	closedir(dir);
+	return dirsize;
 }
 
 int
@@ -218,6 +304,8 @@ main(int argc, char *argv[])
 	int			option_index;
 	bool		crc_ok;
 
+	struct sigaction act; /* to turn progress status info on */
+
 	set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_verify_checksums"));
 
 	progname = get_progname(argv[0]);
@@ -236,7 +324,7 @@ main(int argc, char *argv[])
 		}
 	}
 
-	while ((c = getopt_long(argc, argv, "D:r:v", long_options, &option_index)) != -1)
+	while ((c = getopt_long(argc, argv, "D:r:vP", long_options, &option_index)) != -1)
 	{
 		switch (c)
 		{
@@ -254,6 +342,9 @@ main(int argc, char *argv[])
 				}
 				only_relfilenode = pstrdup(optarg);
 				break;
+			case 'P':
+				show_progress = true;
+				break;
 			default:
 				fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
 				exit(1);
@@ -307,10 +398,56 @@ main(int argc, char *argv[])
 		exit(1);
 	}
 
+
+	/*
+	 * Assign SIGUSR1 signal handler to turn progress status information
+	 * on in case someone has forgotten -P (and doesn't want to
+	 * restart)...
+	 */
+	memset(&act, '\0', sizeof(act));
+	act.sa_sigaction = &enable_progress_report;
+	act.sa_flags = SA_SIGINFO;
+
+	/*
+	 * Enable signal handler, but don't treat it as severe if we don't
+	 * succeed here. Just give a message on STDERR.
+	 */
+	if (sigaction(SIGUSR1, &act, NULL) < 0)
+	{
+		fprintf(stderr, _("%s: could not set signal handler to turn on progress bar\n"),
+				progname;
+	}
+
+	/*
+	 * Iff progress status information is requested, we need to scan the
+	 * directory tree(s) twice, once to get the idea how much data we need
+	 * to scan and finally to do the real legwork.
+	 */
+	total_size = scan_directory(DataDir, "global", true);
+	total_size += scan_directory(DataDir, "base", true);
+	total_size += scan_directory(DataDir, "pg_tblspc", true);
+
+	/*
+	 * Remember start time. Required to calculate the current speed in
+	 * report_progress()
+	 */
+	scan_started = time(NULL);
+
 	/* Scan all files */
-	scan_directory(DataDir, "global");
-	scan_directory(DataDir, "base");
-	scan_directory(DataDir, "pg_tblspc");
+	scan_directory(DataDir, "global", false);
+	scan_directory(DataDir, "base", false);
+	scan_directory(DataDir, "pg_tblspc", false);
+
+	/*
+	 * Done. Move to next line in case progress information was shown.
+	 * Otherwise we clutter the summary output.
+	 */
+	if (show_progress)
+	{
+		report_progress(true);
+		if (isatty(fileno(stderr)))
+			fprintf(stderr, "\n");
+	}
 
 	printf(_("Checksum scan completed\n"));
 	printf(_("Data checksum version: %d\n"), ControlFile->data_checksum_version);