From: Andres Freund <andres@anarazel.de>
To: pgsql-hackers@postgresql.org
Subject: Re: [HACKERS] Faster CREATE DATABASE by delaying fsync (was 8.4.1
	ubuntu karmic slow createdb)
Date: Mon, 28 Dec 2009 23:59:43 +0100
User-Agent: KMail/1.13.0 (Linux/2.6.33-rc2-andres-00001-g6e0f169; KDE/4.3.85;
	x86_64; ; )
Cc: pgsql-performance@postgresql.org,
	Michael Clemmons <glassresistor@gmail.com>,
	Hannu Krosing <hannu@2ndquadrant.com>,
	Scott Marlowe <scott.marlowe@gmail.com>
References: <4427a97a0912101235x3ade510blcc00d5ce9883c6b6@mail.gmail.com>
	<200912122138.42404.andres@anarazel.de>
	<200912282354.51892.andres@anarazel.de>
In-Reply-To: <200912282354.51892.andres@anarazel.de>
MIME-Version: 1.0
Content-Type: Multipart/Mixed;
  boundary="Boundary-00=_fhTOLpTKgXC4pzN"
Message-Id: <200912282359.43979.andres@anarazel.de>

--Boundary-00=_fhTOLpTKgXC4pzN
Content-Type: Text/Plain;
  charset="iso-8859-15"
Content-Transfer-Encoding: 7bit

On Monday 28 December 2009 23:54:51 Andres Freund wrote:
> On Saturday 12 December 2009 21:38:41 Andres Freund wrote:
> > On Saturday 12 December 2009 21:36:27 Michael Clemmons wrote:
> > > If ppl think its worth it I'll create a ticket
> >
> > Thanks, no need. I will post a patch tomorrow or so.
> 
> Well. It was a long day...
> 
> Anyway.
> In this patch I delay the fsync done in copy_file and simply do a second
>  pass over the directory in copy_dir and fsync everything in that pass.
> Including the directory - which was not done before and actually might be
> necessary in some cases.
> I added a posix_fadvise(..., FADV_DONTNEED) to make it more likely that the
> copied file reaches storage before the fsync. Without the speed benefits
>  were quite a bit smaller and essentially random (which seems sensible).
> 
> This speeds up CREATE DATABASE from ~9 seconds to something around 0.8s on
>  my laptop.  Still slower than with fsync off (~0.25) but quite a worthy
>  improvement.
> 
> The benefits are obviously bigger if the template database includes
>  anything added.
Obviously the patch would be helpfull.

Andres

--Boundary-00=_fhTOLpTKgXC4pzN
Content-Type: text/x-patch; charset="UTF-8";
	name="0001-Delay-fsyncing-files-during-copying-in-CREATE-DATABA.patch"
Content-Transfer-Encoding: quoted-printable
Content-Disposition: attachment;
	filename="0001-Delay-fsyncing-files-during-copying-in-CREATE-DATABA.patch"

=46rom bd80748883d1328a71607a447677b0bfb1f54ab0 Mon Sep 17 00:00:00 2001
=46rom: Andres Freund <andres@anarazel.de>
Date: Mon, 28 Dec 2009 23:43:57 +0100
Subject: [PATCH] Delay fsyncing files during copying in CREATE DATABASE - t=
his
 dramatically speeds up CREATE DATABASE on non battery backed
 rotational storage.
 Additionally fsync() the directory to ensure all metadata reaches
 storage.

=2D--
 src/port/copydir.c |   58 +++++++++++++++++++++++++++++++++++++++++++++---=
=2D--
 1 files changed, 51 insertions(+), 7 deletions(-)

diff --git a/src/port/copydir.c b/src/port/copydir.c
index a70477e..cde3dc7 100644
*** a/src/port/copydir.c
=2D-- b/src/port/copydir.c
***************
*** 37,42 ****
=2D-- 37,43 ----
 =20
 =20
  static void copy_file(char *fromfile, char *tofile);
+ static void fsync_fname(char *fname);
 =20
 =20
  /*
*************** copydir(char *fromdir, char *todir, bool
*** 64,69 ****
=2D-- 65,73 ----
  				(errcode_for_file_access(),
  				 errmsg("could not open directory \"%s\": %m", fromdir)));
 =20
+ 	/*
+ 	 * Copy all the files
+ 	 */
  	while ((xlde =3D ReadDir(xldir, fromdir)) !=3D NULL)
  	{
  		struct stat fst;
*************** copydir(char *fromdir, char *todir, bool
*** 89,96 ****
  		else if (S_ISREG(fst.st_mode))
  			copy_file(fromfile, tofile);
  	}
=2D=20
  	FreeDir(xldir);
  }
 =20
  /*
=2D-- 93,120 ----
  		else if (S_ISREG(fst.st_mode))
  			copy_file(fromfile, tofile);
  	}
  	FreeDir(xldir);
+=20
+ 	/*
+ 	 * Be paranoid here and fsync all files to ensure we catch problems.
+ 	 */
+ 	xldir =3D AllocateDir(fromdir);
+ 	if (xldir =3D=3D NULL)
+ 		ereport(ERROR,
+ 				(errcode_for_file_access(),
+ 				 errmsg("could not open directory \"%s\": %m", fromdir)));
+=20
+ 	while ((xlde =3D ReadDir(xldir, fromdir)) !=3D NULL)
+ 	{
+ 		struct stat fst;
+=20
+ 		if (strcmp(xlde->d_name, ".") =3D=3D 0 ||
+ 			strcmp(xlde->d_name, "..") =3D=3D 0)
+ 			continue;
+=20
+ 		snprintf(tofile, MAXPGPATH, "%s/%s", todir, xlde->d_name);
+ 		fsync_fname(tofile);
+ 	}
  }
 =20
  /*
*************** copy_file(char *fromfile, char *tofile)
*** 150,162 ****
  	}
 =20
  	/*
! 	 * Be paranoid here to ensure we catch problems.
  	 */
! 	if (pg_fsync(dstfd) !=3D 0)
! 		ereport(ERROR,
! 				(errcode_for_file_access(),
! 				 errmsg("could not fsync file \"%s\": %m", tofile)));
!=20
  	if (close(dstfd))
  		ereport(ERROR,
  				(errcode_for_file_access(),
=2D-- 174,185 ----
  	}
 =20
  	/*
! 	 * We tell the kernel here to write the data back in order to make
! 	 * the later fsync cheaper.
  	 */
! #if defined(USE_POSIX_FADVISE) && defined(POSIX_FADV_DONTNEED)
! 	posix_fadvise(dstfd, 0, 0, POSIX_FADV_DONTNEED);
! #endif
  	if (close(dstfd))
  		ereport(ERROR,
  				(errcode_for_file_access(),
*************** copy_file(char *fromfile, char *tofile)
*** 166,168 ****
=2D-- 189,212 ----
 =20
  	pfree(buffer);
  }
+=20
+ /*
+  * fsync a file
+  */
+ static void
+ fsync_fname(char *fname)
+ {
+ 	int	fd =3D BasicOpenFile(fname, O_RDWR| PG_BINARY,
+ 						  S_IRUSR | S_IWUSR);
+=20
+ 	if (fd < 0)
+ 		ereport(ERROR,
+ 				(errcode_for_file_access(),
+ 				 errmsg("could not create file \"%s\": %m", fname)));
+=20
+ 	if (pg_fsync(fd) !=3D 0)
+ 		ereport(ERROR,
+ 				(errcode_for_file_access(),
+ 				 errmsg("could not fsync file \"%s\": %m", fname)));
+ 	close(fd);
+ }
=2D-=20
1.6.5.12.gd65df24


--Boundary-00=_fhTOLpTKgXC4pzN--