From: Tom Lane <tgl@sss.pgh.pa.us>
To: Thomas Munro <thomas.munro@gmail.com>
cc: Robert Haas <robertmhaas@gmail.com>, Larry Rosenman <ler@lerctr.org>,
        Pgsql hackers <pgsql-hackers@lists.postgresql.org>
Subject: Re: Fwd: Re: A new look at old NFS readdir() problems?
In-reply-to: <319417.1735860015@sss.pgh.pa.us>
References: <CA+hUKGJuG+P74iaVkBXBPE-rDsULjaruhiJ=qKYwbn_cpfpMQg@mail.gmail.com> <CAM5tNy7jFMb_zuLZddpX0dM9Cq=Ww-uVBSwS6b2AYD2bCPdbSw@mail.gmail.com> <04cf05d053e9320012b32370e228fac4@lerctr.org> <302248.1735850933@sss.pgh.pa.us> <CA+Tgmobh=vSvR4uDmPJ6Y00MptUeoPnKdx19yLpR7GpfXXnt5w@mail.gmail.com> <309402.1735854754@sss.pgh.pa.us> <CA+hUKG+rBEN5Z29AGHHGTaKa35nQugCQuGoOSQWGQNz=Rc8_nA@mail.gmail.com> <319417.1735860015@sss.pgh.pa.us>
Comments: In-reply-to Tom Lane <tgl@sss.pgh.pa.us>
	message dated "Thu, 02 Jan 2025 18:20:15 -0500"
MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="----- =_aaaaaaaaaa0"
Content-ID: <337811.1735869378.0@sss.pgh.pa.us>
Date: Thu, 02 Jan 2025 20:58:01 -0500
Message-ID: <338175.1735869481@sss.pgh.pa.us>
Archived-At: <https://www.postgresql.org/message-id/338175.1735869481%40sss.pgh.pa.us>
Precedence: bulk

------- =_aaaaaaaaaa0
Content-Type: text/plain; charset="us-ascii"
Content-ID: <337811.1735869378.1@sss.pgh.pa.us>

I wrote:
> Thomas Munro <thomas.munro@gmail.com> writes:
>> For what little it's worth, I'm not quite convinced yet that FreeBSD's
>> client isn't more broken than it needs to be.

> I'm suspicious of that too.

I poked at this a little further.  I made the attached stand-alone
test case (you don't need any more than "cc -o rmtree rmtree.c"
to build it, then point the script at some NFS-mounted directory).
This fails with my NAS at least as far back as FreeBSD 11.0.
I also tried it on NetBSD 9.2 which seems fine.

			regards, tom lane


------- =_aaaaaaaaaa0
Content-Type: text/x-shellscript; name="rmtree-test.sh"; charset="us-ascii"
Content-ID: <337811.1735869378.2@sss.pgh.pa.us>
Content-Description: rmtree-test.sh

#! /bin/sh

set -e

TESTDIR="$1"

mkdir "$TESTDIR"

i=0
while [ $i -lt 1000 ]
do
  touch "$TESTDIR/$i"
  i=`expr $i + 1`
done

./rmtree "$TESTDIR"

------- =_aaaaaaaaaa0
Content-Type: text/x-c; name="rmtree.c"; charset="us-ascii"
Content-ID: <337811.1735869378.3@sss.pgh.pa.us>
Content-Description: rmtree.c

/*-------------------------------------------------------------------------
 *
 * rmtree.c
 *
 * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
 * Portions Copyright (c) 1994, Regents of the University of California
 *
 * IDENTIFICATION
 *	  src/common/rmtree.c
 *
 *-------------------------------------------------------------------------
 */
#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
#include <string.h>
#include <unistd.h>
#include <errno.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <dirent.h>


typedef enum PGFileType
{
	PGFILETYPE_ERROR,
	PGFILETYPE_UNKNOWN,
	PGFILETYPE_REG,
	PGFILETYPE_DIR,
	PGFILETYPE_LNK,
} PGFileType;


static void *
palloc(size_t size)
{
	void	   *tmp;

	/* Avoid unportable behavior of malloc(0) */
	if (size == 0)
		size = 1;
	tmp = malloc(size);
	if (tmp == NULL)
	{
		fprintf(stderr, "out of memory\n");
		exit(1);
	}
	return tmp;
}

static void *
repalloc(void *ptr, size_t size)
{
	void	   *tmp;

	/* Avoid unportable behavior of realloc(NULL, 0) */
	if (ptr == NULL && size == 0)
		size = 1;
	tmp = realloc(ptr, size);
	if (!tmp)
	{
		fprintf(stderr, "out of memory\n");
		exit(1);
	}
	return tmp;
}

static char *
pstrdup(const char *in)
{
	char	   *tmp;

	if (!in)
	{
		fprintf(stderr,
				"cannot duplicate null pointer (internal error)\n");
		exit(1);
	}
	tmp = strdup(in);
	if (!tmp)
	{
		fprintf(stderr, "out of memory\n");
		exit(1);
	}
	return tmp;
}

/*
 * Return the type of a directory entry.
 */
static PGFileType
get_dirent_type(const char *path,
				const struct dirent *de,
				bool look_through_symlinks)
{
	PGFileType	result;

	/*
	 * Some systems tell us the type directly in the dirent struct, but that's
	 * a BSD and Linux extension not required by POSIX.  Even when the
	 * interface is present, sometimes the type is unknown, depending on the
	 * filesystem.
	 */
#if defined(DT_REG) && defined(DT_DIR) && defined(DT_LNK)
	if (de->d_type == DT_REG)
		result = PGFILETYPE_REG;
	else if (de->d_type == DT_DIR)
		result = PGFILETYPE_DIR;
	else if (de->d_type == DT_LNK && !look_through_symlinks)
		result = PGFILETYPE_LNK;
	else
		result = PGFILETYPE_UNKNOWN;
#else
	result = PGFILETYPE_UNKNOWN;
#endif

	if (result == PGFILETYPE_UNKNOWN)
	{
		struct stat fst;
		int			sret;


		if (look_through_symlinks)
			sret = stat(path, &fst);
		else
			sret = lstat(path, &fst);

		if (sret < 0)
		{
			result = PGFILETYPE_ERROR;
			fprintf(stderr, "could not stat file \"%s\": %m\n", path);
		}
		else if (S_ISREG(fst.st_mode))
			result = PGFILETYPE_REG;
		else if (S_ISDIR(fst.st_mode))
			result = PGFILETYPE_DIR;
		else if (S_ISLNK(fst.st_mode))
			result = PGFILETYPE_LNK;
	}

	return result;
}


/*
 *	rmtree
 *
 *	Delete a directory tree recursively.
 *	Assumes path points to a valid directory.
 *	Deletes everything under path.
 *	If rmtopdir is true deletes the directory too.
 *	Returns true if successful, false if there was any problem.
 *	(The details of the problem are reported already, so caller
 *	doesn't really have to say anything more, but most do.)
 */
static bool
rmtree(const char *path, bool rmtopdir)
{
	char		pathbuf[8192];
	DIR		   *dir;
	struct dirent *de;
	bool		result = true;
	size_t		dirnames_size = 0;
	size_t		dirnames_capacity = 8;
	char	  **dirnames;

	dir = opendir(path);
	if (dir == NULL)
	{
		fprintf(stderr, "could not open directory \"%s\": %m\n", path);
		return false;
	}

	dirnames = (char **) palloc(sizeof(char *) * dirnames_capacity);

	while (errno = 0, (de = readdir(dir)))
	{
		if (strcmp(de->d_name, ".") == 0 ||
			strcmp(de->d_name, "..") == 0)
			continue;
		snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name);
		switch (get_dirent_type(pathbuf, de, false))
		{
			case PGFILETYPE_ERROR:
				/* already logged, press on */
				break;
			case PGFILETYPE_DIR:

				/*
				 * Defer recursion until after we've closed this directory, to
				 * avoid using more than one file descriptor at a time.
				 */
				if (dirnames_size == dirnames_capacity)
				{
					dirnames = repalloc(dirnames,
										sizeof(char *) * dirnames_capacity * 2);
					dirnames_capacity *= 2;
				}
				dirnames[dirnames_size++] = pstrdup(pathbuf);
				break;
			default:
				if (unlink(pathbuf) != 0 && errno != ENOENT)
				{
					fprintf(stderr, "could not remove file \"%s\": %m\n", pathbuf);
					result = false;
				}
				break;
		}
	}

	if (errno != 0)
	{
		fprintf(stderr, "could not read directory \"%s\": %m\n", path);
		result = false;
	}

	closedir(dir);

	/* Now recurse into the subdirectories we found. */
	for (size_t i = 0; i < dirnames_size; ++i)
	{
		if (!rmtree(dirnames[i], true))
			result = false;
		free(dirnames[i]);
	}

	if (rmtopdir)
	{
		if (rmdir(path) != 0)
		{
			fprintf(stderr, "could not remove directory \"%s\": %m\n", path);
			result = false;
		}
	}

	free(dirnames);

	return result;
}

int
main(int argc, char **argv)
{
	if (argc != 2)
	{
		fprintf(stderr, "usage: %s target-directory\n", argv[0]);
		exit(1);
	}

	if (!rmtree(argv[1], true))
	{
		fprintf(stderr, "rmtree failed\n");
		exit(1);
	}

	return 0;
}

------- =_aaaaaaaaaa0--