agora inbox for pgsql-hackers@postgresql.org  

help / color / mirror / Atom feed

Patch for not going beyond NOFILE system limit (updated)
4+ messages / 3 participants
[nested] [flat]

* Patch for not going beyond NOFILE system limit (updated)
@ 2009-02-27 12:50  Peter Eisentraut <peter_e@gmx.net>
  0 siblings, 0 replies; 4+ messages in thread

From: Peter Eisentraut @ 2009-02-27 12:50 UTC (permalink / raw)
  To: pgsql-hackers

This is an updated and properly autoconf-guarded patch for this 
previously reported issue:

"""
Jacek Drobiecki recently sent me a patch which stops postgresql to
actively violate the system limit of maximum open files
(RLIMIT_NOFILE) in src/backend/storage/file/fd.c, function
count_usable_fds().

This avoids irritating kernel logs (if system overstep violations are
enabled) and also the grsecurity alert when starting PostgreSQL.
"""

References:
http://archives.postgresql.org/pgsql-bugs/2004-05/msg00103.php
http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=248967

I think we can put this into 8.4 or the first commit fest of 8.5.


Attachments:

  [text/x-diff] check-rlimits-nofile.patch (1.5K, ../../49A7E1A9.6050405@gmx.net/2-check-rlimits-nofile.patch)
  download | inline diff:
Index: src/backend/storage/file/fd.c
===================================================================
RCS file: /cvsroot/pgsql/src/backend/storage/file/fd.c,v
retrieving revision 1.147
diff -u -3 -p -r1.147 fd.c
--- src/backend/storage/file/fd.c	12 Jan 2009 05:10:44 -0000	1.147
+++ src/backend/storage/file/fd.c	27 Feb 2009 12:45:25 -0000
@@ -45,6 +45,9 @@
 #include <sys/stat.h>
 #include <unistd.h>
 #include <fcntl.h>
+#ifdef HAVE_SYS_RESOURCE_H
+#include <sys/resource.h>		/* for getrlimit */
+#endif
 
 #include "miscadmin.h"
 #include "access/xact.h"
@@ -361,15 +364,35 @@ count_usable_fds(int max_to_probe, int *
 	int			used = 0;
 	int			highestfd = 0;
 	int			j;
+#ifdef HAVE_GETRLIMIT
+	struct rlimit rlim;
+	int			getrlimit_status;
+#endif
 
 	size = 1024;
 	fd = (int *) palloc(size * sizeof(int));
 
+#ifdef HAVE_GETRLIMIT
+# ifdef RLIMIT_NOFILE            /* most platforms use RLIMIT_NOFILE */
+	getrlimit_status = getrlimit(RLIMIT_NOFILE, &rlim);
+# else                           /* but BSD doesn't ... */
+	getrlimit_status = getrlimit(RLIMIT_OFILE, &rlim);
+# endif /* RLIMIT_NOFILE */
+	if (getrlimit_status != 0)
+		ereport(WARNING, (errmsg("getrlimit failed: %m")));
+#endif /* HAVE_GETRLIMIT */
+
 	/* dup until failure or probe limit reached */
 	for (;;)
 	{
 		int			thisfd;
 
+#ifdef HAVE_GETRLIMIT
+		/* don't go beyond RLIMIT_NOFILE */
+		if (getrlimit_status == 0 && highestfd >= rlim.rlim_cur - 1)
+			break;
+#endif
+
 		thisfd = dup(0);
 		if (thisfd < 0)
 		{


^ permalink  raw  reply  [nested|flat] 4+ messages in thread

* [PATCH 2/6] Avoid the most useless instances of nulls[0]=false..
@ 2022-01-15 01:25  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 4+ messages in thread

From: Justin Pryzby @ 2022-01-15 01:25 UTC (permalink / raw)

Getting rid of more of these is apparently too controvercial, but these are the
cases where 1) nulls is always being set to false; and, 2) values[] is not
being memset, so there's no parallel between the two.
---
 contrib/sslinfo/sslinfo.c                |  5 +-
 src/backend/replication/logical/origin.c | 13 +---
 src/backend/utils/adt/misc.c             |  7 +--
 src/backend/utils/misc/pg_controldata.c  | 80 ++----------------------
 4 files changed, 10 insertions(+), 95 deletions(-)

diff --git a/contrib/sslinfo/sslinfo.c b/contrib/sslinfo/sslinfo.c
index 5fd46b98741..d9edfe1b742 100644
--- a/contrib/sslinfo/sslinfo.c
+++ b/contrib/sslinfo/sslinfo.c
@@ -424,7 +424,7 @@ ssl_extension_info(PG_FUNCTION_ARGS)
 	if (call_cntr < max_calls)
 	{
 		Datum		values[3];
-		bool		nulls[3];
+		bool		nulls[3] = {0};
 		char	   *buf;
 		HeapTuple	tuple;
 		Datum		result;
@@ -453,7 +453,6 @@ ssl_extension_info(PG_FUNCTION_ARGS)
 					 errmsg("unknown OpenSSL extension in certificate at position %d",
 							call_cntr)));
 		values[0] = CStringGetTextDatum(OBJ_nid2sn(nid));
-		nulls[0] = false;
 
 		/* Get the extension value */
 		if (X509V3_EXT_print(membuf, ext, 0, 0) <= 0)
@@ -463,11 +462,9 @@ ssl_extension_info(PG_FUNCTION_ARGS)
 							call_cntr)));
 		len = BIO_get_mem_data(membuf, &buf);
 		values[1] = PointerGetDatum(cstring_to_text_with_len(buf, len));
-		nulls[1] = false;
 
 		/* Get critical status */
 		values[2] = BoolGetDatum(X509_EXTENSION_get_critical(ext));
-		nulls[2] = false;
 
 		/* Build tuple */
 		tuple = heap_form_tuple(fctx->tupdesc, values, nulls);
diff --git a/src/backend/replication/logical/origin.c b/src/backend/replication/logical/origin.c
index f134e44878f..adb3430c33a 100644
--- a/src/backend/replication/logical/origin.c
+++ b/src/backend/replication/logical/origin.c
@@ -1527,10 +1527,9 @@ pg_show_replication_origin_status(PG_FUNCTION_ARGS)
 			continue;
 
 		memset(values, 0, sizeof(values));
-		memset(nulls, 1, sizeof(nulls));
+		memset(nulls, 0, sizeof(nulls));
 
 		values[0] = ObjectIdGetDatum(state->roident);
-		nulls[0] = false;
 
 		/*
 		 * We're not preventing the origin to be dropped concurrently, so
@@ -1538,19 +1537,13 @@ pg_show_replication_origin_status(PG_FUNCTION_ARGS)
 		 */
 		if (replorigin_by_oid(state->roident, true,
 							  &roname))
-		{
 			values[1] = CStringGetTextDatum(roname);
-			nulls[1] = false;
-		}
+		else
+			nulls[1] = true;
 
 		LWLockAcquire(&state->lock, LW_SHARED);
-
 		values[2] = LSNGetDatum(state->remote_lsn);
-		nulls[2] = false;
-
 		values[3] = LSNGetDatum(state->local_lsn);
-		nulls[3] = false;
-
 		LWLockRelease(&state->lock);
 
 		tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc,
diff --git a/src/backend/utils/adt/misc.c b/src/backend/utils/adt/misc.c
index 9c132512315..611b492624a 100644
--- a/src/backend/utils/adt/misc.c
+++ b/src/backend/utils/adt/misc.c
@@ -246,7 +246,7 @@ pg_tablespace_databases(PG_FUNCTION_ARGS)
 		char	   *subdir;
 		bool		isempty;
 		Datum		values[1];
-		bool		nulls[1];
+		bool		nulls[1] = {0};
 
 		/* this test skips . and .., but is awfully weak */
 		if (!datOid)
@@ -262,7 +262,6 @@ pg_tablespace_databases(PG_FUNCTION_ARGS)
 			continue;			/* indeed, nothing in it */
 
 		values[0] = ObjectIdGetDatum(datOid);
-		nulls[0] = false;
 
 		tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc,
 							 values, nulls);
@@ -530,11 +529,9 @@ pg_get_catalog_foreign_keys(PG_FUNCTION_ARGS)
 	{
 		const SysFKRelationship *fkrel = &sys_fk_relationships[funcctx->call_cntr];
 		Datum		values[6];
-		bool		nulls[6];
+		bool		nulls[6] = {0};
 		HeapTuple	tuple;
 
-		memset(nulls, false, sizeof(nulls));
-
 		values[0] = ObjectIdGetDatum(fkrel->fk_table);
 		values[1] = FunctionCall3(arrayinp,
 								  CStringGetDatum(fkrel->fk_columns),
diff --git a/src/backend/utils/misc/pg_controldata.c b/src/backend/utils/misc/pg_controldata.c
index 781f8b87580..32be2b4b5d8 100644
--- a/src/backend/utils/misc/pg_controldata.c
+++ b/src/backend/utils/misc/pg_controldata.c
@@ -32,7 +32,7 @@ Datum
 pg_control_system(PG_FUNCTION_ARGS)
 {
 	Datum		values[4];
-	bool		nulls[4];
+	bool		nulls[4] = {0};
 	TupleDesc	tupdesc;
 	HeapTuple	htup;
 	ControlFileData *ControlFile;
@@ -60,16 +60,9 @@ pg_control_system(PG_FUNCTION_ARGS)
 				(errmsg("calculated CRC checksum does not match value stored in file")));
 
 	values[0] = Int32GetDatum(ControlFile->pg_control_version);
-	nulls[0] = false;
-
 	values[1] = Int32GetDatum(ControlFile->catalog_version_no);
-	nulls[1] = false;
-
 	values[2] = Int64GetDatum(ControlFile->system_identifier);
-	nulls[2] = false;
-
 	values[3] = TimestampTzGetDatum(time_t_to_timestamptz(ControlFile->time));
-	nulls[3] = false;
 
 	htup = heap_form_tuple(tupdesc, values, nulls);
 
@@ -80,7 +73,7 @@ Datum
 pg_control_checkpoint(PG_FUNCTION_ARGS)
 {
 	Datum		values[18];
-	bool		nulls[18];
+	bool		nulls[18] = {0};
 	TupleDesc	tupdesc;
 	HeapTuple	htup;
 	ControlFileData *ControlFile;
@@ -147,60 +140,25 @@ pg_control_checkpoint(PG_FUNCTION_ARGS)
 
 	/* Populate the values and null arrays */
 	values[0] = LSNGetDatum(ControlFile->checkPoint);
-	nulls[0] = false;
-
 	values[1] = LSNGetDatum(ControlFile->checkPointCopy.redo);
-	nulls[1] = false;
-
 	values[2] = CStringGetTextDatum(xlogfilename);
-	nulls[2] = false;
-
 	values[3] = Int32GetDatum(ControlFile->checkPointCopy.ThisTimeLineID);
-	nulls[3] = false;
-
 	values[4] = Int32GetDatum(ControlFile->checkPointCopy.PrevTimeLineID);
-	nulls[4] = false;
-
 	values[5] = BoolGetDatum(ControlFile->checkPointCopy.fullPageWrites);
-	nulls[5] = false;
-
 	values[6] = CStringGetTextDatum(psprintf("%u:%u",
 											 EpochFromFullTransactionId(ControlFile->checkPointCopy.nextXid),
 											 XidFromFullTransactionId(ControlFile->checkPointCopy.nextXid)));
-	nulls[6] = false;
-
 	values[7] = ObjectIdGetDatum(ControlFile->checkPointCopy.nextOid);
-	nulls[7] = false;
-
 	values[8] = TransactionIdGetDatum(ControlFile->checkPointCopy.nextMulti);
-	nulls[8] = false;
-
 	values[9] = TransactionIdGetDatum(ControlFile->checkPointCopy.nextMultiOffset);
-	nulls[9] = false;
-
 	values[10] = TransactionIdGetDatum(ControlFile->checkPointCopy.oldestXid);
-	nulls[10] = false;
-
 	values[11] = ObjectIdGetDatum(ControlFile->checkPointCopy.oldestXidDB);
-	nulls[11] = false;
-
 	values[12] = TransactionIdGetDatum(ControlFile->checkPointCopy.oldestActiveXid);
-	nulls[12] = false;
-
 	values[13] = TransactionIdGetDatum(ControlFile->checkPointCopy.oldestMulti);
-	nulls[13] = false;
-
 	values[14] = ObjectIdGetDatum(ControlFile->checkPointCopy.oldestMultiDB);
-	nulls[14] = false;
-
 	values[15] = TransactionIdGetDatum(ControlFile->checkPointCopy.oldestCommitTsXid);
-	nulls[15] = false;
-
 	values[16] = TransactionIdGetDatum(ControlFile->checkPointCopy.newestCommitTsXid);
-	nulls[16] = false;
-
 	values[17] = TimestampTzGetDatum(time_t_to_timestamptz(ControlFile->checkPointCopy.time));
-	nulls[17] = false;
 
 	htup = heap_form_tuple(tupdesc, values, nulls);
 
@@ -211,7 +169,7 @@ Datum
 pg_control_recovery(PG_FUNCTION_ARGS)
 {
 	Datum		values[5];
-	bool		nulls[5];
+	bool		nulls[5] = {0};
 	TupleDesc	tupdesc;
 	HeapTuple	htup;
 	ControlFileData *ControlFile;
@@ -241,19 +199,10 @@ pg_control_recovery(PG_FUNCTION_ARGS)
 				(errmsg("calculated CRC checksum does not match value stored in file")));
 
 	values[0] = LSNGetDatum(ControlFile->minRecoveryPoint);
-	nulls[0] = false;
-
 	values[1] = Int32GetDatum(ControlFile->minRecoveryPointTLI);
-	nulls[1] = false;
-
 	values[2] = LSNGetDatum(ControlFile->backupStartPoint);
-	nulls[2] = false;
-
 	values[3] = LSNGetDatum(ControlFile->backupEndPoint);
-	nulls[3] = false;
-
 	values[4] = BoolGetDatum(ControlFile->backupEndRequired);
-	nulls[4] = false;
 
 	htup = heap_form_tuple(tupdesc, values, nulls);
 
@@ -264,7 +213,7 @@ Datum
 pg_control_init(PG_FUNCTION_ARGS)
 {
 	Datum		values[11];
-	bool		nulls[11];
+	bool		nulls[11] = {0};
 	TupleDesc	tupdesc;
 	HeapTuple	htup;
 	ControlFileData *ControlFile;
@@ -306,37 +255,16 @@ pg_control_init(PG_FUNCTION_ARGS)
 				(errmsg("calculated CRC checksum does not match value stored in file")));
 
 	values[0] = Int32GetDatum(ControlFile->maxAlign);
-	nulls[0] = false;
-
 	values[1] = Int32GetDatum(ControlFile->blcksz);
-	nulls[1] = false;
-
 	values[2] = Int32GetDatum(ControlFile->relseg_size);
-	nulls[2] = false;
-
 	values[3] = Int32GetDatum(ControlFile->xlog_blcksz);
-	nulls[3] = false;
-
 	values[4] = Int32GetDatum(ControlFile->xlog_seg_size);
-	nulls[4] = false;
-
 	values[5] = Int32GetDatum(ControlFile->nameDataLen);
-	nulls[5] = false;
-
 	values[6] = Int32GetDatum(ControlFile->indexMaxKeys);
-	nulls[6] = false;
-
 	values[7] = Int32GetDatum(ControlFile->toast_max_chunk_size);
-	nulls[7] = false;
-
 	values[8] = Int32GetDatum(ControlFile->loblksize);
-	nulls[8] = false;
-
 	values[9] = BoolGetDatum(ControlFile->float8ByVal);
-	nulls[9] = false;
-
 	values[10] = Int32GetDatum(ControlFile->data_checksum_version);
-	nulls[10] = false;
 
 	htup = heap_form_tuple(tupdesc, values, nulls);
 
-- 
2.25.1


--EqVOK5mkaJAMmtSx
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-selfuncs.c-refactor-parent-ACL-check.patch"



^ permalink  raw  reply  [nested|flat] 4+ messages in thread

* [PATCH 2/6] Avoid the most useless instances of nulls[0]=false..
@ 2022-01-15 01:25  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 4+ messages in thread

From: Justin Pryzby @ 2022-01-15 01:25 UTC (permalink / raw)

Getting rid of more of these is apparently too controvercial, but these are the
cases where 1) nulls is always being set to false; and, 2) values[] is not
being memset, so there's no parallel between the two.
---
 contrib/sslinfo/sslinfo.c                |  5 +-
 src/backend/replication/logical/origin.c | 13 +---
 src/backend/utils/adt/misc.c             |  7 +--
 src/backend/utils/misc/pg_controldata.c  | 80 ++----------------------
 4 files changed, 10 insertions(+), 95 deletions(-)

diff --git a/contrib/sslinfo/sslinfo.c b/contrib/sslinfo/sslinfo.c
index 5fd46b98741..d9edfe1b742 100644
--- a/contrib/sslinfo/sslinfo.c
+++ b/contrib/sslinfo/sslinfo.c
@@ -424,7 +424,7 @@ ssl_extension_info(PG_FUNCTION_ARGS)
 	if (call_cntr < max_calls)
 	{
 		Datum		values[3];
-		bool		nulls[3];
+		bool		nulls[3] = {0};
 		char	   *buf;
 		HeapTuple	tuple;
 		Datum		result;
@@ -453,7 +453,6 @@ ssl_extension_info(PG_FUNCTION_ARGS)
 					 errmsg("unknown OpenSSL extension in certificate at position %d",
 							call_cntr)));
 		values[0] = CStringGetTextDatum(OBJ_nid2sn(nid));
-		nulls[0] = false;
 
 		/* Get the extension value */
 		if (X509V3_EXT_print(membuf, ext, 0, 0) <= 0)
@@ -463,11 +462,9 @@ ssl_extension_info(PG_FUNCTION_ARGS)
 							call_cntr)));
 		len = BIO_get_mem_data(membuf, &buf);
 		values[1] = PointerGetDatum(cstring_to_text_with_len(buf, len));
-		nulls[1] = false;
 
 		/* Get critical status */
 		values[2] = BoolGetDatum(X509_EXTENSION_get_critical(ext));
-		nulls[2] = false;
 
 		/* Build tuple */
 		tuple = heap_form_tuple(fctx->tupdesc, values, nulls);
diff --git a/src/backend/replication/logical/origin.c b/src/backend/replication/logical/origin.c
index f134e44878f..adb3430c33a 100644
--- a/src/backend/replication/logical/origin.c
+++ b/src/backend/replication/logical/origin.c
@@ -1527,10 +1527,9 @@ pg_show_replication_origin_status(PG_FUNCTION_ARGS)
 			continue;
 
 		memset(values, 0, sizeof(values));
-		memset(nulls, 1, sizeof(nulls));
+		memset(nulls, 0, sizeof(nulls));
 
 		values[0] = ObjectIdGetDatum(state->roident);
-		nulls[0] = false;
 
 		/*
 		 * We're not preventing the origin to be dropped concurrently, so
@@ -1538,19 +1537,13 @@ pg_show_replication_origin_status(PG_FUNCTION_ARGS)
 		 */
 		if (replorigin_by_oid(state->roident, true,
 							  &roname))
-		{
 			values[1] = CStringGetTextDatum(roname);
-			nulls[1] = false;
-		}
+		else
+			nulls[1] = true;
 
 		LWLockAcquire(&state->lock, LW_SHARED);
-
 		values[2] = LSNGetDatum(state->remote_lsn);
-		nulls[2] = false;
-
 		values[3] = LSNGetDatum(state->local_lsn);
-		nulls[3] = false;
-
 		LWLockRelease(&state->lock);
 
 		tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc,
diff --git a/src/backend/utils/adt/misc.c b/src/backend/utils/adt/misc.c
index 9c132512315..611b492624a 100644
--- a/src/backend/utils/adt/misc.c
+++ b/src/backend/utils/adt/misc.c
@@ -246,7 +246,7 @@ pg_tablespace_databases(PG_FUNCTION_ARGS)
 		char	   *subdir;
 		bool		isempty;
 		Datum		values[1];
-		bool		nulls[1];
+		bool		nulls[1] = {0};
 
 		/* this test skips . and .., but is awfully weak */
 		if (!datOid)
@@ -262,7 +262,6 @@ pg_tablespace_databases(PG_FUNCTION_ARGS)
 			continue;			/* indeed, nothing in it */
 
 		values[0] = ObjectIdGetDatum(datOid);
-		nulls[0] = false;
 
 		tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc,
 							 values, nulls);
@@ -530,11 +529,9 @@ pg_get_catalog_foreign_keys(PG_FUNCTION_ARGS)
 	{
 		const SysFKRelationship *fkrel = &sys_fk_relationships[funcctx->call_cntr];
 		Datum		values[6];
-		bool		nulls[6];
+		bool		nulls[6] = {0};
 		HeapTuple	tuple;
 
-		memset(nulls, false, sizeof(nulls));
-
 		values[0] = ObjectIdGetDatum(fkrel->fk_table);
 		values[1] = FunctionCall3(arrayinp,
 								  CStringGetDatum(fkrel->fk_columns),
diff --git a/src/backend/utils/misc/pg_controldata.c b/src/backend/utils/misc/pg_controldata.c
index 781f8b87580..32be2b4b5d8 100644
--- a/src/backend/utils/misc/pg_controldata.c
+++ b/src/backend/utils/misc/pg_controldata.c
@@ -32,7 +32,7 @@ Datum
 pg_control_system(PG_FUNCTION_ARGS)
 {
 	Datum		values[4];
-	bool		nulls[4];
+	bool		nulls[4] = {0};
 	TupleDesc	tupdesc;
 	HeapTuple	htup;
 	ControlFileData *ControlFile;
@@ -60,16 +60,9 @@ pg_control_system(PG_FUNCTION_ARGS)
 				(errmsg("calculated CRC checksum does not match value stored in file")));
 
 	values[0] = Int32GetDatum(ControlFile->pg_control_version);
-	nulls[0] = false;
-
 	values[1] = Int32GetDatum(ControlFile->catalog_version_no);
-	nulls[1] = false;
-
 	values[2] = Int64GetDatum(ControlFile->system_identifier);
-	nulls[2] = false;
-
 	values[3] = TimestampTzGetDatum(time_t_to_timestamptz(ControlFile->time));
-	nulls[3] = false;
 
 	htup = heap_form_tuple(tupdesc, values, nulls);
 
@@ -80,7 +73,7 @@ Datum
 pg_control_checkpoint(PG_FUNCTION_ARGS)
 {
 	Datum		values[18];
-	bool		nulls[18];
+	bool		nulls[18] = {0};
 	TupleDesc	tupdesc;
 	HeapTuple	htup;
 	ControlFileData *ControlFile;
@@ -147,60 +140,25 @@ pg_control_checkpoint(PG_FUNCTION_ARGS)
 
 	/* Populate the values and null arrays */
 	values[0] = LSNGetDatum(ControlFile->checkPoint);
-	nulls[0] = false;
-
 	values[1] = LSNGetDatum(ControlFile->checkPointCopy.redo);
-	nulls[1] = false;
-
 	values[2] = CStringGetTextDatum(xlogfilename);
-	nulls[2] = false;
-
 	values[3] = Int32GetDatum(ControlFile->checkPointCopy.ThisTimeLineID);
-	nulls[3] = false;
-
 	values[4] = Int32GetDatum(ControlFile->checkPointCopy.PrevTimeLineID);
-	nulls[4] = false;
-
 	values[5] = BoolGetDatum(ControlFile->checkPointCopy.fullPageWrites);
-	nulls[5] = false;
-
 	values[6] = CStringGetTextDatum(psprintf("%u:%u",
 											 EpochFromFullTransactionId(ControlFile->checkPointCopy.nextXid),
 											 XidFromFullTransactionId(ControlFile->checkPointCopy.nextXid)));
-	nulls[6] = false;
-
 	values[7] = ObjectIdGetDatum(ControlFile->checkPointCopy.nextOid);
-	nulls[7] = false;
-
 	values[8] = TransactionIdGetDatum(ControlFile->checkPointCopy.nextMulti);
-	nulls[8] = false;
-
 	values[9] = TransactionIdGetDatum(ControlFile->checkPointCopy.nextMultiOffset);
-	nulls[9] = false;
-
 	values[10] = TransactionIdGetDatum(ControlFile->checkPointCopy.oldestXid);
-	nulls[10] = false;
-
 	values[11] = ObjectIdGetDatum(ControlFile->checkPointCopy.oldestXidDB);
-	nulls[11] = false;
-
 	values[12] = TransactionIdGetDatum(ControlFile->checkPointCopy.oldestActiveXid);
-	nulls[12] = false;
-
 	values[13] = TransactionIdGetDatum(ControlFile->checkPointCopy.oldestMulti);
-	nulls[13] = false;
-
 	values[14] = ObjectIdGetDatum(ControlFile->checkPointCopy.oldestMultiDB);
-	nulls[14] = false;
-
 	values[15] = TransactionIdGetDatum(ControlFile->checkPointCopy.oldestCommitTsXid);
-	nulls[15] = false;
-
 	values[16] = TransactionIdGetDatum(ControlFile->checkPointCopy.newestCommitTsXid);
-	nulls[16] = false;
-
 	values[17] = TimestampTzGetDatum(time_t_to_timestamptz(ControlFile->checkPointCopy.time));
-	nulls[17] = false;
 
 	htup = heap_form_tuple(tupdesc, values, nulls);
 
@@ -211,7 +169,7 @@ Datum
 pg_control_recovery(PG_FUNCTION_ARGS)
 {
 	Datum		values[5];
-	bool		nulls[5];
+	bool		nulls[5] = {0};
 	TupleDesc	tupdesc;
 	HeapTuple	htup;
 	ControlFileData *ControlFile;
@@ -241,19 +199,10 @@ pg_control_recovery(PG_FUNCTION_ARGS)
 				(errmsg("calculated CRC checksum does not match value stored in file")));
 
 	values[0] = LSNGetDatum(ControlFile->minRecoveryPoint);
-	nulls[0] = false;
-
 	values[1] = Int32GetDatum(ControlFile->minRecoveryPointTLI);
-	nulls[1] = false;
-
 	values[2] = LSNGetDatum(ControlFile->backupStartPoint);
-	nulls[2] = false;
-
 	values[3] = LSNGetDatum(ControlFile->backupEndPoint);
-	nulls[3] = false;
-
 	values[4] = BoolGetDatum(ControlFile->backupEndRequired);
-	nulls[4] = false;
 
 	htup = heap_form_tuple(tupdesc, values, nulls);
 
@@ -264,7 +213,7 @@ Datum
 pg_control_init(PG_FUNCTION_ARGS)
 {
 	Datum		values[11];
-	bool		nulls[11];
+	bool		nulls[11] = {0};
 	TupleDesc	tupdesc;
 	HeapTuple	htup;
 	ControlFileData *ControlFile;
@@ -306,37 +255,16 @@ pg_control_init(PG_FUNCTION_ARGS)
 				(errmsg("calculated CRC checksum does not match value stored in file")));
 
 	values[0] = Int32GetDatum(ControlFile->maxAlign);
-	nulls[0] = false;
-
 	values[1] = Int32GetDatum(ControlFile->blcksz);
-	nulls[1] = false;
-
 	values[2] = Int32GetDatum(ControlFile->relseg_size);
-	nulls[2] = false;
-
 	values[3] = Int32GetDatum(ControlFile->xlog_blcksz);
-	nulls[3] = false;
-
 	values[4] = Int32GetDatum(ControlFile->xlog_seg_size);
-	nulls[4] = false;
-
 	values[5] = Int32GetDatum(ControlFile->nameDataLen);
-	nulls[5] = false;
-
 	values[6] = Int32GetDatum(ControlFile->indexMaxKeys);
-	nulls[6] = false;
-
 	values[7] = Int32GetDatum(ControlFile->toast_max_chunk_size);
-	nulls[7] = false;
-
 	values[8] = Int32GetDatum(ControlFile->loblksize);
-	nulls[8] = false;
-
 	values[9] = BoolGetDatum(ControlFile->float8ByVal);
-	nulls[9] = false;
-
 	values[10] = Int32GetDatum(ControlFile->data_checksum_version);
-	nulls[10] = false;
 
 	htup = heap_form_tuple(tupdesc, values, nulls);
 
-- 
2.25.1


--EqVOK5mkaJAMmtSx
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-selfuncs.c-refactor-parent-ACL-check.patch"



^ permalink  raw  reply  [nested|flat] 4+ messages in thread

* [PATCH v4 7/8] Use anonymous files to back shared memory segments
@ 2025-03-15 15:39  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 4+ messages in thread

From: Dmitrii Dolgov @ 2025-03-15 15:39 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f90cde00000-7f90d5126000 rw-s 00000000 00:01 5463 /memfd:main (deleted)
7f90d5126000-7f914de00000 ---p 00000000 00:00 0
7f914de00000-7f9175128000 rw-s 00000000 00:01 5466 /memfd:buffers (deleted)
7f9175128000-7f944de00000 ---p 00000000 00:00 0
7f944de00000-7f9455528000 rw-s 00000000 00:01 5469 /memfd:descriptors (deleted)
7f9455528000-7f94cde00000 ---p 00000000 00:00 0
7f94cde00000-7f94d5228000 rw-s 00000000 00:01 5472 /memfd:iocv (deleted)
7f94d5228000-7f954de00000 ---p 00000000 00:00 0
7f954de00000-7f9555266000 rw-s 00000000 00:01 5475 /memfd:checkpoint (deleted)
7f9555266000-7f958de00000 ---p 00000000 00:00 0
7f958de00000-7f95954aa000 rw-s 00000000 00:01 5478 /memfd:strategy (deleted)
7f95954aa000-7f95cde00000 ---p 00000000 00:00 0

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c  | 73 +++++++++++++++++++++++++++++-----
 src/backend/port/win32_shmem.c |  2 +-
 src/backend/storage/ipc/ipci.c |  2 +-
 src/include/portability/mem.h  |  2 +-
 src/include/storage/pg_shmem.h |  3 +-
 5 files changed, 68 insertions(+), 14 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index a3437973784..87000a24eea 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -107,6 +107,7 @@ typedef struct AnonymousMapping
 	Pointer shmem; 				/* Pointer to the start of the mapped memory */
 	Pointer seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -127,7 +128,7 @@ static int next_free_segment = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -150,9 +151,9 @@ static int next_free_segment = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * 7f442e010000-7f443a800000                     # reserved empty space
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * 7f444196c000-7f470a800000                     # reserved empty space
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -643,13 +644,14 @@ PGSharedMemoryAttach(IpcMemoryId shmId,
  * *hugepagesize and *mmap_flags are set to 0.
  */
 void
-GetHugePageSize(Size *hugepagesize, int *mmap_flags)
+GetHugePageSize(Size *hugepagesize, int *mmap_flags, int *memfd_flags)
 {
 #ifdef MAP_HUGETLB
 
 	Size		default_hugepagesize = 0;
 	Size		hugepagesize_local = 0;
 	int			mmap_flags_local = 0;
+	int			memfd_flags_local = 0;
 
 	/*
 	 * System-dependent code to find out the default huge page size.
@@ -708,6 +710,7 @@ GetHugePageSize(Size *hugepagesize, int *mmap_flags)
 	}
 
 	mmap_flags_local = MAP_HUGETLB;
+	memfd_flags_local = MFD_HUGETLB;
 
 	/*
 	 * On recent enough Linux, also include the explicit page size, if
@@ -718,7 +721,16 @@ GetHugePageSize(Size *hugepagesize, int *mmap_flags)
 	{
 		int			shift = pg_ceil_log2_64(hugepagesize_local);
 
-		mmap_flags_local |= (shift & MAP_HUGE_MASK) << MAP_HUGE_SHIFT;
+		memfd_flags_local |= (shift & MAP_HUGE_MASK) << MAP_HUGE_SHIFT;
+	}
+#endif
+
+#if defined(MFD_HUGE_MASK) && defined(MFD_HUGE_SHIFT)
+	if (hugepagesize_local != default_hugepagesize)
+	{
+		int			shift = pg_ceil_log2_64(hugepagesize_local);
+
+		memfd_flags_local |= (shift & MAP_HUGE_MASK) << MAP_HUGE_SHIFT;
 	}
 #endif
 
@@ -727,6 +739,8 @@ GetHugePageSize(Size *hugepagesize, int *mmap_flags)
 		*mmap_flags = mmap_flags_local;
 	if (hugepagesize)
 		*hugepagesize = hugepagesize_local;
+	if (memfd_flags)
+		*memfd_flags = memfd_flags_local;
 
 #else
 
@@ -734,6 +748,8 @@ GetHugePageSize(Size *hugepagesize, int *mmap_flags)
 		*hugepagesize = 0;
 	if (mmap_flags)
 		*mmap_flags = 0;
+	if (memfd_flags)
+		*memfd_flags = 0;
 
 #endif							/* MAP_HUGETLB */
 }
@@ -771,7 +787,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping, Pointer base)
 	Size		allocsize = mapping->shmem_size;
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
-	int			mmap_flags = PG_MMAP_FLAGS;
+	int			mmap_flags = PG_MMAP_FLAGS, memfd_flags = 0;
 
 #ifndef MAP_HUGETLB
 	/* ReserveAnonymousMemory should have dealt with this case */
@@ -785,7 +801,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping, Pointer base)
 		Assert(huge_pages == HUGE_PAGES_ON || huge_pages == HUGE_PAGES_TRY);
 
 		/* Round up the request size to a suitable large value */
-		GetHugePageSize(&hugepagesize, &mmap_flags);
+		GetHugePageSize(&hugepagesize, &mmap_flags, &memfd_flags);
 
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
@@ -794,6 +810,29 @@ CreateAnonymousSegment(AnonymousMapping *mapping, Pointer base)
 	}
 #endif
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_segment),
+									   memfd_flags);
+
+	/*
+	 * Specify the segment file size using allocsize, which contains
+	 * potentially modified size.
+	 */
+	if(ftruncate(mapping->segment_fd, allocsize) == -1)
+		ereport(FATAL,
+				(errcode(ERRCODE_SYSTEM_ERROR),
+				 errmsg("could not truncase anonymous file for \"%s\": %m",
+						MappingName(mapping->shmem_segment))));
+
 	elog(DEBUG1, "segment[%s]: mmap(%zu) at address %p",
 		 MappingName(mapping->shmem_segment), allocsize, base + reserved_offset);
 
@@ -807,7 +846,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping, Pointer base)
 	 * a restart.
 	 */
 	ptr = mmap(base + reserved_offset, allocsize, PROT_READ | PROT_WRITE,
-			   mmap_flags | MAP_FIXED, -1, 0);
+			   mmap_flags | MAP_FIXED, mapping->segment_fd, 0);
 	mmap_errno = errno;
 
 	if (ptr == MAP_FAILED)
@@ -817,8 +856,15 @@ CreateAnonymousSegment(AnonymousMapping *mapping, Pointer base)
 					 "fallback to the non-resizable allocation",
 			 MappingName(mapping->shmem_segment), allocsize, base + reserved_offset);
 
+		/* Specify the segment file size using allocsize. */
+		if(ftruncate(mapping->segment_fd, allocsize) == -1)
+			ereport(FATAL,
+					(errcode(ERRCODE_SYSTEM_ERROR),
+					 errmsg("could not truncase anonymous file for \"%s\": %m",
+							MappingName(mapping->shmem_segment))));
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 	else
@@ -889,7 +935,7 @@ ReserveAnonymousMemory(Size reserve_size)
 		Size		hugepagesize, total_size = 0;
 		int			mmap_flags;
 
-		GetHugePageSize(&hugepagesize, &mmap_flags);
+		GetHugePageSize(&hugepagesize, &mmap_flags, NULL);
 
 		/*
 		 * Figure out how much memory is needed for all segments, keeping in
@@ -1070,6 +1116,13 @@ AnonymousShmemResize(void)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		if(ftruncate(m->segment_fd, new_size) == -1)
+			ereport(FATAL,
+					(errcode(ERRCODE_SYSTEM_ERROR),
+					 errmsg("could not truncase anonymous file for \"%s\": %m",
+							MappingName(m->shmem_segment))));
+
 		/* Clean up some reserved space to resize into */
 		if (munmap(m->shmem + m->shmem_size, new_size - m->shmem_size) == -1)
 			ereport(FATAL,
diff --git a/src/backend/port/win32_shmem.c b/src/backend/port/win32_shmem.c
index ce719f1b412..ba972106de1 100644
--- a/src/backend/port/win32_shmem.c
+++ b/src/backend/port/win32_shmem.c
@@ -627,7 +627,7 @@ pgwin32_ReserveSharedMemoryRegion(HANDLE hChild)
  * use GetLargePageMinimum() instead.
  */
 void
-GetHugePageSize(Size *hugepagesize, int *mmap_flags)
+GetHugePageSize(Size *hugepagesize, int *mmap_flags, int *memfd_flags)
 {
 	if (hugepagesize)
 		*hugepagesize = 0;
diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c
index abeb91e24fd..dc2b4becf4a 100644
--- a/src/backend/storage/ipc/ipci.c
+++ b/src/backend/storage/ipc/ipci.c
@@ -396,7 +396,7 @@ InitializeShmemGUCs(void)
 	/*
 	 * Calculate the number of huge pages required.
 	 */
-	GetHugePageSize(&hp_size, NULL);
+	GetHugePageSize(&hp_size, NULL, NULL);
 	if (hp_size != 0)
 	{
 		Size		hp_required;
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index ef9800732d9..40588ff6968 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
diff --git a/src/include/storage/pg_shmem.h b/src/include/storage/pg_shmem.h
index 2e47b222cbb..b9573520d9a 100644
--- a/src/include/storage/pg_shmem.h
+++ b/src/include/storage/pg_shmem.h
@@ -124,7 +124,8 @@ extern PGShmemHeader *PGSharedMemoryCreate(Size size,
 										   PGShmemHeader **shim, Pointer base);
 extern bool PGSharedMemoryIsInUse(unsigned long id1, unsigned long id2);
 extern void PGSharedMemoryDetach(void);
-extern void GetHugePageSize(Size *hugepagesize, int *mmap_flags);
+extern void GetHugePageSize(Size *hugepagesize, int *mmap_flags,
+							int *memfd_flags);
 void *ReserveAnonymousMemory(Size reserve_size);
 
 bool ProcessBarrierShmemResize(Barrier *barrier);
-- 
2.45.1


--vninua6xybvzgrci
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment;
	filename="v4-0008-Support-resize-for-hugetlb.patch"



^ permalink  raw  reply  [nested|flat] 4+ messages in thread

end of thread, other threads:[~2025-03-15 15:39 UTC | newest]

Thread overview: 4+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2009-02-27 12:50 Patch for not going beyond NOFILE system limit (updated) Peter Eisentraut <peter_e@gmx.net>
2022-01-15 01:25 [PATCH 2/6] Avoid the most useless instances of nulls[0]=false.. Justin Pryzby <pryzbyj@telsasoft.com>
2022-01-15 01:25 [PATCH 2/6] Avoid the most useless instances of nulls[0]=false.. Justin Pryzby <pryzbyj@telsasoft.com>
2025-03-15 15:39 [PATCH v4 7/8] Use anonymous files to back shared memory segments Dmitrii Dolgov <9erthalion6@gmail.com>

This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox