agora inbox for pgsql-hackers@postgresql.org  

help / color / mirror / Atom feed

[PATCH 2/6] Avoid the most useless instances of nulls[0]=false..
319+ messages / 2 participants
[nested] [flat]

* [PATCH 2/6] Avoid the most useless instances of nulls[0]=false..
@ 2022-01-15 01:25  Justin Pryzby <pryzbyj@telsasoft.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Justin Pryzby @ 2022-01-15 01:25 UTC (permalink / raw)

Getting rid of more of these is apparently too controvercial, but these are the
cases where 1) nulls is always being set to false; and, 2) values[] is not
being memset, so there's no parallel between the two.
---
 contrib/sslinfo/sslinfo.c                |  5 +-
 src/backend/replication/logical/origin.c | 13 +---
 src/backend/utils/adt/misc.c             |  7 +--
 src/backend/utils/misc/pg_controldata.c  | 80 ++----------------------
 4 files changed, 10 insertions(+), 95 deletions(-)

diff --git a/contrib/sslinfo/sslinfo.c b/contrib/sslinfo/sslinfo.c
index 5fd46b98741..d9edfe1b742 100644
--- a/contrib/sslinfo/sslinfo.c
+++ b/contrib/sslinfo/sslinfo.c
@@ -424,7 +424,7 @@ ssl_extension_info(PG_FUNCTION_ARGS)
 	if (call_cntr < max_calls)
 	{
 		Datum		values[3];
-		bool		nulls[3];
+		bool		nulls[3] = {0};
 		char	   *buf;
 		HeapTuple	tuple;
 		Datum		result;
@@ -453,7 +453,6 @@ ssl_extension_info(PG_FUNCTION_ARGS)
 					 errmsg("unknown OpenSSL extension in certificate at position %d",
 							call_cntr)));
 		values[0] = CStringGetTextDatum(OBJ_nid2sn(nid));
-		nulls[0] = false;
 
 		/* Get the extension value */
 		if (X509V3_EXT_print(membuf, ext, 0, 0) <= 0)
@@ -463,11 +462,9 @@ ssl_extension_info(PG_FUNCTION_ARGS)
 							call_cntr)));
 		len = BIO_get_mem_data(membuf, &buf);
 		values[1] = PointerGetDatum(cstring_to_text_with_len(buf, len));
-		nulls[1] = false;
 
 		/* Get critical status */
 		values[2] = BoolGetDatum(X509_EXTENSION_get_critical(ext));
-		nulls[2] = false;
 
 		/* Build tuple */
 		tuple = heap_form_tuple(fctx->tupdesc, values, nulls);
diff --git a/src/backend/replication/logical/origin.c b/src/backend/replication/logical/origin.c
index f134e44878f..adb3430c33a 100644
--- a/src/backend/replication/logical/origin.c
+++ b/src/backend/replication/logical/origin.c
@@ -1527,10 +1527,9 @@ pg_show_replication_origin_status(PG_FUNCTION_ARGS)
 			continue;
 
 		memset(values, 0, sizeof(values));
-		memset(nulls, 1, sizeof(nulls));
+		memset(nulls, 0, sizeof(nulls));
 
 		values[0] = ObjectIdGetDatum(state->roident);
-		nulls[0] = false;
 
 		/*
 		 * We're not preventing the origin to be dropped concurrently, so
@@ -1538,19 +1537,13 @@ pg_show_replication_origin_status(PG_FUNCTION_ARGS)
 		 */
 		if (replorigin_by_oid(state->roident, true,
 							  &roname))
-		{
 			values[1] = CStringGetTextDatum(roname);
-			nulls[1] = false;
-		}
+		else
+			nulls[1] = true;
 
 		LWLockAcquire(&state->lock, LW_SHARED);
-
 		values[2] = LSNGetDatum(state->remote_lsn);
-		nulls[2] = false;
-
 		values[3] = LSNGetDatum(state->local_lsn);
-		nulls[3] = false;
-
 		LWLockRelease(&state->lock);
 
 		tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc,
diff --git a/src/backend/utils/adt/misc.c b/src/backend/utils/adt/misc.c
index 9c132512315..611b492624a 100644
--- a/src/backend/utils/adt/misc.c
+++ b/src/backend/utils/adt/misc.c
@@ -246,7 +246,7 @@ pg_tablespace_databases(PG_FUNCTION_ARGS)
 		char	   *subdir;
 		bool		isempty;
 		Datum		values[1];
-		bool		nulls[1];
+		bool		nulls[1] = {0};
 
 		/* this test skips . and .., but is awfully weak */
 		if (!datOid)
@@ -262,7 +262,6 @@ pg_tablespace_databases(PG_FUNCTION_ARGS)
 			continue;			/* indeed, nothing in it */
 
 		values[0] = ObjectIdGetDatum(datOid);
-		nulls[0] = false;
 
 		tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc,
 							 values, nulls);
@@ -530,11 +529,9 @@ pg_get_catalog_foreign_keys(PG_FUNCTION_ARGS)
 	{
 		const SysFKRelationship *fkrel = &sys_fk_relationships[funcctx->call_cntr];
 		Datum		values[6];
-		bool		nulls[6];
+		bool		nulls[6] = {0};
 		HeapTuple	tuple;
 
-		memset(nulls, false, sizeof(nulls));
-
 		values[0] = ObjectIdGetDatum(fkrel->fk_table);
 		values[1] = FunctionCall3(arrayinp,
 								  CStringGetDatum(fkrel->fk_columns),
diff --git a/src/backend/utils/misc/pg_controldata.c b/src/backend/utils/misc/pg_controldata.c
index 781f8b87580..32be2b4b5d8 100644
--- a/src/backend/utils/misc/pg_controldata.c
+++ b/src/backend/utils/misc/pg_controldata.c
@@ -32,7 +32,7 @@ Datum
 pg_control_system(PG_FUNCTION_ARGS)
 {
 	Datum		values[4];
-	bool		nulls[4];
+	bool		nulls[4] = {0};
 	TupleDesc	tupdesc;
 	HeapTuple	htup;
 	ControlFileData *ControlFile;
@@ -60,16 +60,9 @@ pg_control_system(PG_FUNCTION_ARGS)
 				(errmsg("calculated CRC checksum does not match value stored in file")));
 
 	values[0] = Int32GetDatum(ControlFile->pg_control_version);
-	nulls[0] = false;
-
 	values[1] = Int32GetDatum(ControlFile->catalog_version_no);
-	nulls[1] = false;
-
 	values[2] = Int64GetDatum(ControlFile->system_identifier);
-	nulls[2] = false;
-
 	values[3] = TimestampTzGetDatum(time_t_to_timestamptz(ControlFile->time));
-	nulls[3] = false;
 
 	htup = heap_form_tuple(tupdesc, values, nulls);
 
@@ -80,7 +73,7 @@ Datum
 pg_control_checkpoint(PG_FUNCTION_ARGS)
 {
 	Datum		values[18];
-	bool		nulls[18];
+	bool		nulls[18] = {0};
 	TupleDesc	tupdesc;
 	HeapTuple	htup;
 	ControlFileData *ControlFile;
@@ -147,60 +140,25 @@ pg_control_checkpoint(PG_FUNCTION_ARGS)
 
 	/* Populate the values and null arrays */
 	values[0] = LSNGetDatum(ControlFile->checkPoint);
-	nulls[0] = false;
-
 	values[1] = LSNGetDatum(ControlFile->checkPointCopy.redo);
-	nulls[1] = false;
-
 	values[2] = CStringGetTextDatum(xlogfilename);
-	nulls[2] = false;
-
 	values[3] = Int32GetDatum(ControlFile->checkPointCopy.ThisTimeLineID);
-	nulls[3] = false;
-
 	values[4] = Int32GetDatum(ControlFile->checkPointCopy.PrevTimeLineID);
-	nulls[4] = false;
-
 	values[5] = BoolGetDatum(ControlFile->checkPointCopy.fullPageWrites);
-	nulls[5] = false;
-
 	values[6] = CStringGetTextDatum(psprintf("%u:%u",
 											 EpochFromFullTransactionId(ControlFile->checkPointCopy.nextXid),
 											 XidFromFullTransactionId(ControlFile->checkPointCopy.nextXid)));
-	nulls[6] = false;
-
 	values[7] = ObjectIdGetDatum(ControlFile->checkPointCopy.nextOid);
-	nulls[7] = false;
-
 	values[8] = TransactionIdGetDatum(ControlFile->checkPointCopy.nextMulti);
-	nulls[8] = false;
-
 	values[9] = TransactionIdGetDatum(ControlFile->checkPointCopy.nextMultiOffset);
-	nulls[9] = false;
-
 	values[10] = TransactionIdGetDatum(ControlFile->checkPointCopy.oldestXid);
-	nulls[10] = false;
-
 	values[11] = ObjectIdGetDatum(ControlFile->checkPointCopy.oldestXidDB);
-	nulls[11] = false;
-
 	values[12] = TransactionIdGetDatum(ControlFile->checkPointCopy.oldestActiveXid);
-	nulls[12] = false;
-
 	values[13] = TransactionIdGetDatum(ControlFile->checkPointCopy.oldestMulti);
-	nulls[13] = false;
-
 	values[14] = ObjectIdGetDatum(ControlFile->checkPointCopy.oldestMultiDB);
-	nulls[14] = false;
-
 	values[15] = TransactionIdGetDatum(ControlFile->checkPointCopy.oldestCommitTsXid);
-	nulls[15] = false;
-
 	values[16] = TransactionIdGetDatum(ControlFile->checkPointCopy.newestCommitTsXid);
-	nulls[16] = false;
-
 	values[17] = TimestampTzGetDatum(time_t_to_timestamptz(ControlFile->checkPointCopy.time));
-	nulls[17] = false;
 
 	htup = heap_form_tuple(tupdesc, values, nulls);
 
@@ -211,7 +169,7 @@ Datum
 pg_control_recovery(PG_FUNCTION_ARGS)
 {
 	Datum		values[5];
-	bool		nulls[5];
+	bool		nulls[5] = {0};
 	TupleDesc	tupdesc;
 	HeapTuple	htup;
 	ControlFileData *ControlFile;
@@ -241,19 +199,10 @@ pg_control_recovery(PG_FUNCTION_ARGS)
 				(errmsg("calculated CRC checksum does not match value stored in file")));
 
 	values[0] = LSNGetDatum(ControlFile->minRecoveryPoint);
-	nulls[0] = false;
-
 	values[1] = Int32GetDatum(ControlFile->minRecoveryPointTLI);
-	nulls[1] = false;
-
 	values[2] = LSNGetDatum(ControlFile->backupStartPoint);
-	nulls[2] = false;
-
 	values[3] = LSNGetDatum(ControlFile->backupEndPoint);
-	nulls[3] = false;
-
 	values[4] = BoolGetDatum(ControlFile->backupEndRequired);
-	nulls[4] = false;
 
 	htup = heap_form_tuple(tupdesc, values, nulls);
 
@@ -264,7 +213,7 @@ Datum
 pg_control_init(PG_FUNCTION_ARGS)
 {
 	Datum		values[11];
-	bool		nulls[11];
+	bool		nulls[11] = {0};
 	TupleDesc	tupdesc;
 	HeapTuple	htup;
 	ControlFileData *ControlFile;
@@ -306,37 +255,16 @@ pg_control_init(PG_FUNCTION_ARGS)
 				(errmsg("calculated CRC checksum does not match value stored in file")));
 
 	values[0] = Int32GetDatum(ControlFile->maxAlign);
-	nulls[0] = false;
-
 	values[1] = Int32GetDatum(ControlFile->blcksz);
-	nulls[1] = false;
-
 	values[2] = Int32GetDatum(ControlFile->relseg_size);
-	nulls[2] = false;
-
 	values[3] = Int32GetDatum(ControlFile->xlog_blcksz);
-	nulls[3] = false;
-
 	values[4] = Int32GetDatum(ControlFile->xlog_seg_size);
-	nulls[4] = false;
-
 	values[5] = Int32GetDatum(ControlFile->nameDataLen);
-	nulls[5] = false;
-
 	values[6] = Int32GetDatum(ControlFile->indexMaxKeys);
-	nulls[6] = false;
-
 	values[7] = Int32GetDatum(ControlFile->toast_max_chunk_size);
-	nulls[7] = false;
-
 	values[8] = Int32GetDatum(ControlFile->loblksize);
-	nulls[8] = false;
-
 	values[9] = BoolGetDatum(ControlFile->float8ByVal);
-	nulls[9] = false;
-
 	values[10] = Int32GetDatum(ControlFile->data_checksum_version);
-	nulls[10] = false;
 
 	htup = heap_form_tuple(tupdesc, values, nulls);
 
-- 
2.25.1


--EqVOK5mkaJAMmtSx
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
 filename="0003-selfuncs.c-refactor-parent-ACL-check.patch"



^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread

* [PATCH v1 5/5] Use anonymous files to back shared memory segments
@ 2024-10-15 14:18  Dmitrii Dolgov <9erthalion6@gmail.com>
  0 siblings, 0 replies; 319+ messages in thread

From: Dmitrii Dolgov @ 2024-10-15 14:18 UTC (permalink / raw)

Allow to use anonymous files for shared memory, instead of plain
anonymous memory. Such an anonymous file is created via memfd_create, it
lives in memory, behaves like a regular file and semantically equivalent
to an anonymous memory allocated via mmap with MAP_ANONYMOUS.

Advantages of using anon files are following:

* We've got a file descriptor, which could be used for regular file
  operations (modification, truncation, you name it).

* The file could be given a name, which improves readability when it
  comes to process maps. Here is how it looks like

7f5a2bd04000-7f5a32e52000 rw-s 00000000 00:01 1845 /memfd:strategy (deleted)
7f5a39252000-7f5a4030e000 rw-s 00000000 00:01 1842 /memfd:checkpoint (deleted)
7f5a4670e000-7f5a4d7ba000 rw-s 00000000 00:01 1839 /memfd:iocv (deleted)
7f5a53bba000-7f5a5ad26000 rw-s 00000000 00:01 1836 /memfd:descriptors (deleted)
7f5a9ad26000-7f5aa9d94000 rw-s 00000000 00:01 1833 /memfd:buffers (deleted)
7f5d29d94000-7f5d30e00000 rw-s 00000000 00:01 1830 /memfd:main (deleted)

* By default, Linux will not add file-backed shared mappings into a core dump,
  making it more convenient to work with them in PostgreSQL: no more huge dumps
  to process.

The downside is that memfd_create is Linux specific.
---
 src/backend/port/sysv_shmem.c | 47 +++++++++++++++++++++++++++++------
 src/include/portability/mem.h |  2 +-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c
index 4bdadbb0e2..a01c3e4789 100644
--- a/src/backend/port/sysv_shmem.c
+++ b/src/backend/port/sysv_shmem.c
@@ -103,6 +103,7 @@ typedef struct AnonymousMapping
 	void *shmem; 				/* Pointer to the start of the mapped memory */
 	void *seg_addr; 			/* SysV shared memory for the header */
 	unsigned long seg_id; 		/* IPC key */
+	int segment_fd; 			/* fd for the backing anon file */
 } AnonymousMapping;
 
 static AnonymousMapping Mappings[ANON_MAPPINGS];
@@ -116,7 +117,7 @@ static int next_free_slot = 0;
  * 00400000-00490000         /path/bin/postgres
  * ...
  * 012d9000-0133e000         [heap]
- * 7f443a800000-7f470a800000 /dev/zero (deleted)
+ * 7f443a800000-7f470a800000 /memfd:main (deleted)
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
  * ...
@@ -143,9 +144,9 @@ static int next_free_slot = 0;
  * The result would look like this:
  *
  * 012d9000-0133e000         [heap]
- * 7f4426f54000-7f442e010000 /dev/zero (deleted)
+ * 7f4426f54000-7f442e010000 /memfd:main (deleted)
  * [...free space...]
- * 7f443a800000-7f444196c000 /dev/zero (deleted)
+ * 7f443a800000-7f444196c000 /memfd:buffers (deleted)
  * [...free space...]
  * 7f470a800000-7f471831d000 /usr/lib/locale/locale-archive
  * 7f4718400000-7f4718401000 /usr/lib64/libicudata.so.74.2
@@ -708,6 +709,18 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 	void	   *ptr = MAP_FAILED;
 	int			mmap_errno = 0;
 
+	/*
+	 * Prepare an anonymous file backing the segment. Its size will be
+	 * specified later via ftruncate.
+	 *
+	 * The file behaves like a regular file, but lives in memory. Once all
+	 * references to the file are dropped,  it is automatically released.
+	 * Anonymous memory is used for all backing pages of the file, thus it has
+	 * the same semantics as anonymous memory allocations using mmap with the
+	 * MAP_ANONYMOUS flag.
+	 */
+	mapping->segment_fd = memfd_create(MappingName(mapping->shmem_slot), 0);
+
 #ifndef MAP_HUGETLB
 	/* PGSharedMemoryCreate should have dealt with this case */
 	Assert(huge_pages != HUGE_PAGES_ON);
@@ -725,8 +738,13 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		if (allocsize % hugepagesize != 0)
 			allocsize += hugepagesize - (allocsize % hugepagesize);
 
+		/*
+		 * Do not use an anonymous file here yet. When adding it, do not forget
+		 * to use ftruncate and flags MFD_HUGETLB & MFD_HUGE_2MB/MFD_HUGE_1GB
+		 * in memfd_create.
+		 */
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS | mmap_flags, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS | mmap_flags, -1, 0);
 		mmap_errno = errno;
 		if (huge_pages == HUGE_PAGES_TRY && ptr == MAP_FAILED)
 		{
@@ -762,7 +780,8 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 * - First create the temporary probe mapping of a fixed size and let
 		 *   kernel to place it at address of its choice. By the virtue of the
 		 *   probe mapping size we expect it to be located at the lowest
-		 *   possible address, expecting some non mapped space above.
+		 *   possible address, expecting some non mapped space above. The probe
+		 *   is does not need to be  backed by an anonymous file.
 		 *
 		 * - Unmap the probe mapping, remember the address.
 		 *
@@ -777,7 +796,7 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 *   without a restart.
 		 */
 		probe = mmap(NULL, PROBE_MAPPING_SIZE, PROT_READ | PROT_WRITE,
-				   PG_MMAP_FLAGS, -1, 0);
+				   PG_MMAP_FLAGS | MAP_ANONYMOUS, -1, 0);
 
 		if (probe == MAP_FAILED)
 		{
@@ -793,8 +812,14 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 
 			munmap(probe, PROBE_MAPPING_SIZE);
 
+			/*
+			 * Specify the segment file size using allocsize, which contains
+			 * potentially modified size.
+			 */
+			ftruncate(mapping->segment_fd, allocsize);
+
 			ptr = mmap(probe - offset, allocsize, PROT_READ | PROT_WRITE,
-					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, -1, 0);
+					   PG_MMAP_FLAGS | MAP_FIXED_NOREPLACE, mapping->segment_fd, 0);
 			mmap_errno = errno;
 			if (ptr == MAP_FAILED)
 			{
@@ -813,8 +838,11 @@ CreateAnonymousSegment(AnonymousMapping *mapping)
 		 */
 		allocsize = mapping->shmem_size;
 
+		/* Specify the segment file size using allocsize. */
+		ftruncate(mapping->segment_fd, allocsize);
+
 		ptr = mmap(NULL, allocsize, PROT_READ | PROT_WRITE,
-						   PG_MMAP_FLAGS, -1, 0);
+						   PG_MMAP_FLAGS, mapping->segment_fd, 0);
 		mmap_errno = errno;
 	}
 
@@ -903,6 +931,9 @@ AnonymousShmemResize(int newval, void *extra)
 		if (m->shmem_size == new_size)
 			continue;
 
+		/* Resize the backing anon file. */
+		ftruncate(m->segment_fd, new_size);
+
 		if (mremap(m->shmem, m->shmem_size, new_size, 0) < 0)
 			elog(LOG, "mremap(%p, %zu) failed: %m",
 				 m->shmem, m->shmem_size);
diff --git a/src/include/portability/mem.h b/src/include/portability/mem.h
index 2cd05313b8..50db0da28d 100644
--- a/src/include/portability/mem.h
+++ b/src/include/portability/mem.h
@@ -38,7 +38,7 @@
 #define MAP_NOSYNC			0
 #endif
 
-#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_ANONYMOUS|MAP_HASSEMAPHORE)
+#define PG_MMAP_FLAGS			(MAP_SHARED|MAP_HASSEMAPHORE)
 
 /* Some really old systems don't define MAP_FAILED. */
 #ifndef MAP_FAILED
-- 
2.45.1


--ers4zrhjqnhlajas--





^ permalink  raw  reply  [nested|flat] 319+ messages in thread