pgvector.git / summary / log / commit / refs

commit    8ee37b60a02c83afd22c3da17eb6669b448ce2c0
Author:   Andrew Kane <andrew@ankane.org>
Date:     Wed Jan 03 18:47:50 2024 +0000

    Improved memory estimate for HNSW index builds


 src/hnsw.h      |  1 -
 src/hnswbuild.c | 23 +++++++----------------
 src/hnswutils.c | 23 -----------------------
 3 files changed, 7 insertions(+), 40 deletions(-)

diff --git a/src/hnsw.h b/src/hnsw.h
index 4dba0282..66624919 100644
--- a/src/hnsw.h
+++ b/src/hnsw.h
@@ -330,7 +330,6 @@ List	   *HnswSearchLayer(Datum q, List *ep, int ef, int lc, Relation index, Fmgr
 HnswElement HnswGetEntryPoint(Relation index);
 void		HnswGetMetaPageInfo(Relation index, int *m, HnswElement * entryPoint);
 HnswElement HnswInitElement(ItemPointer tid, int m, double ml, int maxLevel);
-void		HnswFreeElement(HnswElement element);
 HnswElement HnswInitElementFromBlock(BlockNumber blkno, OffsetNumber offno);
 void		HnswInsertElement(HnswElement element, HnswElement entryPoint, Relation index, FmgrInfo *procinfo, Oid collation, int m, int efConstruction, bool existing);
 HnswElement HnswFindDuplicate(HnswElement e);
diff --git a/src/hnswbuild.c b/src/hnswbuild.c
index 8b053d3a..b40c3150 100644
--- a/src/hnswbuild.c
+++ b/src/hnswbuild.c
@@ -310,11 +310,9 @@ HnswElementMemory(HnswElement e, int m)
 
 	elementSize += sizeof(HnswNeighborArray) * (e->level + 1);
 	elementSize += sizeof(HnswCandidate) * (m * (e->level + 2));
-	elementSize += sizeof(List);
-	elementSize += sizeof(ItemPointerData) + SIZEOF_VOID_P;
 	elementSize += VARSIZE_ANY(DatumGetPointer(e->value));
 	/* Each allocation has a chunk header */
-	elementSize += (e->level + 7) * GENERATIONCHUNK_RAWSIZE;
+	elementSize += (e->level + 4) * GENERATIONCHUNK_RAWSIZE;
 	/* Add an extra 5% for alignment and other overhead */
 	return elementSize * 1.05;
 }
@@ -379,27 +377,20 @@ InsertTupleInMemory(Relation index, Datum *values, ItemPointer heaptid, HnswBuil
 
 	if (dup != NULL)
 	{
+		/* No need to free element since memory unlikely to be reallocated */
 		HnswAddHeapTid(dup, heaptid);
-		HnswFreeElement(element);
-
-#if PG_VERSION_NUM >= 130000
-		buildstate->memoryUsed = MemoryContextMemAllocated(buildstate->graphCtx, false);
-#else
-		buildstate->memoryUsed += sizeof(ItemPointerData) + SIZEOF_VOID_P + GENERATIONCHUNK_RAWSIZE;
-#endif
 	}
 	else
-	{
 		buildstate->elements = lappend(buildstate->elements, element);
 
+	MemoryContextSwitchTo(oldCtx);
+
+	/* Update memory usage */
 #if PG_VERSION_NUM >= 130000
-		buildstate->memoryUsed = MemoryContextMemAllocated(buildstate->graphCtx, false);
+	buildstate->memoryUsed = MemoryContextMemAllocated(buildstate->graphCtx, false);
 #else
-		buildstate->memoryUsed += HnswElementMemory(element, buildstate->m);
+	buildstate->memoryUsed += HnswElementMemory(element, buildstate->m);
 #endif
-	}
-
-	MemoryContextSwitchTo(oldCtx);
 
 	return true;
 }
diff --git a/src/hnswutils.c b/src/hnswutils.c
index 57b89777..192fee5a 100644
--- a/src/hnswutils.c
+++ b/src/hnswutils.c
@@ -206,17 +206,6 @@ HnswInitNeighbors(HnswElement element, int m)
 	}
 }
 
-/*
- * Free neighbors
- */
-static void
-HnswFreeNeighbors(HnswElement element)
-{
-	for (int lc = 0; lc <= element->level; lc++)
-		pfree(element->neighbors[lc].items);
-	pfree(element->neighbors);
-}
-
 /*
  * Allocate an element
  */
@@ -244,18 +233,6 @@ HnswInitElement(ItemPointer heaptid, int m, double ml, int maxLevel)
 	return element;
 }
 
-/*
- * Free an element
- */
-void
-HnswFreeElement(HnswElement element)
-{
-	HnswFreeNeighbors(element);
-	if (DatumGetPointer(element->value))
-		pfree(DatumGetPointer(element->value));
-	pfree(element);
-}
-
 /*
  * Add a heap TID to an element
  */


[parent: 9b73b3d1a652]