Deferred-scan batch freeing, trying to group arena operations

Assisted-by: LLM

malloc/malloc.c

@@ -3007,43 +3007,113 @@ tcache_free_init (void *mem)
   __libc_free (mem);
 }
 
-/* If the arena does not change between chunks, keep the lock.  */
-static inline void
-__libc_free_batched_loop (bool do_lock, mstate av, mchunkptr p, INTERNAL_SIZE_T size,
-			  tcache_perthread_struct *tc, size_t tc_idx)
+/* Adjacent tcache entries are merged locally before touching the heap,
+   so that _int_free_merge_chunk is called once for the whole block
+   instead of inserting and immediately unlinking individual chunks.
+   Used for the single-threaded path where all chunks are in
+   main_arena.  */
+static void
+__libc_free_batched_single (mchunkptr p, INTERNAL_SIZE_T size,
+			    tcache_perthread_struct *tc, size_t tc_idx)
 {
-  /* Empty half of the tcache, for a hysteresis effect.  */
   unsigned int to_free = mp_.tcache_count / 2;
-
-  if (do_lock)
-    __libc_lock_lock (av->mutex);
-
-  _int_free_merge_chunk (av, p, size);
+  INTERNAL_SIZE_T chunk_size = size;
+  mstate av = &main_arena;
 
   while (tc->entries[tc_idx] != NULL && to_free > 0)
     {
       void *mem = tcache_get_n (tc, tc_idx, &tc->entries[tc_idx], false);
-      p = mem2chunk (mem);
-      size = chunksize (p);
+      mchunkptr q = mem2chunk (mem);
 
-      /* Lock a different arena if necessary.  */
-      if (do_lock)
+      if ((char *) q + chunk_size == (char *) p)
 	{
-	  mstate chunk_av = arena_for_chunk (p);
-	  if (chunk_av != av)
-	    {
-	      __libc_lock_unlock (av->mutex);
-	      av = chunk_av;
-	      __libc_lock_lock (av->mutex);
-	    }
+	  p = q;
+	  size += chunk_size;
+	}
+      else if ((char *) p + size == (char *) q)
+	{
+	  size += chunk_size;
+	}
+      else
+	{
+	  _int_free_merge_chunk (av, p, size);
+	  p = q;
+	  size = chunk_size;
 	}
-
-      _int_free_merge_chunk (av, p, size);
       to_free--;
     }
 
-  if (do_lock)
-    __libc_lock_unlock (av->mutex);
+  _int_free_merge_chunk (av, p, size);
+}
+
+/* Multi-threaded batched free.  Pop chunks from the tcache and free
+   them to their arenas.  Chunks matching the current arena are freed
+   immediately (interleaving _int_free_merge_chunk with tcache pops to
+   hide pointer-chase latency).  Chunks from other arenas are deferred
+   to a small buffer and processed afterwards, grouped by arena.  */
+static void
+__libc_free_batched_multi (mchunkptr p, INTERNAL_SIZE_T size,
+			   tcache_perthread_struct *tc, size_t tc_idx)
+{
+  INTERNAL_SIZE_T chunk_size = size;
+  unsigned int to_free = mp_.tcache_count / 2;
+  if (to_free > 16)
+    to_free = 16;
+
+  /* Lock the trigger chunk's arena and free it.  */
+  mstate av = arena_for_chunk (p);
+  __libc_lock_lock (av->mutex);
+  _int_free_merge_chunk (av, p, chunk_size);
+
+  /* Deferred chunks from non-current arenas.  */
+  mchunkptr deferred_chunks[16];
+  mstate deferred_arenas[16];
+  unsigned int deferred_count = 0;
+
+  /* Pop from tcache.  Free same-arena chunks immediately (the
+     _int_free_merge_chunk call between pops hides the latency of
+     the next tcache pointer chase).  Defer arena mismatches.  */
+  while (tc->entries[tc_idx] != NULL && to_free > 0)
+    {
+      void *mem = tcache_get_n (tc, tc_idx, &tc->entries[tc_idx], false);
+      mchunkptr q = mem2chunk (mem);
+      mstate qav = arena_for_chunk (q);
+      if (qav == av)
+	_int_free_merge_chunk (av, q, chunk_size);
+      else
+	{
+	  deferred_chunks[deferred_count] = q;
+	  deferred_arenas[deferred_count] = qav;
+	  deferred_count++;
+	}
+      to_free--;
+    }
+
+  __libc_lock_unlock (av->mutex);
+
+  /* Process deferred chunks grouped by arena.  */
+  for (unsigned int i = 0; i < deferred_count; i++)
+    {
+      if (deferred_chunks[i] == NULL)
+	continue;
+
+      av = deferred_arenas[i];
+      __libc_lock_lock (av->mutex);
+
+      _int_free_merge_chunk (av, deferred_chunks[i], chunk_size);
+      deferred_chunks[i] = NULL;
+
+      for (unsigned int j = i + 1; j < deferred_count; j++)
+	{
+	  if (deferred_chunks[j] != NULL && deferred_arenas[j] == av)
+	    {
+	      _int_free_merge_chunk (av, deferred_chunks[j], chunk_size);
+	      deferred_chunks[j] = NULL;
+	    }
+	}
+
+      __libc_lock_unlock (av->mutex);
+    }
 }
 
 /* Deallocate half of the tcache entries into arenas, to amortize the
@@ -3058,9 +3128,9 @@ __libc_free_batched (mchunkptr p, INTERNAL_SIZE_T size,
     return malloc_printerr_tail ("free(): invalid size (batch)");
 
   if (SINGLE_THREAD_P)
-    __libc_free_batched_loop (false, &main_arena, p, size, tc, tc_idx);
+    __libc_free_batched_single (p, size, tc, tc_idx);
   else
-    __libc_free_batched_loop (true, arena_for_chunk (p), p, size, tc, tc_idx);
+    __libc_free_batched_multi (p, size, tc, tc_idx);
 }
 
 void