Revert "malloc: Remove dynamic mmap/trim threshold [BZ #30769]"

This change causes large degradations on multiple SPEC benchmarks,
so we have to revert this commit.

GLIBC BZ: https://sourceware.org/PR34394

This reverts commit 17a79a5120

Signed-off-by: Peter Bergner <bergner@tenstorrent.com>
Reviewed-by: Adhemerval Zanella  <adhemerval.zanella@linaro.org>

malloc/malloc.c

@@ -1612,6 +1612,10 @@ struct malloc_par
   int n_mmaps;
   int n_mmaps_max;
   int max_n_mmaps;
+  /* the mmap_threshold is dynamic, until the user sets
+     it manually, at which point we need to disable any
+     dynamic behavior. */
+  int no_dyn_threshold;
 
   /* Statistics */
   INTERNAL_SIZE_T mmapped_mem;
@@ -4075,6 +4079,18 @@ _int_free_chunk (mstate av, mchunkptr p, INTERNAL_SIZE_T size)
     /* Preserve errno in case munmap sets it.  */
     int err = errno;
 
+    /* See if the dynamic brk/mmap threshold needs adjusting.
+       Dumped fake mmapped chunks do not affect the threshold.  */
+    if (!mp_.no_dyn_threshold
+        && chunksize_nomask (p) > mp_.mmap_threshold
+        && chunksize_nomask (p) <= DEFAULT_MMAP_THRESHOLD_MAX)
+      {
+        mp_.mmap_threshold = chunksize (p);
+        mp_.trim_threshold = 2 * mp_.mmap_threshold;
+        LIBC_PROBE (memory_mallopt_free_dyn_thresholds, 2,
+		    mp_.mmap_threshold, mp_.trim_threshold);
+      }
+
     munmap_chunk (p);
 
     __set_errno (err);
@@ -4673,32 +4689,40 @@ __malloc_stats (void)
 static __always_inline int
 do_set_trim_threshold (size_t value)
 {
-  LIBC_PROBE (memory_mallopt_trim_threshold, 2, value, mp_.trim_threshold);
+  LIBC_PROBE (memory_mallopt_trim_threshold, 3, value, mp_.trim_threshold,
+	      mp_.no_dyn_threshold);
   mp_.trim_threshold = value;
+  mp_.no_dyn_threshold = 1;
   return 1;
 }
 
 static __always_inline int
 do_set_top_pad (size_t value)
 {
-  LIBC_PROBE (memory_mallopt_top_pad, 2, value, mp_.top_pad);
+  LIBC_PROBE (memory_mallopt_top_pad, 3, value, mp_.top_pad,
+	      mp_.no_dyn_threshold);
   mp_.top_pad = value;
+  mp_.no_dyn_threshold = 1;
   return 1;
 }
 
 static __always_inline int
 do_set_mmap_threshold (size_t value)
 {
-  LIBC_PROBE (memory_mallopt_mmap_threshold, 2, value, mp_.mmap_threshold);
+  LIBC_PROBE (memory_mallopt_mmap_threshold, 3, value, mp_.mmap_threshold,
+	      mp_.no_dyn_threshold);
   mp_.mmap_threshold = value;
+  mp_.no_dyn_threshold = 1;
   return 1;
 }
 
 static __always_inline int
 do_set_mmaps_max (int32_t value)
 {
-  LIBC_PROBE (memory_mallopt_mmap_max, 2, value, mp_.n_mmaps_max);
+  LIBC_PROBE (memory_mallopt_mmap_max, 3, value, mp_.n_mmaps_max,
+	      mp_.no_dyn_threshold);
   mp_.n_mmaps_max = value;
+  mp_.no_dyn_threshold = 1;
   return 1;
 }
 

manual/memory.texi

@@ -1333,7 +1333,10 @@ that the memory for these chunks can be returned to the system on
 @code{free}.  Note that requests smaller than this threshold might still
 be allocated via @code{mmap}.
 
-If this parameter is not set, the default value is set as 128 KiB.
+If this parameter is not set, the default value is set as 128 KiB and the
+threshold is adjusted dynamically to suit the allocation patterns of the
+program. If the parameter is set, the dynamic adjustment is disabled and the
+value is set statically to the input value.
 
 This parameter can also be set for the process at startup by setting the
 environment variable @env{MALLOC_MMAP_THRESHOLD_} to the desired value.
@@ -1368,7 +1371,10 @@ environment variable @env{MALLOC_TOP_PAD_} to the desired value.
 This is the minimum size (in bytes) of the top-most, releasable chunk
 that will trigger a system call in order to return memory to the system.
 
-If this parameter is not set, the default value is set as 128 KiB.
+If this parameter is not set, the default value is set as 128 KiB and the
+threshold is adjusted dynamically to suit the allocation patterns of the
+program. If the parameter is set, the dynamic adjustment is disabled and the
+value is set statically to the provided input.
 
 This parameter can also be set for the process at startup by setting the
 environment variable @env{MALLOC_TRIM_THRESHOLD_} to the desired value.

manual/probes.texi

@@ -161,33 +161,37 @@ value, and @var{$arg2} is the previous value of this @code{malloc}
 parameter.
 @end deftp
 
-@deftp Probe memory_mallopt_trim_threshold (int @var{$arg1}, int @var{$arg2})
+@deftp Probe memory_mallopt_trim_threshold (int @var{$arg1}, int @var{$arg2}, int @var{$arg3})
 This probe is triggered shortly after the @code{memory_mallopt} probe,
 when the parameter to be changed is @code{M_TRIM_THRESHOLD}.  Argument
 @var{$arg1} is the requested value, @var{$arg2} is the previous value of
-this @code{malloc} parameter.
+this @code{malloc} parameter, and @var{$arg3} is nonzero if dynamic
+threshold adjustment was already disabled.
 @end deftp
 
-@deftp Probe memory_mallopt_top_pad (int @var{$arg1}, int @var{$arg2})
+@deftp Probe memory_mallopt_top_pad (int @var{$arg1}, int @var{$arg2}, int @var{$arg3})
 This probe is triggered shortly after the @code{memory_mallopt} probe,
 when the parameter to be changed is @code{M_TOP_PAD}.  Argument
 @var{$arg1} is the requested value, @var{$arg2} is the previous value of
-this @code{malloc} parameter.
+this @code{malloc} parameter, and @var{$arg3} is nonzero if dynamic
+threshold adjustment was already disabled.
 @end deftp
 
-@deftp Probe memory_mallopt_mmap_threshold (int @var{$arg1}, int @var{$arg2})
+@deftp Probe memory_mallopt_mmap_threshold (int @var{$arg1}, int @var{$arg2}, int @var{$arg3})
 This probe is triggered shortly after the @code{memory_mallopt} probe,
 when the parameter to be changed is @code{M_MMAP_THRESHOLD}, and the
 requested value is in an acceptable range.  Argument @var{$arg1} is the
 requested value, @var{$arg2} is the previous value of this @code{malloc}
-parameter.
+parameter, and @var{$arg3} is nonzero if dynamic threshold adjustment
+was already disabled.
 @end deftp
 
-@deftp Probe memory_mallopt_mmap_max (int @var{$arg1}, int @var{$arg2})
+@deftp Probe memory_mallopt_mmap_max (int @var{$arg1}, int @var{$arg2}, int @var{$arg3})
 This probe is triggered shortly after the @code{memory_mallopt} probe,
 when the parameter to be changed is @code{M_MMAP_MAX}.  Argument
 @var{$arg1} is the requested value, @var{$arg2} is the previous value of
-this @code{malloc} parameter.
+this @code{malloc} parameter, and @var{$arg3} is nonzero if dynamic
+threshold adjustment was already disabled.
 @end deftp
 
 @deftp Probe memory_mallopt_perturb (int @var{$arg1}, int @var{$arg2})
@@ -213,6 +217,12 @@ requested value, and @var{$arg2} is the previous value of this
 @code{malloc} parameter.
 @end deftp
 
+@deftp Probe memory_mallopt_free_dyn_thresholds (int @var{$arg1}, int @var{$arg2})
+This probe is triggered when function @code{free} decides to adjust the
+dynamic brk/mmap thresholds.  Argument @var{$arg1} and @var{$arg2} are
+the adjusted mmap and trim thresholds, respectively.
+@end deftp
+
 @deftp Probe memory_tunable_tcache_max_bytes (int @var{$arg1}, int @var{$arg2})
 This probe is triggered when the @code{glibc.malloc.tcache_max}
 tunable is set.  Argument @var{$arg1} is the requested value, and