Compare commits

...
Author SHA1 Message Date
Yury Khrustalev d5d5f03c8f malloc: aarch64: Add MTE memory tagging
Tag memory on systems that support MTE and when memory
tagging is enabled at runtime.

Currently we implement tagging as a wrapper around core malloc
functions. User pointers are expected to be tagged and internal
malloc pointers are untagged (or rather tagged with 0 tag).

Also add tests that check logical and allocation tags.
2026-06-01 12:22:10 +01:00
Yury Khrustalev 0a91309961 malloc: aarch64: Add MTE ifunc resolvers
We introduce the ARCH_INIT_MALLOC() macro that is defined in
the sysdep headers to allow doing target-specific initialisation
in __ptmalloc_init. On AArch64 we use this macro to modify
global variables in malloc.c when MTE is enabled based on the
runtime value calculated from the glibc.mem.aarch64_mte tunable.
On all targets this macro should initialise extra_mmap_prot.

We create AArch64-specific version of the 'malloc-size.h' header
to provide custom 'memsize()' and 'checked_request2size()'. This
is required to support MTE memory tagging without the need to
check its status every time.

We modify ifunc resolvers for AArch64 to return MTE-aware versions
if malloc functions when MTE is active. At this point these *_mte
functions don't actually do any memory tagging. This will be done
in a subsequent commit.

Finally, we add the glibc.mem.aarch64_mte tunable to all malloc
tests on AArch64. On systems without MTE this will have no effect
but on systems with MTE it will make sure that tests are executed
with memory tagging enabled in sync mode.
2026-06-01 12:22:08 +01:00
Yury Khrustalev ce61edbeea aarch64: Add glibc.mem.aarch64_mte tunable
Add new tunable glibc.mem.aarch64_mte of string type with supported
values: none, sync, async, and auto. Use it to control prctl syscall
used to send the PR_SET_TAGGED_ADDR_CTRL command.

To avoid confusion, we also remove the glibc.mem.tagging tunable.

The new tunable defaults to 'none' and only works of MTE is supported.
2026-06-01 12:19:23 +01:00
Yury Khrustalev 8b2f748a86 malloc: Put malloc_chunk into separate header
Refactor malloc code to extract the definition of malloc_chunk
struct into a dedicated header  malloc-chunk.h along with basic
chunk-related macros.

This is a sysdep header.

The new header 'malloc-chunk.h' will include 'malloc-alignment.h'
that has all dependencies necessary for chunk declarations.

In turn, 'malloc-size.h' will now include 'malloc-chunk.h'. We
also move size-related definitions from malloc.c into this header.
2026-06-01 12:00:10 +01:00
Yury Khrustalev 99a6c42ad6 malloc: Put MALLOC_ALIGNMENT depends in one header
The 'malloc-alignment.h' header defines macro MALLOC_ALIGNMENT that
depends on SIZE_SZ which in turn depends on INTERNAL_SIZE_T.

These dependencies used to be defined in 'malloc-size.h'. We should
either merge these two headers or move the dependencies or change the
'malloc-alignment.h' header to make sure it is self-consistent.

This commit does the latter. In addition we fix the tst-mallocalign1.c
test that uses the MALLOC_ALIGN_MASK internal to malloc. Since it's
been moved to 'malloc-alignment.h' it makes sense to include this
header into the test source.
2026-06-01 12:00:10 +01:00
Yury Khrustalev fdfa9e3815 malloc: aarch64: Add ifuncs for malloc functions
Introduce ifuncs and resolvers for functions pertinent to the
malloc interface on the AArch64 target: malloc, calloc, free,
realloc, memalign, valloc, pvalloc, posix_memalign, aligned_alloc,
free_sized, free_aligned_sized, malloc_usable_size.

A target can define the USE_MULTIARCH_MALLOC macro. In this case
it must provide alternative aliases for the malloc functions that
point to the ifuncs.

This implementation respects the --disable-multi-arch configure
flag. If multi-arch support is disabled, the generic aliases
are used on aarch64.

This patch contains aarch64-specific resolvers. At this moment they
return core implementations but in the future they can be changed
to support for features, e.g. to handle memory tagging.
2026-06-01 10:53:56 +01:00
33 changed files with 1253 additions and 147 deletions
+7 -2
View File
@@ -22,6 +22,8 @@
#define TUNABLE_NAMESPACE malloc
#include <elf/dl-tunables.h>
#include <malloc-init.h>
/* Compile-time constants. */
#define HEAP_MIN_SIZE (32 * 1024)
@@ -248,6 +250,9 @@ static void tcache_key_initialize (void);
void
__ptmalloc_init (void)
{
/* Perform any target-specific initialisation. */
ARCH_INIT_MALLOC ();
#if USE_TCACHE
tcache_key_initialize ();
#endif
@@ -403,7 +408,7 @@ alloc_new_heap (size_t size, size_t top_pad, size_t pagesize,
}
}
}
if (__mprotect (p2, size, extra_mmap_prot | PROT_READ | PROT_WRITE) != 0)
if (__mprotect (p2, size, extra_mmap_prot) != 0)
{
__munmap (p2, max_size);
return NULL;
@@ -457,7 +462,7 @@ grow_heap (heap_info *h, long diff)
{
if (__mprotect ((char *) h + h->mprotect_size,
(unsigned long) new_size - h->mprotect_size,
extra_mmap_prot | PROT_READ | PROT_WRITE) != 0)
extra_mmap_prot) != 0)
return -2;
h->mprotect_size = new_size;
+3
View File
@@ -24,6 +24,9 @@
#include <malloc-size.h>
#include <hugepages.h>
#include <calloc-clear-memory.h>
#if defined(USE_MULTIARCH)
# include <malloc-ifuncs.h>
#endif
/* Called in the parent process before a fork. */
void __malloc_fork_lock_parent (void) attribute_hidden;
+42 -104
View File
@@ -499,6 +499,7 @@ libc_hidden_proto (__libc_free)
set to zero.
*/
void* __libc_calloc(size_t, size_t);
libc_hidden_proto (__libc_calloc)
/*
realloc(void* p, size_t n)
@@ -551,6 +552,7 @@ libc_hidden_proto (__libc_memalign)
size of the system. If the pagesize is unknown, 4096 is used.
*/
void* __libc_valloc(size_t);
libc_hidden_proto (__libc_valloc)
@@ -585,6 +587,7 @@ struct mallinfo __libc_mallinfo(void);
round up n to nearest pagesize.
*/
void* __libc_pvalloc(size_t);
libc_hidden_proto (__libc_pvalloc)
/*
malloc_trim(size_t pad);
@@ -628,6 +631,7 @@ int __malloc_trim(size_t);
*/
size_t __malloc_usable_size(void*);
libc_hidden_proto (__malloc_usable_size)
/*
malloc_stats();
@@ -657,6 +661,7 @@ void __malloc_stats(void);
POSIX wrapper like memalign(), checking for validity of size.
*/
int __posix_memalign(void **, size_t, size_t);
libc_hidden_proto (__posix_memalign)
#endif /* IS_IN (libc) */
/*
@@ -924,12 +929,10 @@ libc_hidden_proto (__libc_mallopt)
#define RETURN_ADDRESS(X_) (NULL)
#endif
/* Forward declarations. */
struct malloc_chunk;
typedef struct malloc_chunk* mchunkptr;
/* Internal routines. */
typedef struct malloc_chunk* mchunkptr;
static void* _int_malloc(mstate, size_t);
static void _int_free_chunk (mstate, mchunkptr, INTERNAL_SIZE_T, int);
static void _int_free_merge_chunk (mstate, mchunkptr, INTERNAL_SIZE_T);
@@ -976,27 +979,7 @@ static size_t musable (void *mem);
/*
This struct declaration is misleading (but accurate and necessary).
It declares a "view" into memory allowing access to necessary
fields at known offsets from a given base. See explanation below.
*/
struct malloc_chunk {
INTERNAL_SIZE_T mchunk_prev_size; /* Size of previous chunk (if free). */
INTERNAL_SIZE_T mchunk_size; /* Size in bytes, including overhead. */
struct malloc_chunk* fd; /* double links -- used only if free. */
struct malloc_chunk* bk;
/* Only used for large blocks: pointer to next larger size. */
struct malloc_chunk* fd_nextsize; /* double links -- used only if free. */
struct malloc_chunk* bk_nextsize;
};
/*
malloc_chunk details:
malloc_chunk details (see malloc-chunk.h):
(The following includes lightly edited explanations by Colin Plumb.)
@@ -1102,97 +1085,35 @@ nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
---------- Size and alignment checks and conversions ----------
*/
/* The chunk header is two SIZE_SZ elements, but this is used widely, so
we define it here for clarity later. */
#define CHUNK_HDR_SZ (2 * SIZE_SZ)
/* Convert a chunk address to a user mem pointer. */
#define chunk2mem(p) ((void*)((char*)(p) + CHUNK_HDR_SZ))
/* Convert a user mem pointer to a chunk address. */
#define mem2chunk(mem) ((mchunkptr) (((char*)(mem) - CHUNK_HDR_SZ)))
/* The smallest possible chunk */
#define MIN_CHUNK_SIZE (offsetof(struct malloc_chunk, fd_nextsize))
/* The smallest size we can malloc is an aligned minimal chunk */
#define MINSIZE \
(unsigned long)(((MIN_CHUNK_SIZE+MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK))
/* Check if m has acceptable alignment */
#define misaligned_mem(m) ((uintptr_t)(m) & MALLOC_ALIGN_MASK)
#define misaligned_chunk(p) (misaligned_mem( chunk2mem (p)))
/* pad request bytes into a usable size -- internal version */
/* Note: This must be a macro that evaluates to a compile time constant
if passed a literal constant. */
#define request2size(req) \
(((req) + SIZE_SZ + MALLOC_ALIGN_MASK < MINSIZE) ? \
MINSIZE : \
((req) + SIZE_SZ + MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK)
/* Check if REQ overflows when padded and aligned and if the resulting
value is less than PTRDIFF_T. Returns the requested size or
MINSIZE in case the value is less than MINSIZE, or SIZE_MAX if any
of the previous checks fail. */
static __always_inline size_t
checked_request2size (size_t req) __nonnull (1)
{
if (__glibc_unlikely (req > PTRDIFF_MAX))
return SIZE_MAX;
return request2size (req);
}
#define misaligned_chunk(p) (misaligned_mem (chunk2mem (p)))
/*
--------------- Physical chunk operations ---------------
*/
/* size field is or'ed with PREV_INUSE when previous adjacent chunk in use */
#define PREV_INUSE 0x1
/* extract inuse bit of previous chunk */
#define prev_inuse(p) ((p)->mchunk_size & PREV_INUSE)
/* size field is or'ed with IS_MMAPPED if the chunk was obtained with mmap() */
#define IS_MMAPPED 0x2
/* check for mmap()'ed chunk */
#define chunk_is_mmapped(p) ((p)->mchunk_size & IS_MMAPPED)
/* size field is or'ed with NON_MAIN_ARENA if the chunk was obtained
from a non-main arena. This is only set immediately before handing
the chunk to the user, if necessary. */
#define NON_MAIN_ARENA 0x4
/* Check for chunk from main arena. */
#define chunk_main_arena(p) (((p)->mchunk_size & NON_MAIN_ARENA) == 0)
/* Mark a chunk as not being on the main arena. */
#define set_non_main_arena(p) ((p)->mchunk_size |= NON_MAIN_ARENA)
/*
Bits to mask off when extracting size
Note: IS_MMAPPED is intentionally not masked off from size field in
macros for which mmapped chunks should never be seen. This should
cause helpful core dumps to occur if it is tried by accident by
people extending or adapting this malloc.
*/
#define SIZE_BITS (PREV_INUSE | IS_MMAPPED | NON_MAIN_ARENA)
/* Get size, ignoring use bits */
#define chunksize(p) (chunksize_nomask (p) & ~(SIZE_BITS))
/* Like chunksize, but do not mask SIZE_BITS. */
#define chunksize_nomask(p) ((p)->mchunk_size)
/* Ptr to next physical malloc_chunk. */
#define next_chunk(p) ((mchunkptr) (((char *) (p)) + chunksize (p)))
@@ -1243,10 +1164,6 @@ checked_request2size (size_t req) __nonnull (1)
#pragma GCC poison mchunk_size
#pragma GCC poison mchunk_prev_size
/* This is the size of the real usable data in the chunk. Not valid for
dumped heap chunks. */
#define memsize(p) (chunksize (p) - CHUNK_HDR_SZ + SIZE_SZ)
/* Huge page used for an mmap chunk. */
#define MMAP_HP 0x1
@@ -2075,9 +1992,7 @@ sysmalloc_mmap (INTERNAL_SIZE_T nb, size_t pagesize, int extra_flags)
size_t padding = MALLOC_ALIGNMENT - CHUNK_HDR_SZ;
size_t size = ALIGN_UP (nb + padding + CHUNK_HDR_SZ, pagesize);
char *mm = (char *) MMAP (NULL, size,
extra_mmap_prot | PROT_READ | PROT_WRITE,
extra_flags);
char *mm = (char *) MMAP (NULL, size, extra_mmap_prot, extra_flags);
if (mm == MAP_FAILED)
return mm;
if (extra_flags == 0)
@@ -2116,9 +2031,7 @@ sysmalloc_mmap_fallback (size_t *s, size_t size, size_t minsize,
if (size < minsize)
size = minsize;
char *mbrk = (char *) (MMAP (NULL, size,
extra_mmap_prot | PROT_READ | PROT_WRITE,
extra_flags));
char *mbrk = (char *) MMAP (NULL, size, extra_mmap_prot, extra_flags);
if (mbrk == MAP_FAILED)
return MAP_FAILED;
@@ -3320,8 +3233,11 @@ libc_hidden_def (__libc_memalign)
/* For ISO C17. */
void *
weak_function
aligned_alloc (size_t alignment, size_t bytes)
__aligned_alloc (size_t alignment, size_t bytes);
libc_hidden_proto (__aligned_alloc)
void *
__aligned_alloc (size_t alignment, size_t bytes)
{
/* Starting with ISO C17 the standard requires an error for alignments
that are not supported. Only integral powers of 2 are valid. */
@@ -3333,11 +3249,15 @@ aligned_alloc (size_t alignment, size_t bytes)
return _mid_memalign (alignment, bytes);
}
libc_hidden_def (__aligned_alloc)
/* For ISO C23. */
void
weak_function
free_sized (void *ptr, __attribute_maybe_unused__ size_t size)
__free_sized (void *ptr, __attribute_maybe_unused__ size_t size);
libc_hidden_proto (__free_sized)
void
__free_sized (void *ptr, __attribute_maybe_unused__ size_t size)
{
/* We do not perform validation that size is the same as the original
requested size at this time. We leave that to the sanitizers. We
@@ -3346,11 +3266,16 @@ free_sized (void *ptr, __attribute_maybe_unused__ size_t size)
free (ptr);
}
libc_hidden_def (__free_sized)
/* For ISO C23. */
void
weak_function
free_aligned_sized (void *ptr, __attribute_maybe_unused__ size_t alignment,
__free_aligned_sized (void *ptr, __attribute_maybe_unused__ size_t alignment,
__attribute_maybe_unused__ size_t size);
libc_hidden_proto (__free_aligned_sized)
void
__free_aligned_sized (void *ptr, __attribute_maybe_unused__ size_t alignment,
__attribute_maybe_unused__ size_t size)
{
/* We do not perform validation that size and alignment is the same as
@@ -3360,6 +3285,7 @@ free_aligned_sized (void *ptr, __attribute_maybe_unused__ size_t alignment,
free (ptr);
}
libc_hidden_def (__free_aligned_sized)
static void *
_mid_memalign (size_t alignment, size_t bytes)
@@ -3408,6 +3334,7 @@ __libc_valloc (size_t bytes)
{
return _mid_memalign (GLRO (dl_pagesize), bytes);
}
libc_hidden_def (__libc_valloc)
void *
__libc_pvalloc (size_t bytes)
@@ -3425,6 +3352,7 @@ __libc_pvalloc (size_t bytes)
return _mid_memalign (pagesize, rounded_bytes & -pagesize);
}
libc_hidden_def (__libc_pvalloc)
static void * __attribute_noinline__
__libc_calloc2 (size_t sz)
@@ -3551,6 +3479,7 @@ __libc_calloc (size_t n, size_t elem_size)
#endif
return __libc_calloc2 (bytes);
}
libc_hidden_def (__libc_calloc)
#endif /* IS_IN (libc) */
/*
@@ -4547,6 +4476,7 @@ __malloc_usable_size (void *m)
return 0;
return musable (m);
}
libc_hidden_def (__malloc_usable_size)
#endif /* IS_IN (libc) */
/*
@@ -5074,6 +5004,7 @@ __posix_memalign (void **memptr, size_t alignment, size_t size)
*memptr = mem;
return 0;
}
libc_hidden_def (__posix_memalign)
#endif /* IS_IN (libc) */
@@ -5236,6 +5167,9 @@ __malloc_info (int options, FILE *fp)
}
#if IS_IN (libc)
/* See sysdeps/generic/malloc-ifuncs.h for details. */
# if !defined (USE_MULTIARCH_MALLOC)
strong_alias (__libc_malloc, malloc)
strong_alias (__libc_realloc, realloc)
strong_alias (__libc_free, free)
@@ -5245,6 +5179,10 @@ weak_alias (__posix_memalign, posix_memalign)
weak_alias (__libc_valloc, valloc)
weak_alias (__libc_pvalloc, pvalloc)
weak_alias (__malloc_usable_size, malloc_usable_size)
weak_alias (__aligned_alloc, aligned_alloc)
weak_alias (__free_sized, free_sized)
weak_alias (__free_aligned_sized, free_aligned_sized)
#endif /* !USE_MULTIARCH_MALLOC */
weak_alias (__malloc_info, malloc_info)
weak_alias (__libc_mallinfo, mallinfo)
+1 -1
View File
@@ -19,7 +19,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <inttypes.h>
#include <malloc-size.h>
#include <malloc-alignment.h>
#include <support/check.h>
static void *
+15
View File
@@ -682,6 +682,21 @@ This tunable takes a value of 0 and 1, where 1 enables the feature.
The default value is @samp{0}, which disables the decoration.
@end deftp
@deftp Tunable glibc.mem.aarch64_mte
On AArch64 systems that support Memory Tagging Extension (MTE) this
tunable allows to select tag check fault mode (MTE mode).
Supported values are:
@itemize @bullet
@item @code{none}: (the default), memory tagging is disabled.
@item @code{auto}: enable CPU-preferred tag checking mode.
@item @code{sync}: enable synchronous tag check fault mode.
@item @code{async}: enable asynchronous tag check fault mode.
@end itemize
@end deftp
@node gmon Tunables
@section gmon Tunables
@cindex gmon tunables
+1
View File
@@ -4,6 +4,7 @@ ifeq ($(subdir),elf)
sysdep-dl-routines += \
dl-bti \
dl-gcs \
dl-mte \
# sysdep-dl-routines
tests += \
+8
View File
@@ -59,6 +59,13 @@ enum {
BTI_CHECK_ENFORCED = 1,
};
enum {
MTE_TUNABLE_NONE = 0,
MTE_TUNABLE_AUTO = 1,
MTE_TUNABLE_SYNC = 2,
MTE_TUNABLE_ASYNC = 3,
};
struct cpu_features
{
uint64_t midr_el1;
@@ -69,6 +76,7 @@ struct cpu_features
bool unused;
bool mops;
bool sve2;
bool mte;
};
#endif /* _CPU_FEATURES_AARCH64_H */
+1
View File
@@ -47,6 +47,7 @@ _dl_diagnostics_cpu (void)
print_cpu_features_value ("mops", GLRO (dl_aarch64_cpu_features).mops);
print_cpu_features_value ("sve", GLRO (dl_aarch64_cpu_features).sve);
print_cpu_features_value ("sve2", GLRO (dl_aarch64_cpu_features).sve2);
print_cpu_features_value ("mte", GLRO (dl_aarch64_cpu_features).mte);
print_cpu_features_value ("zva_size",
GLRO (dl_aarch64_cpu_features).zva_size);
+67
View File
@@ -0,0 +1,67 @@
/* AArch64 implementation for MTE (memory tagging).
Copyright (C) 2026 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#include <sys/cdefs.h>
#include <ldsodefs.h>
/* For the prctl syscall. */
#define PR_SET_TAGGED_ADDR_CTRL 55
#define PR_MTE_TAG_SHIFT 3
#define PR_TAGGED_ADDR_ENABLE (1UL << 0)
#define PR_MTE_TCF_SYNC (1UL << 1)
#define PR_MTE_TCF_ASYNC (1UL << 2)
/* The maximal set of permitted tags that the MTE random tag generation
instruction may use. We exclude tag 0 because a) we want to reserve
that for the libc heap structures and b) because it makes it easier
to see when pointer have been correctly tagged. */
#define MTE_ALLOWED_TAGS (0xfffe << PR_MTE_TAG_SHIFT)
void __mte_init (void);
rtld_hidden_proto (__mte_init)
void __mte_init (void)
{
if (!GLRO (dl_aarch64_cpu_features).mte)
return;
int mode = GL (dl_aarch64_mte);
if (mode == MTE_TUNABLE_NONE)
return;
uint64_t flags = PR_TAGGED_ADDR_ENABLE | MTE_ALLOWED_TAGS;
switch (mode)
{
case MTE_TUNABLE_AUTO:
flags |= PR_MTE_TCF_SYNC | PR_MTE_TCF_ASYNC;
break;
case MTE_TUNABLE_SYNC:
flags |= PR_MTE_TCF_SYNC;
break;
case MTE_TUNABLE_ASYNC:
flags |= PR_MTE_TCF_ASYNC;
break;
default:
_dl_fatal_printf ("unknown MTE mode: %d\n", mode);
__builtin_unreachable ();
}
/* We use inline system call to avoid unnecessary dependency
on the sys/prctl.h header. */
int r = INLINE_SYSCALL_CALL (prctl, PR_SET_TAGGED_ADDR_CTRL, flags, 0, 0, 0);
if (r == -1)
_dl_fatal_printf ("failed to enable MTE\n");
}
rtld_hidden_def (__mte_init)
+2
View File
@@ -66,6 +66,8 @@ ENTRY (_start)
cbnz w0, L(failed_gcs_lock)
L(skip_gcs_enable):
bl HIDDEN_JUMPTARGET(__mte_init)
.globl _dl_start_user
.type _dl_start_user, %function
_dl_start_user:
+5
View File
@@ -34,4 +34,9 @@ glibc {
default: 0
}
}
mem {
aarch64_mte {
type: STRING
}
}
}
+9 -2
View File
@@ -18,5 +18,12 @@ sysdep_routines += \
memset_zva64 \
strlen_asimd \
strlen_generic \
# sysdep_routines
endif
# sysdep_routines
endif # ifeq ($(subdir),string)
ifeq ($(subdir),malloc)
sysdep_routines += \
malloc-ifuncs \
malloc-mte \
# sysdep_routines
endif # ifeq ($(subdir),malloc)
+1 -1
View File
@@ -25,7 +25,7 @@
unsigned __attribute__((unused)) zva_size = \
GLRO(dl_aarch64_cpu_features).zva_size; \
bool __attribute__((unused)) bti = GLRO(dl_aarch64_cpu_features).bti; \
bool __attribute__((unused)) mte = GLRO(dl_hwcap2) & HWCAP2_MTE; \
bool __attribute__((unused)) mte = GLRO(dl_aarch64_cpu_features).mte; \
bool __attribute__((unused)) sve = GLRO(dl_aarch64_cpu_features).sve; \
bool __attribute__((unused)) sve2 = GLRO(dl_aarch64_cpu_features).sve2; \
bool __attribute__((unused)) mops = GLRO(dl_aarch64_cpu_features).mops;
+75
View File
@@ -0,0 +1,75 @@
/* Code for ifunc resolvers for malloc: aarch64 version.
Copyright (C) 2026 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#if IS_IN (libc)
#include <stdint.h>
#include <init-arch.h>
#include <malloc-ifuncs.h>
#define MTE_ACTIVE (mte && (GL (dl_aarch64_mte) != MTE_TUNABLE_NONE))
libc_ifunc_hidden (__libc_malloc, __libc_malloc_redirect,
MTE_ACTIVE ? __libc_malloc_mte : __libc_malloc)
strong_alias (__libc_malloc_redirect, malloc)
libc_ifunc_hidden (__libc_calloc, __libc_calloc_redirect,
MTE_ACTIVE ? __libc_calloc_mte : __libc_calloc)
weak_alias (__libc_calloc_redirect, calloc)
libc_ifunc_hidden (__libc_memalign, __libc_memalign_redirect,
MTE_ACTIVE ? __libc_memalign_mte : __libc_memalign)
weak_alias (__libc_memalign_redirect, memalign)
libc_ifunc_hidden (__libc_valloc, __libc_valloc_redirect,
MTE_ACTIVE ? __libc_valloc_mte : __libc_valloc)
weak_alias (__libc_valloc_redirect, valloc)
libc_ifunc_hidden (__libc_pvalloc, __libc_pvalloc_redirect,
MTE_ACTIVE ? __libc_pvalloc_mte : __libc_pvalloc)
weak_alias (__libc_pvalloc_redirect, pvalloc)
libc_ifunc_hidden (__libc_realloc, __libc_realloc_redirect,
MTE_ACTIVE ? __libc_realloc_mte : __libc_realloc)
strong_alias (__libc_realloc_redirect, realloc)
libc_ifunc_hidden (__libc_free, __libc_free_redirect,
MTE_ACTIVE ? __libc_free_mte : __libc_free)
strong_alias (__libc_free_redirect, free)
libc_ifunc_hidden (__malloc_usable_size, __malloc_usable_size_redirect,
MTE_ACTIVE ? __malloc_usable_size_mte : __malloc_usable_size)
weak_alias (__malloc_usable_size_redirect, malloc_usable_size)
libc_ifunc_hidden (__posix_memalign, __posix_memalign_redirect,
MTE_ACTIVE ? __posix_memalign_mte : __posix_memalign)
weak_alias (__posix_memalign_redirect, posix_memalign)
libc_ifunc_hidden (__aligned_alloc, __aligned_alloc_redirect,
MTE_ACTIVE ? __aligned_alloc_mte : __aligned_alloc)
weak_alias (__aligned_alloc_redirect, aligned_alloc)
libc_ifunc_hidden (__free_sized, __free_sized_redirect,
MTE_ACTIVE ? __free_sized_mte : __free_sized)
weak_alias (__free_sized_redirect, free_sized)
libc_ifunc_hidden (__free_aligned_sized, __free_aligned_sized_redirect,
MTE_ACTIVE ? __free_aligned_sized_mte : __free_aligned_sized)
weak_alias (__free_aligned_sized_redirect, free_aligned_sized)
#endif /* IS_IN (libc) */
+93
View File
@@ -0,0 +1,93 @@
/* Definitions for ifunc resolvers for malloc: aarch64 version.
Copyright (C) 2026 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#ifndef _AARCH64_MALLOC_IFUNCS_H
#define _AARCH64_MALLOC_IFUNCS_H
#define USE_MULTIARCH_MALLOC 1
#include <stddef.h>
#include <sys/cdefs.h>
/* Core implementations of malloc functions. An ifunc resolver must
use this implementations as a fallback option. Other implementations
may internally call these core function. */
void *__libc_malloc (size_t);
libc_hidden_proto (__libc_malloc)
void *__libc_calloc (size_t, size_t);
libc_hidden_proto (__libc_calloc)
void *__libc_memalign (size_t, size_t);
libc_hidden_proto (__libc_memalign)
void *__libc_valloc (size_t);
libc_hidden_proto (__libc_valloc)
void *__libc_pvalloc (size_t);
libc_hidden_proto (__libc_pvalloc)
void *__libc_realloc (void *, size_t);
libc_hidden_proto (__libc_realloc)
void __libc_free (void *);
libc_hidden_proto (__libc_free)
size_t __malloc_usable_size (void *);
libc_hidden_proto (__malloc_usable_size)
/* For additions of POSIX. */
int __posix_memalign (void **, size_t, size_t);
libc_hidden_proto (__posix_memalign)
/* For ISO C17. */
void *__aligned_alloc (size_t, size_t);
libc_hidden_proto (__aligned_alloc)
/* For ISO C23. */
void __free_sized (void *, size_t);
libc_hidden_proto (__free_sized)
void __free_aligned_sized (void *, size_t, size_t);
libc_hidden_proto (__free_aligned_sized)
/* MTE implementations of malloc functions. */
void *__libc_malloc_mte (size_t);
libc_hidden_proto (__libc_malloc_mte)
void *__libc_calloc_mte (size_t, size_t);
libc_hidden_proto (__libc_calloc_mte)
void *__libc_memalign_mte (size_t, size_t);
libc_hidden_proto (__libc_memalign_mte)
void *__libc_valloc_mte (size_t);
libc_hidden_proto (__libc_valloc_mte)
void *__libc_pvalloc_mte (size_t);
libc_hidden_proto (__libc_pvalloc_mte)
void *__libc_realloc_mte (void *, size_t);
libc_hidden_proto (__libc_realloc_mte)
void __libc_free_mte (void *);
libc_hidden_proto (__libc_free_mte)
size_t __malloc_usable_size_mte (void *);
libc_hidden_proto (__malloc_usable_size_mte)
/* For additions of POSIX: MTE version. */
int __posix_memalign_mte (void **, size_t, size_t);
libc_hidden_proto (__posix_memalign_mte)
/* For ISO C17: MTE version. */
void *__aligned_alloc_mte (size_t, size_t);
libc_hidden_proto (__aligned_alloc_mte)
/* For ISO C23: MTE version. */
void __free_sized_mte (void *, size_t);
libc_hidden_proto (__free_sized_mte)
void __free_aligned_sized_mte (void *, size_t, size_t);
libc_hidden_proto (__free_aligned_sized_mte)
#endif /* _AARCH64_MALLOC_IFUNCS_H */
+34
View File
@@ -0,0 +1,34 @@
/* Definitions for malloc init: aarch64 version.
Copyright (C) 2026 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#ifndef _AARCH64_MALLOC_INIT_H
#define _AARCH64_MALLOC_INIT_H
#define MTE_ACTIVE \
(GLRO (dl_aarch64_cpu_features).mte) && \
(GL (dl_aarch64_mte) != MTE_TUNABLE_NONE)
#define ARCH_INIT_MALLOC() \
extra_mmap_prot = PROT_READ | PROT_WRITE; \
if (MTE_ACTIVE) \
{ \
extra_mmap_prot |= PROT_MTE; \
__always_fail_morecore = true; \
}
#endif /* _AARCH64_MALLOC_INIT_H */
+176
View File
@@ -0,0 +1,176 @@
/* Implementation for MTE (memory tagging) wrappers in malloc.
Copyright (C) 2026 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
#include "aarch64-mte.h"
#include <malloc-ifuncs.h>
#include <errno.h>
#define TAG_MEM(ptr, tagfun) __glibc_unlikely (ptr == NULL) ? NULL : ({ \
size_t size = __malloc_usable_size (ptr); \
tagfun (__mte_new_tag (ptr), size); \
})
#define UNTAG_MEM(non_null_ptr) ({ \
void *untagged = __mte_clear_tag (non_null_ptr); \
size_t size = __malloc_usable_size (untagged); \
__mte_tag_region (untagged, size); \
})
void *__libc_malloc_mte (size_t bytes)
{
void *untagged = __libc_malloc (bytes);
return TAG_MEM (untagged, __mte_tag_region);
}
libc_hidden_def (__libc_malloc_mte)
void *__libc_calloc_mte (size_t n, size_t elem_size)
{
/* We use core malloc instead of calloc because we can
take advantage of MTE to zero memory region. */
void *untagged = __libc_malloc (n * elem_size);
return TAG_MEM (untagged, __mte_tag_region_zero);
}
libc_hidden_def (__libc_calloc_mte)
void *__libc_memalign_mte (size_t alignment, size_t bytes)
{
void *untagged = __libc_memalign (alignment, bytes);
return TAG_MEM (untagged, __mte_tag_region);
}
libc_hidden_def (__libc_memalign_mte)
void *__libc_valloc_mte (size_t bytes)
{
void *untagged = __libc_valloc (bytes);
return TAG_MEM (untagged, __mte_tag_region);
}
libc_hidden_def (__libc_valloc_mte)
void *__libc_pvalloc_mte (size_t bytes)
{
void *untagged = __libc_pvalloc (bytes);
return TAG_MEM (untagged, __mte_tag_region);
}
libc_hidden_def (__libc_pvalloc_mte)
/* See malloc.c for details. */
#ifndef REALLOC_ZERO_BYTES_FREES
#define REALLOC_ZERO_BYTES_FREES 1
#endif
void *__libc_realloc_mte (void *tagged_oldmem, size_t bytes)
{
/* Quick check: realloc of null is supposed to be same as malloc. */
if (tagged_oldmem == NULL)
return __libc_malloc_mte (bytes);
#if REALLOC_ZERO_BYTES_FREES
/* Quick check: realloc with 0 size is supposed to be same as free. */
if (bytes == 0)
{
__libc_free_mte (tagged_oldmem);
return NULL;
}
#endif
/* Bad size, old memory remains unchanged. */
if (bytes > PTRDIFF_MAX)
{
__set_errno (ENOMEM);
return NULL;
}
/* At this point we untag oldmem allocation. */
void *untagged_oldmem = __mte_clear_tag (tagged_oldmem);
/* Mark the chunk as belonging to the library again. */
size_t size_old = __malloc_usable_size (untagged_oldmem);
untagged_oldmem = __mte_tag_region (untagged_oldmem, size_old);
/* Call realloc core. */
void *untagged_newmem = __libc_realloc (untagged_oldmem, bytes);
if (untagged_newmem == NULL)
return NULL;
size_t size_new = __malloc_usable_size (untagged_newmem);
/* If realloc core returns old pointer, we need re-tag it. */
if (size_new == size_old && untagged_newmem == untagged_oldmem)
return __mte_tag_region (tagged_oldmem, size_new);
/* Otherwise, assign new tag. */
void *tagged_newmem = __mte_new_tag (untagged_newmem);
return __mte_tag_region (tagged_newmem, size_new);
}
libc_hidden_def (__libc_realloc_mte)
void __libc_free_mte (void *tagged)
{
if (__glibc_unlikely (tagged == NULL))
return;
/* Mark the chunk as belonging to the library again. */
void *untagged = UNTAG_MEM (tagged);
/* Call free core. */
__libc_free (untagged);
}
libc_hidden_def (__libc_free_mte)
size_t __malloc_usable_size_mte (void *tagged)
{
/* Clear only logical tag to allow accessing internal malloc
structures via offset from this pointer. */
void *untagged = __mte_clear_tag (tagged);
return __malloc_usable_size (untagged);
}
libc_hidden_def (__malloc_usable_size_mte)
int __posix_memalign_mte (void **memptr, size_t alignment, size_t size)
{
/* Call core function. */
int err = __posix_memalign (memptr, alignment, size);
if (err != 0)
return err;
*memptr = TAG_MEM (*memptr, __mte_tag_region);
return err;
}
libc_hidden_def (__posix_memalign_mte)
void *__aligned_alloc_mte (size_t alignment, size_t bytes)
{
void *untagged = __aligned_alloc (alignment, bytes);
return TAG_MEM (untagged, __mte_tag_region);
}
libc_hidden_def (__aligned_alloc_mte)
void __free_sized_mte (void *tagged, size_t size)
{
/* Mark the chunk as belonging to the library again. */
void *untagged = UNTAG_MEM (tagged);
/* Call core function. */
__free_sized (untagged, size);
}
libc_hidden_def (__free_sized_mte)
void __free_aligned_sized_mte (void *tagged, size_t alignment, size_t size)
{
/* Mark the chunk as belonging to the library again. */
void *untagged = UNTAG_MEM (tagged);
/* Call core function. */
__free_aligned_sized (untagged, alignment, size);
}
libc_hidden_def (__free_aligned_sized_mte)
+69
View File
@@ -0,0 +1,69 @@
/* Size-related definitions for malloc: aarch64 version.
Copyright (C) 2026 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#ifndef _AARCH64_MALLOC_SIZE_H
#define _AARCH64_MALLOC_SIZE_H
#include <malloc-chunk.h>
#include <sys/cdefs.h>
/* The smallest size we can malloc is an aligned minimal chunk. */
#define MINSIZE \
(unsigned long)(((MIN_CHUNK_SIZE + MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK))
/* Pad request bytes into a usable size -- internal version. Note: This must
be a macro that evaluates to a compile time constant if passed a literal
constant. */
#define request2size(req) \
(((req) + SIZE_SZ + MALLOC_ALIGN_MASK < MINSIZE) ? \
MINSIZE : \
((req) + SIZE_SZ + MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK)
/* MTE uses 16-byte granules. */
#define MTE_GRANULE_SIZE 16
/* Memory tagging is not supported with small granule. */
verify (MTE_GRANULE_SIZE > SIZE_SZ);
/* Memory tagging is not supported with large granule. */
verify (MTE_GRANULE_SIZE <= CHUNK_HDR_SZ);
/* Check if REQ overflows when padded and aligned and if the resulting
value is less than PTRDIFF_T. Returns the requested size or
MINSIZE in case the value is less than MINSIZE, or SIZE_MAX if any
of the previous checks fail. */
static __always_inline __attribute_maybe_unused__ size_t
checked_request2size (size_t req) __nonnull (1)
{
if (__glibc_unlikely (req > PTRDIFF_MAX))
return SIZE_MAX;
req = (req + (MTE_GRANULE_SIZE - 1)) & ~(size_t) (MTE_GRANULE_SIZE - 1);
return request2size (req);
}
/* Like chunksize, but do not mask SIZE_BITS. */
#define chunksize_nomask(p) ((p)->mchunk_size)
/* Get size, ignoring use bits. */
#define chunksize(p) (chunksize_nomask (p) & ~(SIZE_BITS))
/* This is the size of the real usable data in the chunk. Not valid for
dumped heap chunks. */
#define memsize(p) (chunksize (p) - CHUNK_HDR_SZ)
#endif /* _AARCH64_MALLOC_SIZE_H */
+42 -1
View File
@@ -1,4 +1,5 @@
/* Define MALLOC_ALIGNMENT for malloc. Generic version.
/* Define INTERNAL_SIZE_T and MALLOC_ALIGNMENT for malloc.
Generic version.
Copyright (C) 2017-2026 Free Software Foundation, Inc.
This file is part of the GNU C Library.
@@ -19,6 +20,44 @@
#ifndef _GENERIC_MALLOC_ALIGNMENT_H
#define _GENERIC_MALLOC_ALIGNMENT_H
#include <stddef.h>
/* INTERNAL_SIZE_T is the word-size used for internal bookkeeping of
chunk sizes.
The default version is the same as size_t.
While not strictly necessary, it is best to define this as an
unsigned type, even if size_t is a signed type. This may avoid some
artificial size limitations on some systems.
On a 64-bit machine, you may be able to reduce malloc overhead by
defining INTERNAL_SIZE_T to be a 32 bit `unsigned int' at the
expense of not being able to handle more than 2^32 of malloced
space. If this limitation is acceptable, you are encouraged to set
this unless you are on a platform requiring 16byte alignments. In
this case the alignment requirements turn out to negate any
potential advantages of decreasing size_t word size.
Implementors: Beware of the possible combinations of:
- INTERNAL_SIZE_T might be signed or unsigned, might be 32 or 64 bits,
and might be the same width as int or as long
- size_t might have different width and signedness as INTERNAL_SIZE_T
- int and long might be 32 or 64 bits, and might be the same width
To deal with this, most comparisons and difference computations
among INTERNAL_SIZE_Ts should cast them to unsigned long, being
aware of the fact that casting an unsigned int to a wider long does
not sign-extend. (This also makes checking for negative numbers
awkward.) Some of these casts result in harmless compiler warnings
on some systems. */
#ifndef INTERNAL_SIZE_T
# define INTERNAL_SIZE_T size_t
#endif
/* The corresponding word size. */
#define SIZE_SZ (sizeof (INTERNAL_SIZE_T))
/* MALLOC_ALIGNMENT is the minimum alignment for malloc'ed chunks. It
must be a power of two at least 2 * SIZE_SZ, even on machines for
which smaller alignments would suffice. It may be defined as larger
@@ -27,5 +66,7 @@
#define MALLOC_ALIGNMENT (2 * SIZE_SZ < __alignof__ (long double) \
? __alignof__ (long double) : 2 * SIZE_SZ)
/* The corresponding bit mask value. */
#define MALLOC_ALIGN_MASK (MALLOC_ALIGNMENT - 1)
#endif /* !defined(_GENERIC_MALLOC_ALIGNMENT_H) */
+70
View File
@@ -0,0 +1,70 @@
/* Malloc chunk struct: generic version.
Copyright (C) 2026 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#ifndef _GENERIC_MALLOC_CHUNK_H
#define _GENERIC_MALLOC_CHUNK_H
#include <malloc-alignment.h>
/*
This struct declaration is misleading (but accurate and necessary).
It declares a "view" into memory allowing access to necessary
fields at known offsets from a given base. See explanation below.
*/
struct malloc_chunk {
INTERNAL_SIZE_T mchunk_prev_size; /* Size of previous chunk (if free). */
INTERNAL_SIZE_T mchunk_size; /* Size in bytes, including overhead. */
struct malloc_chunk* fd; /* double links -- used only if free. */
struct malloc_chunk* bk;
/* Only used for large blocks: pointer to next larger size. */
struct malloc_chunk* fd_nextsize; /* double links -- used only if free. */
struct malloc_chunk* bk_nextsize;
};
/* The chunk header is two SIZE_SZ elements, but this is used widely, so
we define it here for clarity later. */
#define CHUNK_HDR_SZ (2 * SIZE_SZ)
/* The smallest possible chunk */
#define MIN_CHUNK_SIZE (offsetof(struct malloc_chunk, fd_nextsize))
/* size field is or'ed with PREV_INUSE when previous adjacent chunk in use. */
#define PREV_INUSE 0x1
/* size field is or'ed with IS_MMAPPED if the chunk was obtained with mmap() */
#define IS_MMAPPED 0x2
/* size field is or'ed with NON_MAIN_ARENA if the chunk was obtained
from a non-main arena. This is only set immediately before handing
the chunk to the user, if necessary. */
#define NON_MAIN_ARENA 0x4
/*
Bits to mask off when extracting size.
Note: IS_MMAPPED is intentionally not masked off from size field in
macros for which mmapped chunks should never be seen. This should
cause helpful core dumps to occur if it is tried by accident by
people extending or adapting this malloc.
*/
#define SIZE_BITS (PREV_INUSE | IS_MMAPPED | NON_MAIN_ARENA)
#endif /* _GENERIC_MALLOC_CHUNK_H */
+37
View File
@@ -0,0 +1,37 @@
/* Definitions for ifunc resolvers for malloc: generic version.
Copyright (C) 2026 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#ifndef _GENERIC_MALLOC_IFUNCS_H
#define _GENERIC_MALLOC_IFUNCS_H
/* Targets should define this macro if they provide ifuncs for
malloc functions. When USE_MULTIARCH_MALLOC is defined, the
following functions should be implemented via ifuncs:
malloc, calloc, free, realloc
memalign, valloc, pvalloc
posix_memalign
malloc_usable_size
aligned_alloc, free_sized, free_aligned_sized
*/
#if defined(USE_MULTIARCH_MALLOC) || !defined(USE_MULTIARCH)
# undef USE_MULTIARCH_MALLOC
#endif
#endif /* GENERIC_MALLOC_IFUNCS_H */
+25
View File
@@ -0,0 +1,25 @@
/* Definition for malloc init: generic version.
Copyright (C) 2026 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#ifndef _GENERIC_MALLOC_INIT_H
#define _GENERIC_MALLOC_INIT_H
#define ARCH_INIT_MALLOC() \
extra_mmap_prot = PROT_READ | PROT_WRITE;
#endif /* _GENERIC_MALLOC_INIT_H */
+32 -36
View File
@@ -1,5 +1,4 @@
/* Define INTERNAL_SIZE_T, SIZE_SZ, MALLOC_ALIGNMENT and MALLOC_ALIGN_MASK
for malloc.
/* Size-related definitions for malloc: generic version.
Copyright (C) 2021-2026 Free Software Foundation, Inc.
This file is part of the GNU C Library.
@@ -20,45 +19,42 @@
#ifndef _GENERIC_MALLOC_SIZE_H
#define _GENERIC_MALLOC_SIZE_H
/* INTERNAL_SIZE_T is the word-size used for internal bookkeeping of
chunk sizes.
#include <malloc-chunk.h>
#include <stdint.h>
#include <sys/cdefs.h>
The default version is the same as size_t.
/* The smallest size we can malloc is an aligned minimal chunk. */
#define MINSIZE \
(unsigned long)(((MIN_CHUNK_SIZE + MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK))
While not strictly necessary, it is best to define this as an
unsigned type, even if size_t is a signed type. This may avoid some
artificial size limitations on some systems.
/* Pad request bytes into a usable size -- internal version. Note: This must
be a macro that evaluates to a compile time constant if passed a literal
constant. */
#define request2size(req) \
(((req) + SIZE_SZ + MALLOC_ALIGN_MASK < MINSIZE) ? \
MINSIZE : \
((req) + SIZE_SZ + MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK)
On a 64-bit machine, you may be able to reduce malloc overhead by
defining INTERNAL_SIZE_T to be a 32 bit `unsigned int' at the
expense of not being able to handle more than 2^32 of malloced
space. If this limitation is acceptable, you are encouraged to set
this unless you are on a platform requiring 16byte alignments. In
this case the alignment requirements turn out to negate any
potential advantages of decreasing size_t word size.
/* Check if REQ overflows when padded and aligned and if the resulting
value is less than PTRDIFF_T. Returns the requested size or
MINSIZE in case the value is less than MINSIZE, or SIZE_MAX if any
of the previous checks fail. */
static __always_inline __attribute_maybe_unused__ size_t
checked_request2size (size_t req) __nonnull (1)
{
if (__glibc_unlikely (req > PTRDIFF_MAX))
return SIZE_MAX;
return request2size (req);
}
Implementors: Beware of the possible combinations of:
- INTERNAL_SIZE_T might be signed or unsigned, might be 32 or 64 bits,
and might be the same width as int or as long
- size_t might have different width and signedness as INTERNAL_SIZE_T
- int and long might be 32 or 64 bits, and might be the same width
/* Like chunksize, but do not mask SIZE_BITS. */
#define chunksize_nomask(p) ((p)->mchunk_size)
To deal with this, most comparisons and difference computations
among INTERNAL_SIZE_Ts should cast them to unsigned long, being
aware of the fact that casting an unsigned int to a wider long does
not sign-extend. (This also makes checking for negative numbers
awkward.) Some of these casts result in harmless compiler warnings
on some systems. */
#ifndef INTERNAL_SIZE_T
# define INTERNAL_SIZE_T size_t
#endif
/* Get size, ignoring use bits. */
#define chunksize(p) (chunksize_nomask (p) & ~(SIZE_BITS))
/* The corresponding word size. */
#define SIZE_SZ (sizeof (INTERNAL_SIZE_T))
#include <malloc-alignment.h>
/* The corresponding bit mask value. */
#define MALLOC_ALIGN_MASK (MALLOC_ALIGNMENT - 1)
/* This is the size of the real usable data in the chunk. Not valid for
dumped heap chunks. */
#define memsize(p) (chunksize (p) - CHUNK_HDR_SZ + SIZE_SZ)
#endif /* _GENERIC_MALLOC_SIZE_H */
+14
View File
@@ -19,6 +19,20 @@
#ifndef _I386_MALLOC_ALIGNMENT_H
#define _I386_MALLOC_ALIGNMENT_H
#include <stddef.h>
/* INTERNAL_SIZE_T is the word-size used for internal bookkeeping of
chunk sizes. See sysdeps/generic/malloc-alignment.h for details. */
#ifndef INTERNAL_SIZE_T
# define INTERNAL_SIZE_T size_t
#endif
/* The corresponding word size. */
#define SIZE_SZ (sizeof (INTERNAL_SIZE_T))
#define MALLOC_ALIGNMENT 16
/* The corresponding bit mask value. */
#define MALLOC_ALIGN_MASK (MALLOC_ALIGNMENT - 1)
#endif /* !defined(_I386_MALLOC_ALIGNMENT_H) */
+43
View File
@@ -371,6 +371,49 @@ ifeq ($(subdir),stdlib)
gen-as-const-headers += ucontext_i.sym
endif
ifeq ($(subdir),malloc)
ifneq ($(multi-arch),no)
# Special test that check MTE memory tagging
tests-for-mte += \
tst-mte-malloc \
tst-mte-malloc-static \
tst-mte-realloc \
tst-mte-realloc-static \
# tests-for-mte
tests += $(tests-for-mte)
tests-static += \
tst-mte-malloc-static \
tst-mte-realloc-static \
# tests-static
CFLAGS-tst-mte-malloc.o += -march=armv9-a+memtag
CFLAGS-tst-mte-malloc-static.o += -march=armv9-a+memtag
CFLAGS-tst-mte-realloc.o += -march=armv9-a+memtag
CFLAGS-tst-mte-realloc-static.o += -march=armv9-a+memtag
tests-exclude-malloc-check += $(tests-for-mte)
tests-exclude-mcheck += $(tests-for-mte)
tests-exclude-hugetlb1 += $(tests-for-mte)
tests-exclude-hugetlb2 += $(tests-for-mte)
endif # ifneq ($(multi-arch),no)
# Add MTE tunable to all malloc tests except malloc-check and mcheck variants
tests-malloc-mte = \
$(tests) \
$(tests-malloc-hugetlb1:%=%-malloc-hugetlb1) \
$(tests-malloc-hugetlb2:%=%-malloc-hugetlb2) \
$(tests-malloc-largetcache:%=%-malloc-largetcache) \
# tests-malloc-mte
define tests-malloc-mte-ENVS
$(1)-TUNABLES += glibc.mem.aarch64_mte=sync
endef
$(foreach t,$(tests-malloc-mte),$(eval $(call tests-malloc-mte-ENVS,$(t))))
endif # ifeq ($(subdir),malloc)
abi-variants := lp64
abi-variants += lp64_be
@@ -59,6 +59,19 @@ get_midr_from_mcpu (const struct tunable_str_t *mcpu)
return UINT64_MAX;
}
static void
TUNABLE_CALLBACK (set_aarch64_mte) (tunable_val_t *val)
{
if (tunable_strcmp_cte (val, "auto"))
GL (dl_aarch64_mte) = MTE_TUNABLE_AUTO;
else if (tunable_strcmp_cte (val, "sync"))
GL (dl_aarch64_mte) = MTE_TUNABLE_SYNC;
else if (tunable_strcmp_cte (val, "async"))
GL (dl_aarch64_mte) = MTE_TUNABLE_ASYNC;
else
GL (dl_aarch64_mte) = MTE_TUNABLE_NONE;
}
static inline void
init_cpu_features (struct cpu_features *cpu_features)
{
@@ -95,6 +108,14 @@ init_cpu_features (struct cpu_features *cpu_features)
if (cpu_features->bti)
GLRO (dl_aarch64_bti) = TUNABLE_GET (glibc, cpu, aarch64_bti, uint64_t, 0);
/* Check if MTE is supported. */
cpu_features->mte = GLRO (dl_hwcap2) & HWCAP2_MTE;
if (cpu_features->mte)
TUNABLE_GET (glibc, mem, aarch64_mte, tunable_val_t *,
TUNABLE_CALLBACK (set_aarch64_mte));
else
GL (dl_aarch64_mte) = MTE_TUNABLE_NONE;
/* Check if SVE is supported. */
cpu_features->sve = GLRO (dl_hwcap) & HWCAP_SVE;
cpu_features->sve2 = GLRO (dl_hwcap2) & HWCAP2_SVE2;
@@ -35,3 +35,19 @@ PROCINFO_CLASS unsigned long _dl_aarch64_gcs
,
# endif
#endif
#if !IS_IN (ldconfig)
# if !defined PROCINFO_DECL && defined SHARED
._dl_aarch64_mte
# else
PROCINFO_CLASS int _dl_aarch64_mte
# endif
# ifndef PROCINFO_DECL
= 0
# endif
# if !defined SHARED || defined PROCINFO_DECL
;
# else
,
# endif
#endif
@@ -36,6 +36,9 @@
# define GCS_POLICY_OPTIONAL 2
# endif
void __mte_init (void);
rtld_hidden_proto (__mte_init)
/* Must be on a top-level stack frame that does not return. */
static inline void __attribute__((always_inline))
aarch64_libc_setup_tls (void)
@@ -72,6 +75,9 @@ aarch64_libc_setup_tls (void)
_dl_fatal_printf ("failed to lock GCS: %d\n", -ret);
}
}
__mte_init ();
}
# define ARCH_SETUP_IREL() apply_irel ()
@@ -0,0 +1,110 @@
/* AArch64 test helper functions for MTE.
Copyright (C) 2026 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#ifndef TST_MTE_HELPER_H
#define TST_MTE_HELPER_H
#include <support/check.h>
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <malloc.h>
#include <sys/auxv.h>
#include <sys/prctl.h>
#define GRANULE_SIZE 16
/* Extract logical tag from pointer PTR. */
static __always_inline
uint64_t get_logical_tag (const void *ptr)
{
uint64_t t = (uint64_t)ptr;
return t >> 56ul & 0xf;
}
/* Load allocation tag from memory pointed-to by the PTR pointer. */
static __always_inline
uint64_t get_allocation_tag (const void *ptr)
{
uint64_t t;
asm volatile ("ldg %0, [%1]" : "=r" (t) : "r" (ptr));
return t >> 56ul & 0xf;
}
/* Read the Tag Check Override bit. */
static __always_inline
uint64_t get_pstate_tco (void) {
uint64_t t;
asm volatile ("mrs %0, tco" : "=r" (t));
return t;
}
static __always_inline
bool check_tags (void *tm)
{
size_t len = malloc_usable_size (tm);
TEST_VERIFY (len % GRANULE_SIZE == 0);
uint64_t ltag = get_logical_tag (tm);
TEST_VERIFY (ltag != 0);
for (size_t offset = 0; offset < len; offset += GRANULE_SIZE)
{
const char *g = (char *)tm + offset;
uint64_t atag = get_allocation_tag (g);
TEST_COMPARE (ltag, atag);
if (ltag != atag)
{
printf ("tagged ptr: %p usable size: %zu\n", tm, len);
printf ("tags mismatch at offset %zu: logical=%lu, allocation=%lu\n",
offset, ltag, atag);
return false;
}
}
return ltag != 0;
}
static __always_inline
void check_mte_enabled (void)
{
/* Check if MTE is supported. */
if (!(getauxval (AT_HWCAP2) & HWCAP2_MTE))
FAIL_UNSUPPORTED ("kernel or CPU does not support HWCAP2_MTE");
/* Check if Tag Check Override bit is set. */
if (get_pstate_tco () != 0)
FAIL_UNSUPPORTED ("MTE tag check override is enabled");
/* Check applied MTE params. */
uint64_t x = (uint64_t) prctl (PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0);
uint64_t status = (x & 1ul);
uint64_t mode = (x & PR_MTE_TCF_MASK) >> PR_MTE_TCF_SHIFT;
uint64_t tags = (x & PR_MTE_TAG_MASK) >> PR_MTE_TAG_SHIFT;
printf ("MTE status: %4lx\n", status);
printf ("MTE mode: %4lx\n", mode);
printf ("MTE tags: %4lx\n", tags);
/* This test should be run in sync mode for tag checks. */
TEST_VERIFY (status == 1);
TEST_VERIFY (mode == PR_MTE_TCF_SYNC >> PR_MTE_TCF_SHIFT);
TEST_VERIFY (tags == 0xfffe);
}
#endif // TST_MTE_HELPER_H
@@ -0,0 +1 @@
#include "tst-mte-malloc.c"
@@ -0,0 +1,154 @@
/* AArch64 tests for heap memory tagging.
Copyright (C) 2026 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#include <support/check.h>
#include <support/support.h>
#include <support/xsignal.h>
#include <support/test-driver.h>
#include <array_length.h>
#include "tst-mte-helper.h"
/* Characteristic malloc sizes to cover various allocation methods. */
size_t sizes[] = {
1,
16, 40, 64, 120,
128, 500, 1000,
1050, 4096, 5000, 65000,
131072, 2000000
};
static void check_malloc (size_t len)
{
printf ("testing malloc with req size %zu\n", len);
void *tm = malloc (len);
if (!check_tags (tm))
printf ("tagged pointer?: %016lx\n", (uintptr_t)tm);
free (tm);
}
static void check_calloc (size_t len)
{
size_t num = len / sizeof (uint64_t) + 1;
printf ("testing calloc with req size %zu\n", num * sizeof (uint64_t));
uint64_t *tm = calloc (num, sizeof (uint64_t));
if (check_tags (tm))
for (int n = 0; n < num; n ++)
TEST_VERIFY_EXIT (tm[n] == 0);
else
printf ("tagged pointer?: %016lx\n", (uintptr_t)tm);
free (tm);
}
static void check_memalign (size_t len, size_t alignment)
{
printf ("testing memalign(%zu) with req size %zu\n", alignment, len);
void *tm = memalign (alignment, len);
if (!check_tags (tm))
printf ("tagged pointer?: %016lx\n", (uintptr_t)tm);
free (tm);
}
static void check_valloc (size_t len)
{
printf ("testing valloc with req size %zu\n", len);
void *tm = valloc (len);
if (!check_tags (tm))
printf ("tagged pointer?: %016lx\n", (uintptr_t)tm);
free_sized (tm, len);
}
static void check_pvalloc (size_t len)
{
printf ("testing pvalloc with req size %zu\n", len);
void *tm = pvalloc (len);
if (!check_tags (tm))
printf ("tagged pointer?: %016lx\n", (uintptr_t)tm);
free_sized (tm, len);
}
static void check_posix_memalign (size_t len, size_t alignment)
{
printf ("testing posix_memalign(%zu) with req size %zu\n", alignment, len);
void *p = NULL;
int err = posix_memalign (&p, alignment, len);
if (err)
perror ("posix_memalign");
TEST_VERIFY (p != NULL);
TEST_VERIFY (err == 0);
if (!check_tags (p))
printf ("tagged pointer?: %016lx\n", (uintptr_t)p);
free_aligned_sized (p, alignment, len);
}
static void check_aligned_alloc (size_t len, size_t alignment)
{
printf ("testing aligned_alloc(%zu) with req size %zu\n", alignment, len);
void *tm = aligned_alloc (alignment, len);
if (!check_tags (tm))
printf ("tagged pointer?: %016lx\n", (uintptr_t)tm);
free_aligned_sized (tm, alignment, len);
}
static int
do_test (void)
{
/* Check if MTE is supported, configured and enabled. */
check_mte_enabled ();
array_foreach_const (plen, sizes)
check_malloc (*plen);
array_foreach_const (plen, sizes)
check_calloc (*plen);
array_foreach_const (plen, sizes)
{
check_memalign (*plen, 2);
check_memalign (*plen, 4);
check_memalign (*plen, 8);
check_memalign (*plen, 16);
check_memalign (*plen, 32);
}
array_foreach_const (plen, sizes)
check_valloc (*plen);
array_foreach_const (plen, sizes)
check_pvalloc (*plen);
array_foreach_const (plen, sizes)
{
check_posix_memalign (*plen, sizeof (void *) * 1);
check_posix_memalign (*plen, sizeof (void *) * 2);
check_posix_memalign (*plen, sizeof (void *) * 4);
}
array_foreach_const (plen, sizes)
{
check_aligned_alloc (*plen, 2);
check_aligned_alloc (*plen, 4);
check_aligned_alloc (*plen, 8);
check_aligned_alloc (*plen, 16);
check_aligned_alloc (*plen, 32);
}
return 0;
}
#include <support/test-driver.c>
@@ -0,0 +1 @@
#include "tst-mte-realloc.c"
@@ -0,0 +1,72 @@
/* AArch64 tests for heap memory tagging.
Copyright (C) 2026 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#include <support/check.h>
#include <support/support.h>
#include <support/xsignal.h>
#include <support/test-driver.h>
#include <array_length.h>
#include "tst-mte-helper.h"
/* Characteristic malloc sizes to cover various allocation methods. */
size_t sizes[] = {
1,
16, 40, 64, 120,
128, 500, 1000,
1050, 4096, 5000, 65000,
131072, 2000000
};
static void check_realloc (size_t len)
{
/* Tagged pointers. */
void *tm, *new_tm;
printf ("testing realloc (NULL) for req size %zu\n", len);
tm = realloc (NULL, len);
check_tags (tm);
/* Reduce size. */
printf ("testing realloc (decreased size) for req size %zu\n", len);
new_tm = realloc (tm, len / 2 + 1);
check_tags (new_tm);
/* Increase size. */
printf ("testing realloc (increased size) for req size %zu\n", len);
new_tm = realloc (new_tm, len + 2);
check_tags (new_tm);
free (new_tm);
}
static int
do_test (void)
{
/* Check if MTE is supported, configured and enabled. */
check_mte_enabled ();
array_foreach_const (plen, sizes)
check_realloc (*plen);
return 0;
}
#include <support/test-driver.c>