mirror of
git://sourceware.org/git/glibc.git
synced 2026-09-09 11:58:23 +08:00
Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d5d5f03c8f | ||
|
|
0a91309961 | ||
|
|
ce61edbeea | ||
|
|
8b2f748a86 | ||
|
|
99a6c42ad6 | ||
|
|
fdfa9e3815 |
+7
-2
@@ -22,6 +22,8 @@
|
||||
#define TUNABLE_NAMESPACE malloc
|
||||
#include <elf/dl-tunables.h>
|
||||
|
||||
#include <malloc-init.h>
|
||||
|
||||
/* Compile-time constants. */
|
||||
|
||||
#define HEAP_MIN_SIZE (32 * 1024)
|
||||
@@ -248,6 +250,9 @@ static void tcache_key_initialize (void);
|
||||
void
|
||||
__ptmalloc_init (void)
|
||||
{
|
||||
/* Perform any target-specific initialisation. */
|
||||
ARCH_INIT_MALLOC ();
|
||||
|
||||
#if USE_TCACHE
|
||||
tcache_key_initialize ();
|
||||
#endif
|
||||
@@ -403,7 +408,7 @@ alloc_new_heap (size_t size, size_t top_pad, size_t pagesize,
|
||||
}
|
||||
}
|
||||
}
|
||||
if (__mprotect (p2, size, extra_mmap_prot | PROT_READ | PROT_WRITE) != 0)
|
||||
if (__mprotect (p2, size, extra_mmap_prot) != 0)
|
||||
{
|
||||
__munmap (p2, max_size);
|
||||
return NULL;
|
||||
@@ -457,7 +462,7 @@ grow_heap (heap_info *h, long diff)
|
||||
{
|
||||
if (__mprotect ((char *) h + h->mprotect_size,
|
||||
(unsigned long) new_size - h->mprotect_size,
|
||||
extra_mmap_prot | PROT_READ | PROT_WRITE) != 0)
|
||||
extra_mmap_prot) != 0)
|
||||
return -2;
|
||||
|
||||
h->mprotect_size = new_size;
|
||||
|
||||
@@ -24,6 +24,9 @@
|
||||
#include <malloc-size.h>
|
||||
#include <hugepages.h>
|
||||
#include <calloc-clear-memory.h>
|
||||
#if defined(USE_MULTIARCH)
|
||||
# include <malloc-ifuncs.h>
|
||||
#endif
|
||||
|
||||
/* Called in the parent process before a fork. */
|
||||
void __malloc_fork_lock_parent (void) attribute_hidden;
|
||||
|
||||
+42
-104
@@ -499,6 +499,7 @@ libc_hidden_proto (__libc_free)
|
||||
set to zero.
|
||||
*/
|
||||
void* __libc_calloc(size_t, size_t);
|
||||
libc_hidden_proto (__libc_calloc)
|
||||
|
||||
/*
|
||||
realloc(void* p, size_t n)
|
||||
@@ -551,6 +552,7 @@ libc_hidden_proto (__libc_memalign)
|
||||
size of the system. If the pagesize is unknown, 4096 is used.
|
||||
*/
|
||||
void* __libc_valloc(size_t);
|
||||
libc_hidden_proto (__libc_valloc)
|
||||
|
||||
|
||||
|
||||
@@ -585,6 +587,7 @@ struct mallinfo __libc_mallinfo(void);
|
||||
round up n to nearest pagesize.
|
||||
*/
|
||||
void* __libc_pvalloc(size_t);
|
||||
libc_hidden_proto (__libc_pvalloc)
|
||||
|
||||
/*
|
||||
malloc_trim(size_t pad);
|
||||
@@ -628,6 +631,7 @@ int __malloc_trim(size_t);
|
||||
|
||||
*/
|
||||
size_t __malloc_usable_size(void*);
|
||||
libc_hidden_proto (__malloc_usable_size)
|
||||
|
||||
/*
|
||||
malloc_stats();
|
||||
@@ -657,6 +661,7 @@ void __malloc_stats(void);
|
||||
POSIX wrapper like memalign(), checking for validity of size.
|
||||
*/
|
||||
int __posix_memalign(void **, size_t, size_t);
|
||||
libc_hidden_proto (__posix_memalign)
|
||||
#endif /* IS_IN (libc) */
|
||||
|
||||
/*
|
||||
@@ -924,12 +929,10 @@ libc_hidden_proto (__libc_mallopt)
|
||||
#define RETURN_ADDRESS(X_) (NULL)
|
||||
#endif
|
||||
|
||||
/* Forward declarations. */
|
||||
struct malloc_chunk;
|
||||
typedef struct malloc_chunk* mchunkptr;
|
||||
|
||||
/* Internal routines. */
|
||||
|
||||
typedef struct malloc_chunk* mchunkptr;
|
||||
|
||||
static void* _int_malloc(mstate, size_t);
|
||||
static void _int_free_chunk (mstate, mchunkptr, INTERNAL_SIZE_T, int);
|
||||
static void _int_free_merge_chunk (mstate, mchunkptr, INTERNAL_SIZE_T);
|
||||
@@ -976,27 +979,7 @@ static size_t musable (void *mem);
|
||||
|
||||
|
||||
/*
|
||||
This struct declaration is misleading (but accurate and necessary).
|
||||
It declares a "view" into memory allowing access to necessary
|
||||
fields at known offsets from a given base. See explanation below.
|
||||
*/
|
||||
|
||||
struct malloc_chunk {
|
||||
|
||||
INTERNAL_SIZE_T mchunk_prev_size; /* Size of previous chunk (if free). */
|
||||
INTERNAL_SIZE_T mchunk_size; /* Size in bytes, including overhead. */
|
||||
|
||||
struct malloc_chunk* fd; /* double links -- used only if free. */
|
||||
struct malloc_chunk* bk;
|
||||
|
||||
/* Only used for large blocks: pointer to next larger size. */
|
||||
struct malloc_chunk* fd_nextsize; /* double links -- used only if free. */
|
||||
struct malloc_chunk* bk_nextsize;
|
||||
};
|
||||
|
||||
|
||||
/*
|
||||
malloc_chunk details:
|
||||
malloc_chunk details (see malloc-chunk.h):
|
||||
|
||||
(The following includes lightly edited explanations by Colin Plumb.)
|
||||
|
||||
@@ -1102,97 +1085,35 @@ nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
||||
---------- Size and alignment checks and conversions ----------
|
||||
*/
|
||||
|
||||
/* The chunk header is two SIZE_SZ elements, but this is used widely, so
|
||||
we define it here for clarity later. */
|
||||
#define CHUNK_HDR_SZ (2 * SIZE_SZ)
|
||||
|
||||
/* Convert a chunk address to a user mem pointer. */
|
||||
#define chunk2mem(p) ((void*)((char*)(p) + CHUNK_HDR_SZ))
|
||||
|
||||
/* Convert a user mem pointer to a chunk address. */
|
||||
#define mem2chunk(mem) ((mchunkptr) (((char*)(mem) - CHUNK_HDR_SZ)))
|
||||
|
||||
/* The smallest possible chunk */
|
||||
#define MIN_CHUNK_SIZE (offsetof(struct malloc_chunk, fd_nextsize))
|
||||
|
||||
/* The smallest size we can malloc is an aligned minimal chunk */
|
||||
|
||||
#define MINSIZE \
|
||||
(unsigned long)(((MIN_CHUNK_SIZE+MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK))
|
||||
|
||||
/* Check if m has acceptable alignment */
|
||||
|
||||
#define misaligned_mem(m) ((uintptr_t)(m) & MALLOC_ALIGN_MASK)
|
||||
|
||||
#define misaligned_chunk(p) (misaligned_mem( chunk2mem (p)))
|
||||
|
||||
/* pad request bytes into a usable size -- internal version */
|
||||
/* Note: This must be a macro that evaluates to a compile time constant
|
||||
if passed a literal constant. */
|
||||
#define request2size(req) \
|
||||
(((req) + SIZE_SZ + MALLOC_ALIGN_MASK < MINSIZE) ? \
|
||||
MINSIZE : \
|
||||
((req) + SIZE_SZ + MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK)
|
||||
|
||||
/* Check if REQ overflows when padded and aligned and if the resulting
|
||||
value is less than PTRDIFF_T. Returns the requested size or
|
||||
MINSIZE in case the value is less than MINSIZE, or SIZE_MAX if any
|
||||
of the previous checks fail. */
|
||||
static __always_inline size_t
|
||||
checked_request2size (size_t req) __nonnull (1)
|
||||
{
|
||||
if (__glibc_unlikely (req > PTRDIFF_MAX))
|
||||
return SIZE_MAX;
|
||||
return request2size (req);
|
||||
}
|
||||
#define misaligned_chunk(p) (misaligned_mem (chunk2mem (p)))
|
||||
|
||||
/*
|
||||
--------------- Physical chunk operations ---------------
|
||||
*/
|
||||
|
||||
|
||||
/* size field is or'ed with PREV_INUSE when previous adjacent chunk in use */
|
||||
#define PREV_INUSE 0x1
|
||||
|
||||
/* extract inuse bit of previous chunk */
|
||||
#define prev_inuse(p) ((p)->mchunk_size & PREV_INUSE)
|
||||
|
||||
|
||||
/* size field is or'ed with IS_MMAPPED if the chunk was obtained with mmap() */
|
||||
#define IS_MMAPPED 0x2
|
||||
|
||||
/* check for mmap()'ed chunk */
|
||||
#define chunk_is_mmapped(p) ((p)->mchunk_size & IS_MMAPPED)
|
||||
|
||||
|
||||
/* size field is or'ed with NON_MAIN_ARENA if the chunk was obtained
|
||||
from a non-main arena. This is only set immediately before handing
|
||||
the chunk to the user, if necessary. */
|
||||
#define NON_MAIN_ARENA 0x4
|
||||
|
||||
/* Check for chunk from main arena. */
|
||||
#define chunk_main_arena(p) (((p)->mchunk_size & NON_MAIN_ARENA) == 0)
|
||||
|
||||
/* Mark a chunk as not being on the main arena. */
|
||||
#define set_non_main_arena(p) ((p)->mchunk_size |= NON_MAIN_ARENA)
|
||||
|
||||
|
||||
/*
|
||||
Bits to mask off when extracting size
|
||||
|
||||
Note: IS_MMAPPED is intentionally not masked off from size field in
|
||||
macros for which mmapped chunks should never be seen. This should
|
||||
cause helpful core dumps to occur if it is tried by accident by
|
||||
people extending or adapting this malloc.
|
||||
*/
|
||||
#define SIZE_BITS (PREV_INUSE | IS_MMAPPED | NON_MAIN_ARENA)
|
||||
|
||||
/* Get size, ignoring use bits */
|
||||
#define chunksize(p) (chunksize_nomask (p) & ~(SIZE_BITS))
|
||||
|
||||
/* Like chunksize, but do not mask SIZE_BITS. */
|
||||
#define chunksize_nomask(p) ((p)->mchunk_size)
|
||||
|
||||
/* Ptr to next physical malloc_chunk. */
|
||||
#define next_chunk(p) ((mchunkptr) (((char *) (p)) + chunksize (p)))
|
||||
|
||||
@@ -1243,10 +1164,6 @@ checked_request2size (size_t req) __nonnull (1)
|
||||
#pragma GCC poison mchunk_size
|
||||
#pragma GCC poison mchunk_prev_size
|
||||
|
||||
/* This is the size of the real usable data in the chunk. Not valid for
|
||||
dumped heap chunks. */
|
||||
#define memsize(p) (chunksize (p) - CHUNK_HDR_SZ + SIZE_SZ)
|
||||
|
||||
/* Huge page used for an mmap chunk. */
|
||||
#define MMAP_HP 0x1
|
||||
|
||||
@@ -2075,9 +1992,7 @@ sysmalloc_mmap (INTERNAL_SIZE_T nb, size_t pagesize, int extra_flags)
|
||||
size_t padding = MALLOC_ALIGNMENT - CHUNK_HDR_SZ;
|
||||
size_t size = ALIGN_UP (nb + padding + CHUNK_HDR_SZ, pagesize);
|
||||
|
||||
char *mm = (char *) MMAP (NULL, size,
|
||||
extra_mmap_prot | PROT_READ | PROT_WRITE,
|
||||
extra_flags);
|
||||
char *mm = (char *) MMAP (NULL, size, extra_mmap_prot, extra_flags);
|
||||
if (mm == MAP_FAILED)
|
||||
return mm;
|
||||
if (extra_flags == 0)
|
||||
@@ -2116,9 +2031,7 @@ sysmalloc_mmap_fallback (size_t *s, size_t size, size_t minsize,
|
||||
if (size < minsize)
|
||||
size = minsize;
|
||||
|
||||
char *mbrk = (char *) (MMAP (NULL, size,
|
||||
extra_mmap_prot | PROT_READ | PROT_WRITE,
|
||||
extra_flags));
|
||||
char *mbrk = (char *) MMAP (NULL, size, extra_mmap_prot, extra_flags);
|
||||
if (mbrk == MAP_FAILED)
|
||||
return MAP_FAILED;
|
||||
|
||||
@@ -3320,8 +3233,11 @@ libc_hidden_def (__libc_memalign)
|
||||
|
||||
/* For ISO C17. */
|
||||
void *
|
||||
weak_function
|
||||
aligned_alloc (size_t alignment, size_t bytes)
|
||||
__aligned_alloc (size_t alignment, size_t bytes);
|
||||
libc_hidden_proto (__aligned_alloc)
|
||||
|
||||
void *
|
||||
__aligned_alloc (size_t alignment, size_t bytes)
|
||||
{
|
||||
/* Starting with ISO C17 the standard requires an error for alignments
|
||||
that are not supported. Only integral powers of 2 are valid. */
|
||||
@@ -3333,11 +3249,15 @@ aligned_alloc (size_t alignment, size_t bytes)
|
||||
|
||||
return _mid_memalign (alignment, bytes);
|
||||
}
|
||||
libc_hidden_def (__aligned_alloc)
|
||||
|
||||
/* For ISO C23. */
|
||||
void
|
||||
weak_function
|
||||
free_sized (void *ptr, __attribute_maybe_unused__ size_t size)
|
||||
__free_sized (void *ptr, __attribute_maybe_unused__ size_t size);
|
||||
libc_hidden_proto (__free_sized)
|
||||
|
||||
void
|
||||
__free_sized (void *ptr, __attribute_maybe_unused__ size_t size)
|
||||
{
|
||||
/* We do not perform validation that size is the same as the original
|
||||
requested size at this time. We leave that to the sanitizers. We
|
||||
@@ -3346,11 +3266,16 @@ free_sized (void *ptr, __attribute_maybe_unused__ size_t size)
|
||||
|
||||
free (ptr);
|
||||
}
|
||||
libc_hidden_def (__free_sized)
|
||||
|
||||
/* For ISO C23. */
|
||||
void
|
||||
weak_function
|
||||
free_aligned_sized (void *ptr, __attribute_maybe_unused__ size_t alignment,
|
||||
__free_aligned_sized (void *ptr, __attribute_maybe_unused__ size_t alignment,
|
||||
__attribute_maybe_unused__ size_t size);
|
||||
libc_hidden_proto (__free_aligned_sized)
|
||||
|
||||
void
|
||||
__free_aligned_sized (void *ptr, __attribute_maybe_unused__ size_t alignment,
|
||||
__attribute_maybe_unused__ size_t size)
|
||||
{
|
||||
/* We do not perform validation that size and alignment is the same as
|
||||
@@ -3360,6 +3285,7 @@ free_aligned_sized (void *ptr, __attribute_maybe_unused__ size_t alignment,
|
||||
|
||||
free (ptr);
|
||||
}
|
||||
libc_hidden_def (__free_aligned_sized)
|
||||
|
||||
static void *
|
||||
_mid_memalign (size_t alignment, size_t bytes)
|
||||
@@ -3408,6 +3334,7 @@ __libc_valloc (size_t bytes)
|
||||
{
|
||||
return _mid_memalign (GLRO (dl_pagesize), bytes);
|
||||
}
|
||||
libc_hidden_def (__libc_valloc)
|
||||
|
||||
void *
|
||||
__libc_pvalloc (size_t bytes)
|
||||
@@ -3425,6 +3352,7 @@ __libc_pvalloc (size_t bytes)
|
||||
|
||||
return _mid_memalign (pagesize, rounded_bytes & -pagesize);
|
||||
}
|
||||
libc_hidden_def (__libc_pvalloc)
|
||||
|
||||
static void * __attribute_noinline__
|
||||
__libc_calloc2 (size_t sz)
|
||||
@@ -3551,6 +3479,7 @@ __libc_calloc (size_t n, size_t elem_size)
|
||||
#endif
|
||||
return __libc_calloc2 (bytes);
|
||||
}
|
||||
libc_hidden_def (__libc_calloc)
|
||||
#endif /* IS_IN (libc) */
|
||||
|
||||
/*
|
||||
@@ -4547,6 +4476,7 @@ __malloc_usable_size (void *m)
|
||||
return 0;
|
||||
return musable (m);
|
||||
}
|
||||
libc_hidden_def (__malloc_usable_size)
|
||||
#endif /* IS_IN (libc) */
|
||||
|
||||
/*
|
||||
@@ -5074,6 +5004,7 @@ __posix_memalign (void **memptr, size_t alignment, size_t size)
|
||||
*memptr = mem;
|
||||
return 0;
|
||||
}
|
||||
libc_hidden_def (__posix_memalign)
|
||||
#endif /* IS_IN (libc) */
|
||||
|
||||
|
||||
@@ -5236,6 +5167,9 @@ __malloc_info (int options, FILE *fp)
|
||||
}
|
||||
|
||||
#if IS_IN (libc)
|
||||
|
||||
/* See sysdeps/generic/malloc-ifuncs.h for details. */
|
||||
# if !defined (USE_MULTIARCH_MALLOC)
|
||||
strong_alias (__libc_malloc, malloc)
|
||||
strong_alias (__libc_realloc, realloc)
|
||||
strong_alias (__libc_free, free)
|
||||
@@ -5245,6 +5179,10 @@ weak_alias (__posix_memalign, posix_memalign)
|
||||
weak_alias (__libc_valloc, valloc)
|
||||
weak_alias (__libc_pvalloc, pvalloc)
|
||||
weak_alias (__malloc_usable_size, malloc_usable_size)
|
||||
weak_alias (__aligned_alloc, aligned_alloc)
|
||||
weak_alias (__free_sized, free_sized)
|
||||
weak_alias (__free_aligned_sized, free_aligned_sized)
|
||||
#endif /* !USE_MULTIARCH_MALLOC */
|
||||
|
||||
weak_alias (__malloc_info, malloc_info)
|
||||
weak_alias (__libc_mallinfo, mallinfo)
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <inttypes.h>
|
||||
#include <malloc-size.h>
|
||||
#include <malloc-alignment.h>
|
||||
#include <support/check.h>
|
||||
|
||||
static void *
|
||||
|
||||
@@ -682,6 +682,21 @@ This tunable takes a value of 0 and 1, where 1 enables the feature.
|
||||
The default value is @samp{0}, which disables the decoration.
|
||||
@end deftp
|
||||
|
||||
@deftp Tunable glibc.mem.aarch64_mte
|
||||
On AArch64 systems that support Memory Tagging Extension (MTE) this
|
||||
tunable allows to select tag check fault mode (MTE mode).
|
||||
|
||||
Supported values are:
|
||||
|
||||
@itemize @bullet
|
||||
@item @code{none}: (the default), memory tagging is disabled.
|
||||
@item @code{auto}: enable CPU-preferred tag checking mode.
|
||||
@item @code{sync}: enable synchronous tag check fault mode.
|
||||
@item @code{async}: enable asynchronous tag check fault mode.
|
||||
@end itemize
|
||||
|
||||
@end deftp
|
||||
|
||||
@node gmon Tunables
|
||||
@section gmon Tunables
|
||||
@cindex gmon tunables
|
||||
|
||||
@@ -4,6 +4,7 @@ ifeq ($(subdir),elf)
|
||||
sysdep-dl-routines += \
|
||||
dl-bti \
|
||||
dl-gcs \
|
||||
dl-mte \
|
||||
# sysdep-dl-routines
|
||||
|
||||
tests += \
|
||||
|
||||
@@ -59,6 +59,13 @@ enum {
|
||||
BTI_CHECK_ENFORCED = 1,
|
||||
};
|
||||
|
||||
enum {
|
||||
MTE_TUNABLE_NONE = 0,
|
||||
MTE_TUNABLE_AUTO = 1,
|
||||
MTE_TUNABLE_SYNC = 2,
|
||||
MTE_TUNABLE_ASYNC = 3,
|
||||
};
|
||||
|
||||
struct cpu_features
|
||||
{
|
||||
uint64_t midr_el1;
|
||||
@@ -69,6 +76,7 @@ struct cpu_features
|
||||
bool unused;
|
||||
bool mops;
|
||||
bool sve2;
|
||||
bool mte;
|
||||
};
|
||||
|
||||
#endif /* _CPU_FEATURES_AARCH64_H */
|
||||
|
||||
@@ -47,6 +47,7 @@ _dl_diagnostics_cpu (void)
|
||||
print_cpu_features_value ("mops", GLRO (dl_aarch64_cpu_features).mops);
|
||||
print_cpu_features_value ("sve", GLRO (dl_aarch64_cpu_features).sve);
|
||||
print_cpu_features_value ("sve2", GLRO (dl_aarch64_cpu_features).sve2);
|
||||
print_cpu_features_value ("mte", GLRO (dl_aarch64_cpu_features).mte);
|
||||
print_cpu_features_value ("zva_size",
|
||||
GLRO (dl_aarch64_cpu_features).zva_size);
|
||||
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
/* AArch64 implementation for MTE (memory tagging).
|
||||
Copyright (C) 2026 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
#include <ldsodefs.h>
|
||||
|
||||
/* For the prctl syscall. */
|
||||
#define PR_SET_TAGGED_ADDR_CTRL 55
|
||||
#define PR_MTE_TAG_SHIFT 3
|
||||
#define PR_TAGGED_ADDR_ENABLE (1UL << 0)
|
||||
#define PR_MTE_TCF_SYNC (1UL << 1)
|
||||
#define PR_MTE_TCF_ASYNC (1UL << 2)
|
||||
|
||||
/* The maximal set of permitted tags that the MTE random tag generation
|
||||
instruction may use. We exclude tag 0 because a) we want to reserve
|
||||
that for the libc heap structures and b) because it makes it easier
|
||||
to see when pointer have been correctly tagged. */
|
||||
#define MTE_ALLOWED_TAGS (0xfffe << PR_MTE_TAG_SHIFT)
|
||||
|
||||
void __mte_init (void);
|
||||
rtld_hidden_proto (__mte_init)
|
||||
|
||||
void __mte_init (void)
|
||||
{
|
||||
if (!GLRO (dl_aarch64_cpu_features).mte)
|
||||
return;
|
||||
int mode = GL (dl_aarch64_mte);
|
||||
if (mode == MTE_TUNABLE_NONE)
|
||||
return;
|
||||
uint64_t flags = PR_TAGGED_ADDR_ENABLE | MTE_ALLOWED_TAGS;
|
||||
switch (mode)
|
||||
{
|
||||
case MTE_TUNABLE_AUTO:
|
||||
flags |= PR_MTE_TCF_SYNC | PR_MTE_TCF_ASYNC;
|
||||
break;
|
||||
case MTE_TUNABLE_SYNC:
|
||||
flags |= PR_MTE_TCF_SYNC;
|
||||
break;
|
||||
case MTE_TUNABLE_ASYNC:
|
||||
flags |= PR_MTE_TCF_ASYNC;
|
||||
break;
|
||||
default:
|
||||
_dl_fatal_printf ("unknown MTE mode: %d\n", mode);
|
||||
__builtin_unreachable ();
|
||||
}
|
||||
/* We use inline system call to avoid unnecessary dependency
|
||||
on the sys/prctl.h header. */
|
||||
int r = INLINE_SYSCALL_CALL (prctl, PR_SET_TAGGED_ADDR_CTRL, flags, 0, 0, 0);
|
||||
if (r == -1)
|
||||
_dl_fatal_printf ("failed to enable MTE\n");
|
||||
}
|
||||
rtld_hidden_def (__mte_init)
|
||||
@@ -66,6 +66,8 @@ ENTRY (_start)
|
||||
cbnz w0, L(failed_gcs_lock)
|
||||
L(skip_gcs_enable):
|
||||
|
||||
bl HIDDEN_JUMPTARGET(__mte_init)
|
||||
|
||||
.globl _dl_start_user
|
||||
.type _dl_start_user, %function
|
||||
_dl_start_user:
|
||||
|
||||
@@ -34,4 +34,9 @@ glibc {
|
||||
default: 0
|
||||
}
|
||||
}
|
||||
mem {
|
||||
aarch64_mte {
|
||||
type: STRING
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,5 +18,12 @@ sysdep_routines += \
|
||||
memset_zva64 \
|
||||
strlen_asimd \
|
||||
strlen_generic \
|
||||
# sysdep_routines
|
||||
endif
|
||||
# sysdep_routines
|
||||
endif # ifeq ($(subdir),string)
|
||||
|
||||
ifeq ($(subdir),malloc)
|
||||
sysdep_routines += \
|
||||
malloc-ifuncs \
|
||||
malloc-mte \
|
||||
# sysdep_routines
|
||||
endif # ifeq ($(subdir),malloc)
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
unsigned __attribute__((unused)) zva_size = \
|
||||
GLRO(dl_aarch64_cpu_features).zva_size; \
|
||||
bool __attribute__((unused)) bti = GLRO(dl_aarch64_cpu_features).bti; \
|
||||
bool __attribute__((unused)) mte = GLRO(dl_hwcap2) & HWCAP2_MTE; \
|
||||
bool __attribute__((unused)) mte = GLRO(dl_aarch64_cpu_features).mte; \
|
||||
bool __attribute__((unused)) sve = GLRO(dl_aarch64_cpu_features).sve; \
|
||||
bool __attribute__((unused)) sve2 = GLRO(dl_aarch64_cpu_features).sve2; \
|
||||
bool __attribute__((unused)) mops = GLRO(dl_aarch64_cpu_features).mops;
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
/* Code for ifunc resolvers for malloc: aarch64 version.
|
||||
Copyright (C) 2026 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#if IS_IN (libc)
|
||||
|
||||
#include <stdint.h>
|
||||
#include <init-arch.h>
|
||||
#include <malloc-ifuncs.h>
|
||||
|
||||
#define MTE_ACTIVE (mte && (GL (dl_aarch64_mte) != MTE_TUNABLE_NONE))
|
||||
|
||||
libc_ifunc_hidden (__libc_malloc, __libc_malloc_redirect,
|
||||
MTE_ACTIVE ? __libc_malloc_mte : __libc_malloc)
|
||||
strong_alias (__libc_malloc_redirect, malloc)
|
||||
|
||||
libc_ifunc_hidden (__libc_calloc, __libc_calloc_redirect,
|
||||
MTE_ACTIVE ? __libc_calloc_mte : __libc_calloc)
|
||||
weak_alias (__libc_calloc_redirect, calloc)
|
||||
|
||||
libc_ifunc_hidden (__libc_memalign, __libc_memalign_redirect,
|
||||
MTE_ACTIVE ? __libc_memalign_mte : __libc_memalign)
|
||||
weak_alias (__libc_memalign_redirect, memalign)
|
||||
|
||||
libc_ifunc_hidden (__libc_valloc, __libc_valloc_redirect,
|
||||
MTE_ACTIVE ? __libc_valloc_mte : __libc_valloc)
|
||||
weak_alias (__libc_valloc_redirect, valloc)
|
||||
|
||||
libc_ifunc_hidden (__libc_pvalloc, __libc_pvalloc_redirect,
|
||||
MTE_ACTIVE ? __libc_pvalloc_mte : __libc_pvalloc)
|
||||
weak_alias (__libc_pvalloc_redirect, pvalloc)
|
||||
|
||||
libc_ifunc_hidden (__libc_realloc, __libc_realloc_redirect,
|
||||
MTE_ACTIVE ? __libc_realloc_mte : __libc_realloc)
|
||||
strong_alias (__libc_realloc_redirect, realloc)
|
||||
|
||||
libc_ifunc_hidden (__libc_free, __libc_free_redirect,
|
||||
MTE_ACTIVE ? __libc_free_mte : __libc_free)
|
||||
strong_alias (__libc_free_redirect, free)
|
||||
|
||||
libc_ifunc_hidden (__malloc_usable_size, __malloc_usable_size_redirect,
|
||||
MTE_ACTIVE ? __malloc_usable_size_mte : __malloc_usable_size)
|
||||
weak_alias (__malloc_usable_size_redirect, malloc_usable_size)
|
||||
|
||||
libc_ifunc_hidden (__posix_memalign, __posix_memalign_redirect,
|
||||
MTE_ACTIVE ? __posix_memalign_mte : __posix_memalign)
|
||||
weak_alias (__posix_memalign_redirect, posix_memalign)
|
||||
|
||||
libc_ifunc_hidden (__aligned_alloc, __aligned_alloc_redirect,
|
||||
MTE_ACTIVE ? __aligned_alloc_mte : __aligned_alloc)
|
||||
weak_alias (__aligned_alloc_redirect, aligned_alloc)
|
||||
|
||||
libc_ifunc_hidden (__free_sized, __free_sized_redirect,
|
||||
MTE_ACTIVE ? __free_sized_mte : __free_sized)
|
||||
weak_alias (__free_sized_redirect, free_sized)
|
||||
|
||||
libc_ifunc_hidden (__free_aligned_sized, __free_aligned_sized_redirect,
|
||||
MTE_ACTIVE ? __free_aligned_sized_mte : __free_aligned_sized)
|
||||
weak_alias (__free_aligned_sized_redirect, free_aligned_sized)
|
||||
|
||||
#endif /* IS_IN (libc) */
|
||||
@@ -0,0 +1,93 @@
|
||||
/* Definitions for ifunc resolvers for malloc: aarch64 version.
|
||||
Copyright (C) 2026 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#ifndef _AARCH64_MALLOC_IFUNCS_H
|
||||
#define _AARCH64_MALLOC_IFUNCS_H
|
||||
|
||||
#define USE_MULTIARCH_MALLOC 1
|
||||
|
||||
#include <stddef.h>
|
||||
#include <sys/cdefs.h>
|
||||
|
||||
/* Core implementations of malloc functions. An ifunc resolver must
|
||||
use this implementations as a fallback option. Other implementations
|
||||
may internally call these core function. */
|
||||
void *__libc_malloc (size_t);
|
||||
libc_hidden_proto (__libc_malloc)
|
||||
void *__libc_calloc (size_t, size_t);
|
||||
libc_hidden_proto (__libc_calloc)
|
||||
void *__libc_memalign (size_t, size_t);
|
||||
libc_hidden_proto (__libc_memalign)
|
||||
void *__libc_valloc (size_t);
|
||||
libc_hidden_proto (__libc_valloc)
|
||||
void *__libc_pvalloc (size_t);
|
||||
libc_hidden_proto (__libc_pvalloc)
|
||||
void *__libc_realloc (void *, size_t);
|
||||
libc_hidden_proto (__libc_realloc)
|
||||
void __libc_free (void *);
|
||||
libc_hidden_proto (__libc_free)
|
||||
size_t __malloc_usable_size (void *);
|
||||
libc_hidden_proto (__malloc_usable_size)
|
||||
|
||||
/* For additions of POSIX. */
|
||||
int __posix_memalign (void **, size_t, size_t);
|
||||
libc_hidden_proto (__posix_memalign)
|
||||
|
||||
/* For ISO C17. */
|
||||
void *__aligned_alloc (size_t, size_t);
|
||||
libc_hidden_proto (__aligned_alloc)
|
||||
|
||||
/* For ISO C23. */
|
||||
void __free_sized (void *, size_t);
|
||||
libc_hidden_proto (__free_sized)
|
||||
void __free_aligned_sized (void *, size_t, size_t);
|
||||
libc_hidden_proto (__free_aligned_sized)
|
||||
|
||||
/* MTE implementations of malloc functions. */
|
||||
void *__libc_malloc_mte (size_t);
|
||||
libc_hidden_proto (__libc_malloc_mte)
|
||||
void *__libc_calloc_mte (size_t, size_t);
|
||||
libc_hidden_proto (__libc_calloc_mte)
|
||||
void *__libc_memalign_mte (size_t, size_t);
|
||||
libc_hidden_proto (__libc_memalign_mte)
|
||||
void *__libc_valloc_mte (size_t);
|
||||
libc_hidden_proto (__libc_valloc_mte)
|
||||
void *__libc_pvalloc_mte (size_t);
|
||||
libc_hidden_proto (__libc_pvalloc_mte)
|
||||
void *__libc_realloc_mte (void *, size_t);
|
||||
libc_hidden_proto (__libc_realloc_mte)
|
||||
void __libc_free_mte (void *);
|
||||
libc_hidden_proto (__libc_free_mte)
|
||||
size_t __malloc_usable_size_mte (void *);
|
||||
libc_hidden_proto (__malloc_usable_size_mte)
|
||||
|
||||
/* For additions of POSIX: MTE version. */
|
||||
int __posix_memalign_mte (void **, size_t, size_t);
|
||||
libc_hidden_proto (__posix_memalign_mte)
|
||||
|
||||
/* For ISO C17: MTE version. */
|
||||
void *__aligned_alloc_mte (size_t, size_t);
|
||||
libc_hidden_proto (__aligned_alloc_mte)
|
||||
|
||||
/* For ISO C23: MTE version. */
|
||||
void __free_sized_mte (void *, size_t);
|
||||
libc_hidden_proto (__free_sized_mte)
|
||||
void __free_aligned_sized_mte (void *, size_t, size_t);
|
||||
libc_hidden_proto (__free_aligned_sized_mte)
|
||||
|
||||
#endif /* _AARCH64_MALLOC_IFUNCS_H */
|
||||
@@ -0,0 +1,34 @@
|
||||
/* Definitions for malloc init: aarch64 version.
|
||||
Copyright (C) 2026 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#ifndef _AARCH64_MALLOC_INIT_H
|
||||
#define _AARCH64_MALLOC_INIT_H
|
||||
|
||||
#define MTE_ACTIVE \
|
||||
(GLRO (dl_aarch64_cpu_features).mte) && \
|
||||
(GL (dl_aarch64_mte) != MTE_TUNABLE_NONE)
|
||||
|
||||
#define ARCH_INIT_MALLOC() \
|
||||
extra_mmap_prot = PROT_READ | PROT_WRITE; \
|
||||
if (MTE_ACTIVE) \
|
||||
{ \
|
||||
extra_mmap_prot |= PROT_MTE; \
|
||||
__always_fail_morecore = true; \
|
||||
}
|
||||
|
||||
#endif /* _AARCH64_MALLOC_INIT_H */
|
||||
@@ -0,0 +1,176 @@
|
||||
/* Implementation for MTE (memory tagging) wrappers in malloc.
|
||||
Copyright (C) 2026 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<http://www.gnu.org/licenses/>. */
|
||||
|
||||
#include "aarch64-mte.h"
|
||||
|
||||
#include <malloc-ifuncs.h>
|
||||
#include <errno.h>
|
||||
|
||||
#define TAG_MEM(ptr, tagfun) __glibc_unlikely (ptr == NULL) ? NULL : ({ \
|
||||
size_t size = __malloc_usable_size (ptr); \
|
||||
tagfun (__mte_new_tag (ptr), size); \
|
||||
})
|
||||
|
||||
#define UNTAG_MEM(non_null_ptr) ({ \
|
||||
void *untagged = __mte_clear_tag (non_null_ptr); \
|
||||
size_t size = __malloc_usable_size (untagged); \
|
||||
__mte_tag_region (untagged, size); \
|
||||
})
|
||||
|
||||
void *__libc_malloc_mte (size_t bytes)
|
||||
{
|
||||
void *untagged = __libc_malloc (bytes);
|
||||
return TAG_MEM (untagged, __mte_tag_region);
|
||||
}
|
||||
libc_hidden_def (__libc_malloc_mte)
|
||||
|
||||
void *__libc_calloc_mte (size_t n, size_t elem_size)
|
||||
{
|
||||
/* We use core malloc instead of calloc because we can
|
||||
take advantage of MTE to zero memory region. */
|
||||
void *untagged = __libc_malloc (n * elem_size);
|
||||
return TAG_MEM (untagged, __mte_tag_region_zero);
|
||||
}
|
||||
libc_hidden_def (__libc_calloc_mte)
|
||||
|
||||
void *__libc_memalign_mte (size_t alignment, size_t bytes)
|
||||
{
|
||||
void *untagged = __libc_memalign (alignment, bytes);
|
||||
return TAG_MEM (untagged, __mte_tag_region);
|
||||
}
|
||||
libc_hidden_def (__libc_memalign_mte)
|
||||
|
||||
void *__libc_valloc_mte (size_t bytes)
|
||||
{
|
||||
void *untagged = __libc_valloc (bytes);
|
||||
return TAG_MEM (untagged, __mte_tag_region);
|
||||
}
|
||||
libc_hidden_def (__libc_valloc_mte)
|
||||
|
||||
void *__libc_pvalloc_mte (size_t bytes)
|
||||
{
|
||||
void *untagged = __libc_pvalloc (bytes);
|
||||
return TAG_MEM (untagged, __mte_tag_region);
|
||||
}
|
||||
libc_hidden_def (__libc_pvalloc_mte)
|
||||
|
||||
/* See malloc.c for details. */
|
||||
#ifndef REALLOC_ZERO_BYTES_FREES
|
||||
#define REALLOC_ZERO_BYTES_FREES 1
|
||||
#endif
|
||||
|
||||
void *__libc_realloc_mte (void *tagged_oldmem, size_t bytes)
|
||||
{
|
||||
/* Quick check: realloc of null is supposed to be same as malloc. */
|
||||
if (tagged_oldmem == NULL)
|
||||
return __libc_malloc_mte (bytes);
|
||||
|
||||
#if REALLOC_ZERO_BYTES_FREES
|
||||
/* Quick check: realloc with 0 size is supposed to be same as free. */
|
||||
if (bytes == 0)
|
||||
{
|
||||
__libc_free_mte (tagged_oldmem);
|
||||
return NULL;
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Bad size, old memory remains unchanged. */
|
||||
if (bytes > PTRDIFF_MAX)
|
||||
{
|
||||
__set_errno (ENOMEM);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* At this point we untag oldmem allocation. */
|
||||
void *untagged_oldmem = __mte_clear_tag (tagged_oldmem);
|
||||
|
||||
/* Mark the chunk as belonging to the library again. */
|
||||
size_t size_old = __malloc_usable_size (untagged_oldmem);
|
||||
untagged_oldmem = __mte_tag_region (untagged_oldmem, size_old);
|
||||
|
||||
/* Call realloc core. */
|
||||
void *untagged_newmem = __libc_realloc (untagged_oldmem, bytes);
|
||||
if (untagged_newmem == NULL)
|
||||
return NULL;
|
||||
size_t size_new = __malloc_usable_size (untagged_newmem);
|
||||
|
||||
/* If realloc core returns old pointer, we need re-tag it. */
|
||||
if (size_new == size_old && untagged_newmem == untagged_oldmem)
|
||||
return __mte_tag_region (tagged_oldmem, size_new);
|
||||
|
||||
/* Otherwise, assign new tag. */
|
||||
void *tagged_newmem = __mte_new_tag (untagged_newmem);
|
||||
return __mte_tag_region (tagged_newmem, size_new);
|
||||
}
|
||||
libc_hidden_def (__libc_realloc_mte)
|
||||
|
||||
void __libc_free_mte (void *tagged)
|
||||
{
|
||||
if (__glibc_unlikely (tagged == NULL))
|
||||
return;
|
||||
/* Mark the chunk as belonging to the library again. */
|
||||
void *untagged = UNTAG_MEM (tagged);
|
||||
/* Call free core. */
|
||||
__libc_free (untagged);
|
||||
}
|
||||
libc_hidden_def (__libc_free_mte)
|
||||
|
||||
size_t __malloc_usable_size_mte (void *tagged)
|
||||
{
|
||||
/* Clear only logical tag to allow accessing internal malloc
|
||||
structures via offset from this pointer. */
|
||||
void *untagged = __mte_clear_tag (tagged);
|
||||
return __malloc_usable_size (untagged);
|
||||
}
|
||||
libc_hidden_def (__malloc_usable_size_mte)
|
||||
|
||||
int __posix_memalign_mte (void **memptr, size_t alignment, size_t size)
|
||||
{
|
||||
/* Call core function. */
|
||||
int err = __posix_memalign (memptr, alignment, size);
|
||||
if (err != 0)
|
||||
return err;
|
||||
*memptr = TAG_MEM (*memptr, __mte_tag_region);
|
||||
return err;
|
||||
}
|
||||
libc_hidden_def (__posix_memalign_mte)
|
||||
|
||||
void *__aligned_alloc_mte (size_t alignment, size_t bytes)
|
||||
{
|
||||
void *untagged = __aligned_alloc (alignment, bytes);
|
||||
return TAG_MEM (untagged, __mte_tag_region);
|
||||
}
|
||||
libc_hidden_def (__aligned_alloc_mte)
|
||||
|
||||
void __free_sized_mte (void *tagged, size_t size)
|
||||
{
|
||||
/* Mark the chunk as belonging to the library again. */
|
||||
void *untagged = UNTAG_MEM (tagged);
|
||||
/* Call core function. */
|
||||
__free_sized (untagged, size);
|
||||
}
|
||||
libc_hidden_def (__free_sized_mte)
|
||||
|
||||
void __free_aligned_sized_mte (void *tagged, size_t alignment, size_t size)
|
||||
{
|
||||
/* Mark the chunk as belonging to the library again. */
|
||||
void *untagged = UNTAG_MEM (tagged);
|
||||
/* Call core function. */
|
||||
__free_aligned_sized (untagged, alignment, size);
|
||||
}
|
||||
libc_hidden_def (__free_aligned_sized_mte)
|
||||
@@ -0,0 +1,69 @@
|
||||
/* Size-related definitions for malloc: aarch64 version.
|
||||
Copyright (C) 2026 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#ifndef _AARCH64_MALLOC_SIZE_H
|
||||
#define _AARCH64_MALLOC_SIZE_H
|
||||
|
||||
#include <malloc-chunk.h>
|
||||
#include <sys/cdefs.h>
|
||||
|
||||
/* The smallest size we can malloc is an aligned minimal chunk. */
|
||||
#define MINSIZE \
|
||||
(unsigned long)(((MIN_CHUNK_SIZE + MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK))
|
||||
|
||||
/* Pad request bytes into a usable size -- internal version. Note: This must
|
||||
be a macro that evaluates to a compile time constant if passed a literal
|
||||
constant. */
|
||||
#define request2size(req) \
|
||||
(((req) + SIZE_SZ + MALLOC_ALIGN_MASK < MINSIZE) ? \
|
||||
MINSIZE : \
|
||||
((req) + SIZE_SZ + MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK)
|
||||
|
||||
/* MTE uses 16-byte granules. */
|
||||
#define MTE_GRANULE_SIZE 16
|
||||
|
||||
/* Memory tagging is not supported with small granule. */
|
||||
verify (MTE_GRANULE_SIZE > SIZE_SZ);
|
||||
|
||||
/* Memory tagging is not supported with large granule. */
|
||||
verify (MTE_GRANULE_SIZE <= CHUNK_HDR_SZ);
|
||||
|
||||
/* Check if REQ overflows when padded and aligned and if the resulting
|
||||
value is less than PTRDIFF_T. Returns the requested size or
|
||||
MINSIZE in case the value is less than MINSIZE, or SIZE_MAX if any
|
||||
of the previous checks fail. */
|
||||
static __always_inline __attribute_maybe_unused__ size_t
|
||||
checked_request2size (size_t req) __nonnull (1)
|
||||
{
|
||||
if (__glibc_unlikely (req > PTRDIFF_MAX))
|
||||
return SIZE_MAX;
|
||||
req = (req + (MTE_GRANULE_SIZE - 1)) & ~(size_t) (MTE_GRANULE_SIZE - 1);
|
||||
return request2size (req);
|
||||
}
|
||||
|
||||
/* Like chunksize, but do not mask SIZE_BITS. */
|
||||
#define chunksize_nomask(p) ((p)->mchunk_size)
|
||||
|
||||
/* Get size, ignoring use bits. */
|
||||
#define chunksize(p) (chunksize_nomask (p) & ~(SIZE_BITS))
|
||||
|
||||
/* This is the size of the real usable data in the chunk. Not valid for
|
||||
dumped heap chunks. */
|
||||
#define memsize(p) (chunksize (p) - CHUNK_HDR_SZ)
|
||||
|
||||
#endif /* _AARCH64_MALLOC_SIZE_H */
|
||||
@@ -1,4 +1,5 @@
|
||||
/* Define MALLOC_ALIGNMENT for malloc. Generic version.
|
||||
/* Define INTERNAL_SIZE_T and MALLOC_ALIGNMENT for malloc.
|
||||
Generic version.
|
||||
Copyright (C) 2017-2026 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
@@ -19,6 +20,44 @@
|
||||
#ifndef _GENERIC_MALLOC_ALIGNMENT_H
|
||||
#define _GENERIC_MALLOC_ALIGNMENT_H
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
/* INTERNAL_SIZE_T is the word-size used for internal bookkeeping of
|
||||
chunk sizes.
|
||||
|
||||
The default version is the same as size_t.
|
||||
|
||||
While not strictly necessary, it is best to define this as an
|
||||
unsigned type, even if size_t is a signed type. This may avoid some
|
||||
artificial size limitations on some systems.
|
||||
|
||||
On a 64-bit machine, you may be able to reduce malloc overhead by
|
||||
defining INTERNAL_SIZE_T to be a 32 bit `unsigned int' at the
|
||||
expense of not being able to handle more than 2^32 of malloced
|
||||
space. If this limitation is acceptable, you are encouraged to set
|
||||
this unless you are on a platform requiring 16byte alignments. In
|
||||
this case the alignment requirements turn out to negate any
|
||||
potential advantages of decreasing size_t word size.
|
||||
|
||||
Implementors: Beware of the possible combinations of:
|
||||
- INTERNAL_SIZE_T might be signed or unsigned, might be 32 or 64 bits,
|
||||
and might be the same width as int or as long
|
||||
- size_t might have different width and signedness as INTERNAL_SIZE_T
|
||||
- int and long might be 32 or 64 bits, and might be the same width
|
||||
|
||||
To deal with this, most comparisons and difference computations
|
||||
among INTERNAL_SIZE_Ts should cast them to unsigned long, being
|
||||
aware of the fact that casting an unsigned int to a wider long does
|
||||
not sign-extend. (This also makes checking for negative numbers
|
||||
awkward.) Some of these casts result in harmless compiler warnings
|
||||
on some systems. */
|
||||
#ifndef INTERNAL_SIZE_T
|
||||
# define INTERNAL_SIZE_T size_t
|
||||
#endif
|
||||
|
||||
/* The corresponding word size. */
|
||||
#define SIZE_SZ (sizeof (INTERNAL_SIZE_T))
|
||||
|
||||
/* MALLOC_ALIGNMENT is the minimum alignment for malloc'ed chunks. It
|
||||
must be a power of two at least 2 * SIZE_SZ, even on machines for
|
||||
which smaller alignments would suffice. It may be defined as larger
|
||||
@@ -27,5 +66,7 @@
|
||||
#define MALLOC_ALIGNMENT (2 * SIZE_SZ < __alignof__ (long double) \
|
||||
? __alignof__ (long double) : 2 * SIZE_SZ)
|
||||
|
||||
/* The corresponding bit mask value. */
|
||||
#define MALLOC_ALIGN_MASK (MALLOC_ALIGNMENT - 1)
|
||||
|
||||
#endif /* !defined(_GENERIC_MALLOC_ALIGNMENT_H) */
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
/* Malloc chunk struct: generic version.
|
||||
Copyright (C) 2026 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#ifndef _GENERIC_MALLOC_CHUNK_H
|
||||
#define _GENERIC_MALLOC_CHUNK_H
|
||||
|
||||
#include <malloc-alignment.h>
|
||||
|
||||
/*
|
||||
This struct declaration is misleading (but accurate and necessary).
|
||||
It declares a "view" into memory allowing access to necessary
|
||||
fields at known offsets from a given base. See explanation below.
|
||||
*/
|
||||
|
||||
struct malloc_chunk {
|
||||
INTERNAL_SIZE_T mchunk_prev_size; /* Size of previous chunk (if free). */
|
||||
INTERNAL_SIZE_T mchunk_size; /* Size in bytes, including overhead. */
|
||||
|
||||
struct malloc_chunk* fd; /* double links -- used only if free. */
|
||||
struct malloc_chunk* bk;
|
||||
|
||||
/* Only used for large blocks: pointer to next larger size. */
|
||||
struct malloc_chunk* fd_nextsize; /* double links -- used only if free. */
|
||||
struct malloc_chunk* bk_nextsize;
|
||||
};
|
||||
|
||||
/* The chunk header is two SIZE_SZ elements, but this is used widely, so
|
||||
we define it here for clarity later. */
|
||||
#define CHUNK_HDR_SZ (2 * SIZE_SZ)
|
||||
|
||||
/* The smallest possible chunk */
|
||||
#define MIN_CHUNK_SIZE (offsetof(struct malloc_chunk, fd_nextsize))
|
||||
|
||||
/* size field is or'ed with PREV_INUSE when previous adjacent chunk in use. */
|
||||
#define PREV_INUSE 0x1
|
||||
|
||||
/* size field is or'ed with IS_MMAPPED if the chunk was obtained with mmap() */
|
||||
#define IS_MMAPPED 0x2
|
||||
|
||||
/* size field is or'ed with NON_MAIN_ARENA if the chunk was obtained
|
||||
from a non-main arena. This is only set immediately before handing
|
||||
the chunk to the user, if necessary. */
|
||||
#define NON_MAIN_ARENA 0x4
|
||||
|
||||
/*
|
||||
Bits to mask off when extracting size.
|
||||
|
||||
Note: IS_MMAPPED is intentionally not masked off from size field in
|
||||
macros for which mmapped chunks should never be seen. This should
|
||||
cause helpful core dumps to occur if it is tried by accident by
|
||||
people extending or adapting this malloc.
|
||||
*/
|
||||
#define SIZE_BITS (PREV_INUSE | IS_MMAPPED | NON_MAIN_ARENA)
|
||||
|
||||
#endif /* _GENERIC_MALLOC_CHUNK_H */
|
||||
@@ -0,0 +1,37 @@
|
||||
/* Definitions for ifunc resolvers for malloc: generic version.
|
||||
Copyright (C) 2026 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#ifndef _GENERIC_MALLOC_IFUNCS_H
|
||||
#define _GENERIC_MALLOC_IFUNCS_H
|
||||
|
||||
/* Targets should define this macro if they provide ifuncs for
|
||||
malloc functions. When USE_MULTIARCH_MALLOC is defined, the
|
||||
following functions should be implemented via ifuncs:
|
||||
|
||||
malloc, calloc, free, realloc
|
||||
memalign, valloc, pvalloc
|
||||
posix_memalign
|
||||
malloc_usable_size
|
||||
aligned_alloc, free_sized, free_aligned_sized
|
||||
|
||||
*/
|
||||
#if defined(USE_MULTIARCH_MALLOC) || !defined(USE_MULTIARCH)
|
||||
# undef USE_MULTIARCH_MALLOC
|
||||
#endif
|
||||
|
||||
#endif /* GENERIC_MALLOC_IFUNCS_H */
|
||||
@@ -0,0 +1,25 @@
|
||||
/* Definition for malloc init: generic version.
|
||||
Copyright (C) 2026 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#ifndef _GENERIC_MALLOC_INIT_H
|
||||
#define _GENERIC_MALLOC_INIT_H
|
||||
|
||||
#define ARCH_INIT_MALLOC() \
|
||||
extra_mmap_prot = PROT_READ | PROT_WRITE;
|
||||
|
||||
#endif /* _GENERIC_MALLOC_INIT_H */
|
||||
@@ -1,5 +1,4 @@
|
||||
/* Define INTERNAL_SIZE_T, SIZE_SZ, MALLOC_ALIGNMENT and MALLOC_ALIGN_MASK
|
||||
for malloc.
|
||||
/* Size-related definitions for malloc: generic version.
|
||||
Copyright (C) 2021-2026 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
@@ -20,45 +19,42 @@
|
||||
#ifndef _GENERIC_MALLOC_SIZE_H
|
||||
#define _GENERIC_MALLOC_SIZE_H
|
||||
|
||||
/* INTERNAL_SIZE_T is the word-size used for internal bookkeeping of
|
||||
chunk sizes.
|
||||
#include <malloc-chunk.h>
|
||||
#include <stdint.h>
|
||||
#include <sys/cdefs.h>
|
||||
|
||||
The default version is the same as size_t.
|
||||
/* The smallest size we can malloc is an aligned minimal chunk. */
|
||||
#define MINSIZE \
|
||||
(unsigned long)(((MIN_CHUNK_SIZE + MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK))
|
||||
|
||||
While not strictly necessary, it is best to define this as an
|
||||
unsigned type, even if size_t is a signed type. This may avoid some
|
||||
artificial size limitations on some systems.
|
||||
/* Pad request bytes into a usable size -- internal version. Note: This must
|
||||
be a macro that evaluates to a compile time constant if passed a literal
|
||||
constant. */
|
||||
#define request2size(req) \
|
||||
(((req) + SIZE_SZ + MALLOC_ALIGN_MASK < MINSIZE) ? \
|
||||
MINSIZE : \
|
||||
((req) + SIZE_SZ + MALLOC_ALIGN_MASK) & ~MALLOC_ALIGN_MASK)
|
||||
|
||||
On a 64-bit machine, you may be able to reduce malloc overhead by
|
||||
defining INTERNAL_SIZE_T to be a 32 bit `unsigned int' at the
|
||||
expense of not being able to handle more than 2^32 of malloced
|
||||
space. If this limitation is acceptable, you are encouraged to set
|
||||
this unless you are on a platform requiring 16byte alignments. In
|
||||
this case the alignment requirements turn out to negate any
|
||||
potential advantages of decreasing size_t word size.
|
||||
/* Check if REQ overflows when padded and aligned and if the resulting
|
||||
value is less than PTRDIFF_T. Returns the requested size or
|
||||
MINSIZE in case the value is less than MINSIZE, or SIZE_MAX if any
|
||||
of the previous checks fail. */
|
||||
static __always_inline __attribute_maybe_unused__ size_t
|
||||
checked_request2size (size_t req) __nonnull (1)
|
||||
{
|
||||
if (__glibc_unlikely (req > PTRDIFF_MAX))
|
||||
return SIZE_MAX;
|
||||
return request2size (req);
|
||||
}
|
||||
|
||||
Implementors: Beware of the possible combinations of:
|
||||
- INTERNAL_SIZE_T might be signed or unsigned, might be 32 or 64 bits,
|
||||
and might be the same width as int or as long
|
||||
- size_t might have different width and signedness as INTERNAL_SIZE_T
|
||||
- int and long might be 32 or 64 bits, and might be the same width
|
||||
/* Like chunksize, but do not mask SIZE_BITS. */
|
||||
#define chunksize_nomask(p) ((p)->mchunk_size)
|
||||
|
||||
To deal with this, most comparisons and difference computations
|
||||
among INTERNAL_SIZE_Ts should cast them to unsigned long, being
|
||||
aware of the fact that casting an unsigned int to a wider long does
|
||||
not sign-extend. (This also makes checking for negative numbers
|
||||
awkward.) Some of these casts result in harmless compiler warnings
|
||||
on some systems. */
|
||||
#ifndef INTERNAL_SIZE_T
|
||||
# define INTERNAL_SIZE_T size_t
|
||||
#endif
|
||||
/* Get size, ignoring use bits. */
|
||||
#define chunksize(p) (chunksize_nomask (p) & ~(SIZE_BITS))
|
||||
|
||||
/* The corresponding word size. */
|
||||
#define SIZE_SZ (sizeof (INTERNAL_SIZE_T))
|
||||
|
||||
#include <malloc-alignment.h>
|
||||
|
||||
/* The corresponding bit mask value. */
|
||||
#define MALLOC_ALIGN_MASK (MALLOC_ALIGNMENT - 1)
|
||||
/* This is the size of the real usable data in the chunk. Not valid for
|
||||
dumped heap chunks. */
|
||||
#define memsize(p) (chunksize (p) - CHUNK_HDR_SZ + SIZE_SZ)
|
||||
|
||||
#endif /* _GENERIC_MALLOC_SIZE_H */
|
||||
|
||||
@@ -19,6 +19,20 @@
|
||||
#ifndef _I386_MALLOC_ALIGNMENT_H
|
||||
#define _I386_MALLOC_ALIGNMENT_H
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
/* INTERNAL_SIZE_T is the word-size used for internal bookkeeping of
|
||||
chunk sizes. See sysdeps/generic/malloc-alignment.h for details. */
|
||||
#ifndef INTERNAL_SIZE_T
|
||||
# define INTERNAL_SIZE_T size_t
|
||||
#endif
|
||||
|
||||
/* The corresponding word size. */
|
||||
#define SIZE_SZ (sizeof (INTERNAL_SIZE_T))
|
||||
|
||||
#define MALLOC_ALIGNMENT 16
|
||||
|
||||
/* The corresponding bit mask value. */
|
||||
#define MALLOC_ALIGN_MASK (MALLOC_ALIGNMENT - 1)
|
||||
|
||||
#endif /* !defined(_I386_MALLOC_ALIGNMENT_H) */
|
||||
|
||||
@@ -371,6 +371,49 @@ ifeq ($(subdir),stdlib)
|
||||
gen-as-const-headers += ucontext_i.sym
|
||||
endif
|
||||
|
||||
ifeq ($(subdir),malloc)
|
||||
ifneq ($(multi-arch),no)
|
||||
# Special test that check MTE memory tagging
|
||||
tests-for-mte += \
|
||||
tst-mte-malloc \
|
||||
tst-mte-malloc-static \
|
||||
tst-mte-realloc \
|
||||
tst-mte-realloc-static \
|
||||
# tests-for-mte
|
||||
|
||||
tests += $(tests-for-mte)
|
||||
|
||||
tests-static += \
|
||||
tst-mte-malloc-static \
|
||||
tst-mte-realloc-static \
|
||||
# tests-static
|
||||
|
||||
CFLAGS-tst-mte-malloc.o += -march=armv9-a+memtag
|
||||
CFLAGS-tst-mte-malloc-static.o += -march=armv9-a+memtag
|
||||
CFLAGS-tst-mte-realloc.o += -march=armv9-a+memtag
|
||||
CFLAGS-tst-mte-realloc-static.o += -march=armv9-a+memtag
|
||||
|
||||
tests-exclude-malloc-check += $(tests-for-mte)
|
||||
tests-exclude-mcheck += $(tests-for-mte)
|
||||
tests-exclude-hugetlb1 += $(tests-for-mte)
|
||||
tests-exclude-hugetlb2 += $(tests-for-mte)
|
||||
endif # ifneq ($(multi-arch),no)
|
||||
|
||||
# Add MTE tunable to all malloc tests except malloc-check and mcheck variants
|
||||
tests-malloc-mte = \
|
||||
$(tests) \
|
||||
$(tests-malloc-hugetlb1:%=%-malloc-hugetlb1) \
|
||||
$(tests-malloc-hugetlb2:%=%-malloc-hugetlb2) \
|
||||
$(tests-malloc-largetcache:%=%-malloc-largetcache) \
|
||||
# tests-malloc-mte
|
||||
|
||||
define tests-malloc-mte-ENVS
|
||||
$(1)-TUNABLES += glibc.mem.aarch64_mte=sync
|
||||
endef
|
||||
|
||||
$(foreach t,$(tests-malloc-mte),$(eval $(call tests-malloc-mte-ENVS,$(t))))
|
||||
endif # ifeq ($(subdir),malloc)
|
||||
|
||||
abi-variants := lp64
|
||||
abi-variants += lp64_be
|
||||
|
||||
|
||||
@@ -59,6 +59,19 @@ get_midr_from_mcpu (const struct tunable_str_t *mcpu)
|
||||
return UINT64_MAX;
|
||||
}
|
||||
|
||||
static void
|
||||
TUNABLE_CALLBACK (set_aarch64_mte) (tunable_val_t *val)
|
||||
{
|
||||
if (tunable_strcmp_cte (val, "auto"))
|
||||
GL (dl_aarch64_mte) = MTE_TUNABLE_AUTO;
|
||||
else if (tunable_strcmp_cte (val, "sync"))
|
||||
GL (dl_aarch64_mte) = MTE_TUNABLE_SYNC;
|
||||
else if (tunable_strcmp_cte (val, "async"))
|
||||
GL (dl_aarch64_mte) = MTE_TUNABLE_ASYNC;
|
||||
else
|
||||
GL (dl_aarch64_mte) = MTE_TUNABLE_NONE;
|
||||
}
|
||||
|
||||
static inline void
|
||||
init_cpu_features (struct cpu_features *cpu_features)
|
||||
{
|
||||
@@ -95,6 +108,14 @@ init_cpu_features (struct cpu_features *cpu_features)
|
||||
if (cpu_features->bti)
|
||||
GLRO (dl_aarch64_bti) = TUNABLE_GET (glibc, cpu, aarch64_bti, uint64_t, 0);
|
||||
|
||||
/* Check if MTE is supported. */
|
||||
cpu_features->mte = GLRO (dl_hwcap2) & HWCAP2_MTE;
|
||||
if (cpu_features->mte)
|
||||
TUNABLE_GET (glibc, mem, aarch64_mte, tunable_val_t *,
|
||||
TUNABLE_CALLBACK (set_aarch64_mte));
|
||||
else
|
||||
GL (dl_aarch64_mte) = MTE_TUNABLE_NONE;
|
||||
|
||||
/* Check if SVE is supported. */
|
||||
cpu_features->sve = GLRO (dl_hwcap) & HWCAP_SVE;
|
||||
cpu_features->sve2 = GLRO (dl_hwcap2) & HWCAP2_SVE2;
|
||||
|
||||
@@ -35,3 +35,19 @@ PROCINFO_CLASS unsigned long _dl_aarch64_gcs
|
||||
,
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#if !IS_IN (ldconfig)
|
||||
# if !defined PROCINFO_DECL && defined SHARED
|
||||
._dl_aarch64_mte
|
||||
# else
|
||||
PROCINFO_CLASS int _dl_aarch64_mte
|
||||
# endif
|
||||
# ifndef PROCINFO_DECL
|
||||
= 0
|
||||
# endif
|
||||
# if !defined SHARED || defined PROCINFO_DECL
|
||||
;
|
||||
# else
|
||||
,
|
||||
# endif
|
||||
#endif
|
||||
|
||||
@@ -36,6 +36,9 @@
|
||||
# define GCS_POLICY_OPTIONAL 2
|
||||
# endif
|
||||
|
||||
void __mte_init (void);
|
||||
rtld_hidden_proto (__mte_init)
|
||||
|
||||
/* Must be on a top-level stack frame that does not return. */
|
||||
static inline void __attribute__((always_inline))
|
||||
aarch64_libc_setup_tls (void)
|
||||
@@ -72,6 +75,9 @@ aarch64_libc_setup_tls (void)
|
||||
_dl_fatal_printf ("failed to lock GCS: %d\n", -ret);
|
||||
}
|
||||
}
|
||||
|
||||
__mte_init ();
|
||||
|
||||
}
|
||||
|
||||
# define ARCH_SETUP_IREL() apply_irel ()
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
/* AArch64 test helper functions for MTE.
|
||||
Copyright (C) 2026 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#ifndef TST_MTE_HELPER_H
|
||||
#define TST_MTE_HELPER_H
|
||||
|
||||
#include <support/check.h>
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#include <malloc.h>
|
||||
#include <sys/auxv.h>
|
||||
#include <sys/prctl.h>
|
||||
|
||||
#define GRANULE_SIZE 16
|
||||
|
||||
/* Extract logical tag from pointer PTR. */
|
||||
static __always_inline
|
||||
uint64_t get_logical_tag (const void *ptr)
|
||||
{
|
||||
uint64_t t = (uint64_t)ptr;
|
||||
return t >> 56ul & 0xf;
|
||||
}
|
||||
|
||||
/* Load allocation tag from memory pointed-to by the PTR pointer. */
|
||||
static __always_inline
|
||||
uint64_t get_allocation_tag (const void *ptr)
|
||||
{
|
||||
uint64_t t;
|
||||
asm volatile ("ldg %0, [%1]" : "=r" (t) : "r" (ptr));
|
||||
return t >> 56ul & 0xf;
|
||||
}
|
||||
|
||||
/* Read the Tag Check Override bit. */
|
||||
static __always_inline
|
||||
uint64_t get_pstate_tco (void) {
|
||||
uint64_t t;
|
||||
asm volatile ("mrs %0, tco" : "=r" (t));
|
||||
return t;
|
||||
}
|
||||
|
||||
static __always_inline
|
||||
bool check_tags (void *tm)
|
||||
{
|
||||
size_t len = malloc_usable_size (tm);
|
||||
TEST_VERIFY (len % GRANULE_SIZE == 0);
|
||||
|
||||
uint64_t ltag = get_logical_tag (tm);
|
||||
TEST_VERIFY (ltag != 0);
|
||||
|
||||
for (size_t offset = 0; offset < len; offset += GRANULE_SIZE)
|
||||
{
|
||||
const char *g = (char *)tm + offset;
|
||||
uint64_t atag = get_allocation_tag (g);
|
||||
TEST_COMPARE (ltag, atag);
|
||||
if (ltag != atag)
|
||||
{
|
||||
printf ("tagged ptr: %p usable size: %zu\n", tm, len);
|
||||
printf ("tags mismatch at offset %zu: logical=%lu, allocation=%lu\n",
|
||||
offset, ltag, atag);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return ltag != 0;
|
||||
}
|
||||
|
||||
static __always_inline
|
||||
void check_mte_enabled (void)
|
||||
{
|
||||
/* Check if MTE is supported. */
|
||||
if (!(getauxval (AT_HWCAP2) & HWCAP2_MTE))
|
||||
FAIL_UNSUPPORTED ("kernel or CPU does not support HWCAP2_MTE");
|
||||
|
||||
/* Check if Tag Check Override bit is set. */
|
||||
if (get_pstate_tco () != 0)
|
||||
FAIL_UNSUPPORTED ("MTE tag check override is enabled");
|
||||
|
||||
/* Check applied MTE params. */
|
||||
uint64_t x = (uint64_t) prctl (PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0);
|
||||
uint64_t status = (x & 1ul);
|
||||
uint64_t mode = (x & PR_MTE_TCF_MASK) >> PR_MTE_TCF_SHIFT;
|
||||
uint64_t tags = (x & PR_MTE_TAG_MASK) >> PR_MTE_TAG_SHIFT;
|
||||
|
||||
printf ("MTE status: %4lx\n", status);
|
||||
printf ("MTE mode: %4lx\n", mode);
|
||||
printf ("MTE tags: %4lx\n", tags);
|
||||
|
||||
/* This test should be run in sync mode for tag checks. */
|
||||
TEST_VERIFY (status == 1);
|
||||
TEST_VERIFY (mode == PR_MTE_TCF_SYNC >> PR_MTE_TCF_SHIFT);
|
||||
TEST_VERIFY (tags == 0xfffe);
|
||||
}
|
||||
|
||||
#endif // TST_MTE_HELPER_H
|
||||
@@ -0,0 +1 @@
|
||||
#include "tst-mte-malloc.c"
|
||||
@@ -0,0 +1,154 @@
|
||||
/* AArch64 tests for heap memory tagging.
|
||||
Copyright (C) 2026 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <support/check.h>
|
||||
#include <support/support.h>
|
||||
#include <support/xsignal.h>
|
||||
#include <support/test-driver.h>
|
||||
#include <array_length.h>
|
||||
|
||||
#include "tst-mte-helper.h"
|
||||
|
||||
/* Characteristic malloc sizes to cover various allocation methods. */
|
||||
size_t sizes[] = {
|
||||
1,
|
||||
16, 40, 64, 120,
|
||||
128, 500, 1000,
|
||||
1050, 4096, 5000, 65000,
|
||||
131072, 2000000
|
||||
};
|
||||
|
||||
static void check_malloc (size_t len)
|
||||
{
|
||||
printf ("testing malloc with req size %zu\n", len);
|
||||
void *tm = malloc (len);
|
||||
if (!check_tags (tm))
|
||||
printf ("tagged pointer?: %016lx\n", (uintptr_t)tm);
|
||||
free (tm);
|
||||
}
|
||||
|
||||
static void check_calloc (size_t len)
|
||||
{
|
||||
size_t num = len / sizeof (uint64_t) + 1;
|
||||
printf ("testing calloc with req size %zu\n", num * sizeof (uint64_t));
|
||||
uint64_t *tm = calloc (num, sizeof (uint64_t));
|
||||
if (check_tags (tm))
|
||||
for (int n = 0; n < num; n ++)
|
||||
TEST_VERIFY_EXIT (tm[n] == 0);
|
||||
else
|
||||
printf ("tagged pointer?: %016lx\n", (uintptr_t)tm);
|
||||
free (tm);
|
||||
}
|
||||
|
||||
static void check_memalign (size_t len, size_t alignment)
|
||||
{
|
||||
printf ("testing memalign(%zu) with req size %zu\n", alignment, len);
|
||||
void *tm = memalign (alignment, len);
|
||||
if (!check_tags (tm))
|
||||
printf ("tagged pointer?: %016lx\n", (uintptr_t)tm);
|
||||
free (tm);
|
||||
}
|
||||
|
||||
static void check_valloc (size_t len)
|
||||
{
|
||||
printf ("testing valloc with req size %zu\n", len);
|
||||
void *tm = valloc (len);
|
||||
if (!check_tags (tm))
|
||||
printf ("tagged pointer?: %016lx\n", (uintptr_t)tm);
|
||||
free_sized (tm, len);
|
||||
}
|
||||
|
||||
static void check_pvalloc (size_t len)
|
||||
{
|
||||
printf ("testing pvalloc with req size %zu\n", len);
|
||||
void *tm = pvalloc (len);
|
||||
if (!check_tags (tm))
|
||||
printf ("tagged pointer?: %016lx\n", (uintptr_t)tm);
|
||||
free_sized (tm, len);
|
||||
}
|
||||
|
||||
static void check_posix_memalign (size_t len, size_t alignment)
|
||||
{
|
||||
printf ("testing posix_memalign(%zu) with req size %zu\n", alignment, len);
|
||||
void *p = NULL;
|
||||
int err = posix_memalign (&p, alignment, len);
|
||||
if (err)
|
||||
perror ("posix_memalign");
|
||||
TEST_VERIFY (p != NULL);
|
||||
TEST_VERIFY (err == 0);
|
||||
if (!check_tags (p))
|
||||
printf ("tagged pointer?: %016lx\n", (uintptr_t)p);
|
||||
free_aligned_sized (p, alignment, len);
|
||||
}
|
||||
|
||||
static void check_aligned_alloc (size_t len, size_t alignment)
|
||||
{
|
||||
printf ("testing aligned_alloc(%zu) with req size %zu\n", alignment, len);
|
||||
void *tm = aligned_alloc (alignment, len);
|
||||
if (!check_tags (tm))
|
||||
printf ("tagged pointer?: %016lx\n", (uintptr_t)tm);
|
||||
free_aligned_sized (tm, alignment, len);
|
||||
}
|
||||
|
||||
static int
|
||||
do_test (void)
|
||||
{
|
||||
/* Check if MTE is supported, configured and enabled. */
|
||||
check_mte_enabled ();
|
||||
|
||||
array_foreach_const (plen, sizes)
|
||||
check_malloc (*plen);
|
||||
|
||||
array_foreach_const (plen, sizes)
|
||||
check_calloc (*plen);
|
||||
|
||||
array_foreach_const (plen, sizes)
|
||||
{
|
||||
check_memalign (*plen, 2);
|
||||
check_memalign (*plen, 4);
|
||||
check_memalign (*plen, 8);
|
||||
check_memalign (*plen, 16);
|
||||
check_memalign (*plen, 32);
|
||||
}
|
||||
|
||||
array_foreach_const (plen, sizes)
|
||||
check_valloc (*plen);
|
||||
|
||||
array_foreach_const (plen, sizes)
|
||||
check_pvalloc (*plen);
|
||||
|
||||
array_foreach_const (plen, sizes)
|
||||
{
|
||||
check_posix_memalign (*plen, sizeof (void *) * 1);
|
||||
check_posix_memalign (*plen, sizeof (void *) * 2);
|
||||
check_posix_memalign (*plen, sizeof (void *) * 4);
|
||||
}
|
||||
|
||||
array_foreach_const (plen, sizes)
|
||||
{
|
||||
check_aligned_alloc (*plen, 2);
|
||||
check_aligned_alloc (*plen, 4);
|
||||
check_aligned_alloc (*plen, 8);
|
||||
check_aligned_alloc (*plen, 16);
|
||||
check_aligned_alloc (*plen, 32);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
#include <support/test-driver.c>
|
||||
@@ -0,0 +1 @@
|
||||
#include "tst-mte-realloc.c"
|
||||
@@ -0,0 +1,72 @@
|
||||
/* AArch64 tests for heap memory tagging.
|
||||
Copyright (C) 2026 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <support/check.h>
|
||||
#include <support/support.h>
|
||||
#include <support/xsignal.h>
|
||||
#include <support/test-driver.h>
|
||||
#include <array_length.h>
|
||||
|
||||
#include "tst-mte-helper.h"
|
||||
|
||||
/* Characteristic malloc sizes to cover various allocation methods. */
|
||||
size_t sizes[] = {
|
||||
1,
|
||||
16, 40, 64, 120,
|
||||
128, 500, 1000,
|
||||
1050, 4096, 5000, 65000,
|
||||
131072, 2000000
|
||||
};
|
||||
|
||||
static void check_realloc (size_t len)
|
||||
{
|
||||
/* Tagged pointers. */
|
||||
void *tm, *new_tm;
|
||||
|
||||
printf ("testing realloc (NULL) for req size %zu\n", len);
|
||||
tm = realloc (NULL, len);
|
||||
check_tags (tm);
|
||||
|
||||
/* Reduce size. */
|
||||
printf ("testing realloc (decreased size) for req size %zu\n", len);
|
||||
new_tm = realloc (tm, len / 2 + 1);
|
||||
check_tags (new_tm);
|
||||
|
||||
/* Increase size. */
|
||||
printf ("testing realloc (increased size) for req size %zu\n", len);
|
||||
new_tm = realloc (new_tm, len + 2);
|
||||
check_tags (new_tm);
|
||||
|
||||
free (new_tm);
|
||||
}
|
||||
|
||||
static int
|
||||
do_test (void)
|
||||
{
|
||||
|
||||
/* Check if MTE is supported, configured and enabled. */
|
||||
check_mte_enabled ();
|
||||
|
||||
array_foreach_const (plen, sizes)
|
||||
check_realloc (*plen);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
#include <support/test-driver.c>
|
||||
|
||||
Reference in New Issue
Block a user