mirror of git://sourceware.org/git/glibc.git
linux: Inline __syscall_internal_cancel and __syscall_cancel
It improves some interception tools such as valgrind, however on
multithread the __syscall_cancel_arch is called.
The result libc.so has a slight larger code size:
ABI master patched diff increase
aarch64 1658673 1669121 10448 0.63%
x86_64 1976656 1985744 9088 0.46%
i686 2233622 2251130 17508 0.78%
powerpc64le 2382448 2396768 14320 0.60%
It mimics internally how cancellable entrypoints were implemented
before 89b53077d2, where cancellation
handlign were done inline in the syscall wraper.
This commit is contained in:
Adhemerval Zanella
parent
f66cb3c9eb
commit
df92e9d71d
|
|
@ -1467,7 +1467,9 @@ $(objpfx)dl-allobjs.os: $(all-rtld-routines:%=$(objpfx)%.os)
|
|||
# when compiled for libc.
|
||||
rtld-stubbed-symbols = \
|
||||
__libc_assert_fail \
|
||||
__syscall_cancel \
|
||||
__libc_single_threaded_internal \
|
||||
__syscall_cancel_arch \
|
||||
__syscall_do_cancel \
|
||||
calloc \
|
||||
free \
|
||||
malloc \
|
||||
|
|
|
|||
|
|
@ -19,66 +19,6 @@
|
|||
#include <stdlib.h>
|
||||
#include "pthreadP.h"
|
||||
|
||||
/* Called by the INTERNAL_SYSCALL_CANCEL macro, check for cancellation and
|
||||
returns the syscall value or its negative error code. */
|
||||
long int
|
||||
__internal_syscall_cancel (__syscall_arg_t a1, __syscall_arg_t a2,
|
||||
__syscall_arg_t a3, __syscall_arg_t a4,
|
||||
__syscall_arg_t a5, __syscall_arg_t a6,
|
||||
__SYSCALL_CANCEL7_ARG_DEF
|
||||
__syscall_arg_t nr)
|
||||
{
|
||||
long int result;
|
||||
struct pthread *pd = THREAD_SELF;
|
||||
|
||||
/* If cancellation is not enabled, call the syscall directly and also
|
||||
for thread terminatation to avoid call __syscall_do_cancel while
|
||||
executing cleanup handlers. */
|
||||
int ch = atomic_load_relaxed (&pd->cancelhandling);
|
||||
if (SINGLE_THREAD_P || !cancel_enabled (ch) || cancel_exiting (ch))
|
||||
{
|
||||
result = INTERNAL_SYSCALL_NCS_CALL (nr, a1, a2, a3, a4, a5, a6
|
||||
__SYSCALL_CANCEL7_ARCH_ARG7);
|
||||
if (INTERNAL_SYSCALL_ERROR_P (result))
|
||||
return -INTERNAL_SYSCALL_ERRNO (result);
|
||||
return result;
|
||||
}
|
||||
|
||||
/* Call the arch-specific entry points that contains the globals markers
|
||||
to be checked by SIGCANCEL handler. */
|
||||
result = __syscall_cancel_arch (&pd->cancelhandling, nr, a1, a2, a3, a4, a5,
|
||||
a6 __SYSCALL_CANCEL7_ARCH_ARG7);
|
||||
|
||||
/* If the cancellable syscall was interrupted by SIGCANCEL and it has no
|
||||
side-effect, cancel the thread if cancellation is enabled. */
|
||||
ch = atomic_load_relaxed (&pd->cancelhandling);
|
||||
/* The behaviour here assumes that EINTR is returned only if there are no
|
||||
visible side effects. POSIX Issue 7 has not yet provided any stronger
|
||||
language for close, and in theory the close syscall could return EINTR
|
||||
and leave the file descriptor open (conforming and leaks). It expects
|
||||
that no such kernel is used with glibc. */
|
||||
if (result == -EINTR && cancel_enabled_and_canceled (ch))
|
||||
__syscall_do_cancel ();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/* Called by the SYSCALL_CANCEL macro, check for cancellation and return the
|
||||
syscall expected success value (usually 0) or, in case of failure, -1 and
|
||||
sets errno to syscall return value. */
|
||||
long int
|
||||
__syscall_cancel (__syscall_arg_t a1, __syscall_arg_t a2,
|
||||
__syscall_arg_t a3, __syscall_arg_t a4,
|
||||
__syscall_arg_t a5, __syscall_arg_t a6,
|
||||
__SYSCALL_CANCEL7_ARG_DEF __syscall_arg_t nr)
|
||||
{
|
||||
int r = __internal_syscall_cancel (a1, a2, a3, a4, a5, a6,
|
||||
__SYSCALL_CANCEL7_ARG nr);
|
||||
return __glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (r))
|
||||
? SYSCALL_ERROR_LABEL (INTERNAL_SYSCALL_ERRNO (r))
|
||||
: r;
|
||||
}
|
||||
|
||||
/* Called by __syscall_cancel_arch or function above start the thread
|
||||
cancellation. */
|
||||
_Noreturn void
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <errno.h>
|
||||
#include <sysdep.h>
|
||||
#include <sysdep-cancel.h>
|
||||
#include <time.h>
|
||||
#include <futex-internal.h>
|
||||
#include <kernel-features.h>
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@
|
|||
|
||||
#include <kernel-features.h>
|
||||
#include <errno.h>
|
||||
#include <sysdep.h>
|
||||
#include <sysdep-cancel.h>
|
||||
#include <futex-internal.h>
|
||||
#include <internaltypes.h>
|
||||
#include <semaphore.h>
|
||||
|
|
|
|||
|
|
@ -154,42 +154,31 @@
|
|||
# define __SYSCALL_CANCEL7_ARG7
|
||||
# define __SYSCALL_CANCEL7_ARCH_ARG7
|
||||
#endif
|
||||
long int __internal_syscall_cancel (__syscall_arg_t a1, __syscall_arg_t a2,
|
||||
__syscall_arg_t a3, __syscall_arg_t a4,
|
||||
__syscall_arg_t a5, __syscall_arg_t a6,
|
||||
__SYSCALL_CANCEL7_ARG_DEF
|
||||
__syscall_arg_t nr) attribute_hidden;
|
||||
|
||||
long int __syscall_cancel (__syscall_arg_t arg1, __syscall_arg_t arg2,
|
||||
__syscall_arg_t arg3, __syscall_arg_t arg4,
|
||||
__syscall_arg_t arg5, __syscall_arg_t arg6,
|
||||
__SYSCALL_CANCEL7_ARG_DEF
|
||||
__syscall_arg_t nr) attribute_hidden;
|
||||
|
||||
#define __SYSCALL_CANCEL0(name) \
|
||||
__syscall_cancel (0, 0, 0, 0, 0, 0, __SYSCALL_CANCEL7_ARG __NR_##name)
|
||||
syscall_cancel (0, 0, 0, 0, 0, 0, __SYSCALL_CANCEL7_ARG __NR_##name)
|
||||
#define __SYSCALL_CANCEL1(name, a1) \
|
||||
__syscall_cancel (__SSC (a1), 0, 0, 0, 0, 0, \
|
||||
__SYSCALL_CANCEL7_ARG __NR_##name)
|
||||
syscall_cancel (__SSC (a1), 0, 0, 0, 0, 0, \
|
||||
__SYSCALL_CANCEL7_ARG __NR_##name)
|
||||
#define __SYSCALL_CANCEL2(name, a1, a2) \
|
||||
__syscall_cancel (__SSC (a1), __SSC (a2), 0, 0, 0, 0, \
|
||||
__SYSCALL_CANCEL7_ARG __NR_##name)
|
||||
syscall_cancel (__SSC (a1), __SSC (a2), 0, 0, 0, 0, \
|
||||
__SYSCALL_CANCEL7_ARG __NR_##name)
|
||||
#define __SYSCALL_CANCEL3(name, a1, a2, a3) \
|
||||
__syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), 0, 0, 0, \
|
||||
__SYSCALL_CANCEL7_ARG __NR_##name)
|
||||
syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), 0, 0, 0, \
|
||||
__SYSCALL_CANCEL7_ARG __NR_##name)
|
||||
#define __SYSCALL_CANCEL4(name, a1, a2, a3, a4) \
|
||||
__syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), \
|
||||
__SSC(a4), 0, 0, __SYSCALL_CANCEL7_ARG __NR_##name)
|
||||
syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), \
|
||||
__SSC(a4), 0, 0, __SYSCALL_CANCEL7_ARG __NR_##name)
|
||||
#define __SYSCALL_CANCEL5(name, a1, a2, a3, a4, a5) \
|
||||
__syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), __SSC(a4), \
|
||||
__SSC (a5), 0, __SYSCALL_CANCEL7_ARG __NR_##name)
|
||||
syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), __SSC(a4), \
|
||||
__SSC (a5), 0, __SYSCALL_CANCEL7_ARG __NR_##name)
|
||||
#define __SYSCALL_CANCEL6(name, a1, a2, a3, a4, a5, a6) \
|
||||
__syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), __SSC (a4), \
|
||||
__SSC (a5), __SSC (a6), __SYSCALL_CANCEL7_ARG \
|
||||
__NR_##name)
|
||||
syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), __SSC (a4), \
|
||||
__SSC (a5), __SSC (a6), __SYSCALL_CANCEL7_ARG \
|
||||
__NR_##name)
|
||||
#define __SYSCALL_CANCEL7(name, a1, a2, a3, a4, a5, a6, a7) \
|
||||
__syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), __SSC (a4), \
|
||||
__SSC (a5), __SSC (a6), __SSC (a7), __NR_##name)
|
||||
syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), __SSC (a4), \
|
||||
__SSC (a5), __SSC (a6), __SSC (a7), __NR_##name)
|
||||
|
||||
#define __SYSCALL_CANCEL_NARGS_X(a,b,c,d,e,f,g,h,n,...) n
|
||||
#define __SYSCALL_CANCEL_NARGS(...) \
|
||||
|
|
@ -206,33 +195,33 @@ long int __syscall_cancel (__syscall_arg_t arg1, __syscall_arg_t arg2,
|
|||
__SYSCALL_CANCEL_DISP (__SYSCALL_CANCEL, __VA_ARGS__)
|
||||
|
||||
#define __INTERNAL_SYSCALL_CANCEL0(name) \
|
||||
__internal_syscall_cancel (0, 0, 0, 0, 0, 0, __SYSCALL_CANCEL7_ARG \
|
||||
internal_syscall_cancel (0, 0, 0, 0, 0, 0, __SYSCALL_CANCEL7_ARG \
|
||||
__NR_##name)
|
||||
#define __INTERNAL_SYSCALL_CANCEL1(name, a1) \
|
||||
__internal_syscall_cancel (__SSC (a1), 0, 0, 0, 0, 0, \
|
||||
__SYSCALL_CANCEL7_ARG __NR_##name)
|
||||
internal_syscall_cancel (__SSC (a1), 0, 0, 0, 0, 0, \
|
||||
__SYSCALL_CANCEL7_ARG __NR_##name)
|
||||
#define __INTERNAL_SYSCALL_CANCEL2(name, a1, a2) \
|
||||
__internal_syscall_cancel (__SSC (a1), __SSC (a2), 0, 0, 0, 0, \
|
||||
__SYSCALL_CANCEL7_ARG __NR_##name)
|
||||
internal_syscall_cancel (__SSC (a1), __SSC (a2), 0, 0, 0, 0, \
|
||||
__SYSCALL_CANCEL7_ARG __NR_##name)
|
||||
#define __INTERNAL_SYSCALL_CANCEL3(name, a1, a2, a3) \
|
||||
__internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), 0, \
|
||||
0, 0, __SYSCALL_CANCEL7_ARG __NR_##name)
|
||||
internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), 0, \
|
||||
0, 0, __SYSCALL_CANCEL7_ARG __NR_##name)
|
||||
#define __INTERNAL_SYSCALL_CANCEL4(name, a1, a2, a3, a4) \
|
||||
__internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), \
|
||||
__SSC(a4), 0, 0, \
|
||||
__SYSCALL_CANCEL7_ARG __NR_##name)
|
||||
internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), \
|
||||
__SSC(a4), 0, 0, \
|
||||
__SYSCALL_CANCEL7_ARG __NR_##name)
|
||||
#define __INTERNAL_SYSCALL_CANCEL5(name, a1, a2, a3, a4, a5) \
|
||||
__internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), \
|
||||
__SSC(a4), __SSC (a5), 0, \
|
||||
__SYSCALL_CANCEL7_ARG __NR_##name)
|
||||
internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), \
|
||||
__SSC(a4), __SSC (a5), 0, \
|
||||
__SYSCALL_CANCEL7_ARG __NR_##name)
|
||||
#define __INTERNAL_SYSCALL_CANCEL6(name, a1, a2, a3, a4, a5, a6) \
|
||||
__internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), \
|
||||
__SSC (a4), __SSC (a5), __SSC (a6), \
|
||||
__SYSCALL_CANCEL7_ARG __NR_##name)
|
||||
internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), \
|
||||
__SSC (a4), __SSC (a5), __SSC (a6), \
|
||||
__SYSCALL_CANCEL7_ARG __NR_##name)
|
||||
#define __INTERNAL_SYSCALL_CANCEL7(name, a1, a2, a3, a4, a5, a6, a7) \
|
||||
__internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), \
|
||||
__SSC (a4), __SSC (a5), __SSC (a6), \
|
||||
__SSC (a7), __NR_##name)
|
||||
internal_syscall_cancel (__SSC (a1), __SSC (a2), __SSC (a3), \
|
||||
__SSC (a4), __SSC (a5), __SSC (a6), \
|
||||
__SSC (a7), __NR_##name)
|
||||
|
||||
/* Issue a cancellable syscall defined by syscall number NAME plus any other
|
||||
argument required. If an error occurs its value is returned as an negative
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <sys/epoll.h>
|
||||
#include <sysdep.h>
|
||||
#include <sysdep-cancel.h>
|
||||
|
||||
int
|
||||
__epoll_pwait2_time64 (int fd, struct epoll_event *ev, int maxev,
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@
|
|||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <sys/socket.h>
|
||||
#include <sysdep.h>
|
||||
#include <sysdep-cancel.h>
|
||||
#include <socketcall.h>
|
||||
|
||||
static int
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@
|
|||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <signal.h>
|
||||
#include <sysdep.h>
|
||||
#include <sysdep-cancel.h>
|
||||
|
||||
int
|
||||
__sigtimedwait64 (const sigset_t *set, siginfo_t *info,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* Single-thread optimization definitions. Linux version.
|
||||
/* Cancellable syscall definitions Linux version.
|
||||
Copyright (C) 2017-2025 Free Software Foundation, Inc.
|
||||
|
||||
This file is part of the GNU C Library.
|
||||
|
|
@ -21,5 +21,66 @@
|
|||
#define _SYSDEP_CANCEL_H
|
||||
|
||||
#include <sysdep.h>
|
||||
#include "pthreadP.h"
|
||||
|
||||
/* Called by the INTERNAL_SYSCALL_CANCEL macro, check for cancellation and
|
||||
returns the syscall value or its negative error code. */
|
||||
static __always_inline long int
|
||||
internal_syscall_cancel (__syscall_arg_t a1, __syscall_arg_t a2,
|
||||
__syscall_arg_t a3, __syscall_arg_t a4,
|
||||
__syscall_arg_t a5, __syscall_arg_t a6,
|
||||
__SYSCALL_CANCEL7_ARG_DEF
|
||||
__syscall_arg_t nr)
|
||||
{
|
||||
long int result;
|
||||
struct pthread *pd = THREAD_SELF;
|
||||
|
||||
/* If cancellation is not enabled, call the syscall directly and also
|
||||
for thread terminatation to avoid call __syscall_do_cancel while
|
||||
executing cleanup handlers. */
|
||||
int ch = atomic_load_relaxed (&pd->cancelhandling);
|
||||
if (SINGLE_THREAD_P || !cancel_enabled (ch) || cancel_exiting (ch))
|
||||
{
|
||||
result = INTERNAL_SYSCALL_NCS_CALL (nr, a1, a2, a3, a4, a5, a6
|
||||
__SYSCALL_CANCEL7_ARCH_ARG7);
|
||||
if (INTERNAL_SYSCALL_ERROR_P (result))
|
||||
return -INTERNAL_SYSCALL_ERRNO (result);
|
||||
return result;
|
||||
}
|
||||
|
||||
/* Call the arch-specific entry points that contains the globals markers
|
||||
to be checked by SIGCANCEL handler. */
|
||||
result = __syscall_cancel_arch (&pd->cancelhandling, nr, a1, a2, a3, a4, a5,
|
||||
a6 __SYSCALL_CANCEL7_ARCH_ARG7);
|
||||
|
||||
/* If the cancellable syscall was interrupted by SIGCANCEL and it has no
|
||||
side-effect, cancel the thread if cancellation is enabled. */
|
||||
ch = atomic_load_relaxed (&pd->cancelhandling);
|
||||
/* The behaviour here assumes that EINTR is returned only if there are no
|
||||
visible side effects. POSIX Issue 7 has not yet provided any stronger
|
||||
language for close, and in theory the close syscall could return EINTR
|
||||
and leave the file descriptor open (conforming and leaks). It expects
|
||||
that no such kernel is used with glibc. */
|
||||
if (result == -EINTR && cancel_enabled_and_canceled (ch))
|
||||
__syscall_do_cancel ();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/* Called by the SYSCALL_CANCEL macro, check for cancellation and return the
|
||||
syscall expected success value (usually 0) or, in case of failure, -1 and
|
||||
sets errno to syscall return value. */
|
||||
static __always_inline long int
|
||||
syscall_cancel (__syscall_arg_t a1, __syscall_arg_t a2,
|
||||
__syscall_arg_t a3, __syscall_arg_t a4,
|
||||
__syscall_arg_t a5, __syscall_arg_t a6,
|
||||
__SYSCALL_CANCEL7_ARG_DEF __syscall_arg_t nr)
|
||||
{
|
||||
int r = internal_syscall_cancel (a1, a2, a3, a4, a5, a6,
|
||||
__SYSCALL_CANCEL7_ARG nr);
|
||||
return __glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (r))
|
||||
? SYSCALL_ERROR_LABEL (INTERNAL_SYSCALL_ERRNO (r))
|
||||
: r;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
|
|||
Loading…
Reference in New Issue