qiurui
/
glibc
mirror of git://sourceware.org/git/glibc.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

x86-64: Use ZMM16-ZMM31 in AVX512 memset family functions 

Update ifunc-memset.h/ifunc-wmemset.h to select the function optimized
with AVX512 instructions using ZMM16-ZMM31 registers to avoid RTM abort
with usable AVX512VL and AVX512BW since VZEROUPPER isn't needed at
function exit.

(cherry picked from commit 4e2d8f3527)
This commit is contained in:
H.J. Lu 2021-03-07 09:44:18 -08:00
parent 5d5ef0cd7f
commit a47bf3df2f
4 changed files with 31 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
sysdeps/x86_64/multiarch/ifunc-impl-list.c
View File

@ -211,10 +211,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
&& HAS_ARCH_FEATURE (AVX512BW_Usable)), && HAS_ARCH_FEATURE (AVX512BW_Usable)),
__memset_chk_evex_unaligned_erms) __memset_chk_evex_unaligned_erms)
IFUNC_IMPL_ADD (array, i, __memset_chk, IFUNC_IMPL_ADD (array, i, __memset_chk,
HAS_ARCH_FEATURE (AVX512F_Usable), (HAS_ARCH_FEATURE (AVX512VL_Usable)
&& HAS_ARCH_FEATURE (AVX512BW_Usable)),
__memset_chk_avx512_unaligned_erms) __memset_chk_avx512_unaligned_erms)
IFUNC_IMPL_ADD (array, i, __memset_chk, IFUNC_IMPL_ADD (array, i, __memset_chk,
HAS_ARCH_FEATURE (AVX512F_Usable), (HAS_ARCH_FEATURE (AVX512VL_Usable)
&& HAS_ARCH_FEATURE (AVX512BW_Usable)),
__memset_chk_avx512_unaligned) __memset_chk_avx512_unaligned)
IFUNC_IMPL_ADD (array, i, __memset_chk, IFUNC_IMPL_ADD (array, i, __memset_chk,
HAS_ARCH_FEATURE (AVX512F_Usable), HAS_ARCH_FEATURE (AVX512F_Usable),
@ -252,10 +254,12 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
&& HAS_ARCH_FEATURE (AVX512BW_Usable)), && HAS_ARCH_FEATURE (AVX512BW_Usable)),
__memset_evex_unaligned_erms) __memset_evex_unaligned_erms)
IFUNC_IMPL_ADD (array, i, memset, IFUNC_IMPL_ADD (array, i, memset,
HAS_ARCH_FEATURE (AVX512F_Usable), (HAS_ARCH_FEATURE (AVX512VL_Usable)
&& HAS_ARCH_FEATURE (AVX512BW_Usable)),
__memset_avx512_unaligned_erms) __memset_avx512_unaligned_erms)
IFUNC_IMPL_ADD (array, i, memset, IFUNC_IMPL_ADD (array, i, memset,
HAS_ARCH_FEATURE (AVX512F_Usable), (HAS_ARCH_FEATURE (AVX512VL_Usable)
&& HAS_ARCH_FEATURE (AVX512BW_Usable)),
__memset_avx512_unaligned) __memset_avx512_unaligned)
IFUNC_IMPL_ADD (array, i, memset, IFUNC_IMPL_ADD (array, i, memset,
HAS_ARCH_FEATURE (AVX512F_Usable), HAS_ARCH_FEATURE (AVX512F_Usable),
@ -683,7 +687,7 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
HAS_ARCH_FEATURE (AVX512VL_Usable), HAS_ARCH_FEATURE (AVX512VL_Usable),
__wmemset_evex_unaligned) __wmemset_evex_unaligned)
IFUNC_IMPL_ADD (array, i, wmemset, IFUNC_IMPL_ADD (array, i, wmemset,
HAS_ARCH_FEATURE (AVX512F_Usable), HAS_ARCH_FEATURE (AVX512VL_Usable),
__wmemset_avx512_unaligned)) __wmemset_avx512_unaligned))
#ifdef SHARED #ifdef SHARED

13
sysdeps/x86_64/multiarch/ifunc-memset.h
View File

@ -53,13 +53,16 @@ IFUNC_SELECTOR (void)
if (CPU_FEATURES_ARCH_P (cpu_features, AVX512F_Usable) if (CPU_FEATURES_ARCH_P (cpu_features, AVX512F_Usable)
&& !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512)) && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512))
{ {
if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) if (CPU_FEATURES_ARCH_P (cpu_features, AVX512VL_Usable)
return OPTIMIZE (avx512_no_vzeroupper); && CPU_FEATURES_ARCH_P (cpu_features, AVX512BW_Usable))
{
if (CPU_FEATURES_CPU_P (cpu_features, ERMS))
return OPTIMIZE (avx512_unaligned_erms);
if (CPU_FEATURES_CPU_P (cpu_features, ERMS)) return OPTIMIZE (avx512_unaligned);
return OPTIMIZE (avx512_unaligned_erms); }
return OPTIMIZE (avx512_unaligned); return OPTIMIZE (avx512_no_vzeroupper);
} }
if (CPU_FEATURES_ARCH_P (cpu_features, AVX2_Usable)) if (CPU_FEATURES_ARCH_P (cpu_features, AVX2_Usable))

12
sysdeps/x86_64/multiarch/ifunc-wmemset.h
View File

@ -33,13 +33,13 @@ IFUNC_SELECTOR (void)
if (CPU_FEATURES_ARCH_P (cpu_features, AVX2_Usable) if (CPU_FEATURES_ARCH_P (cpu_features, AVX2_Usable)
&& CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
{ {
if (CPU_FEATURES_ARCH_P (cpu_features, AVX512F_Usable)
&& !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512)
&& !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER))
return OPTIMIZE (avx512_unaligned);
if (CPU_FEATURES_ARCH_P (cpu_features, AVX512VL_Usable)) if (CPU_FEATURES_ARCH_P (cpu_features, AVX512VL_Usable))
return OPTIMIZE (evex_unaligned); {
if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512))
return OPTIMIZE (avx512_unaligned);
return OPTIMIZE (evex_unaligned);
}
if (CPU_FEATURES_CPU_P (cpu_features, RTM)) if (CPU_FEATURES_CPU_P (cpu_features, RTM))
return OPTIMIZE (avx2_unaligned_rtm); return OPTIMIZE (avx2_unaligned_rtm);

16
sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S
View File

@ -1,22 +1,22 @@
#if IS_IN (libc) #if IS_IN (libc)
# define VEC_SIZE 64 # define VEC_SIZE 64
# define VEC(i) zmm##i # define XMM0 xmm16
# define YMM0 ymm16
# define VEC0 zmm16
# define VEC(i) VEC##i
# define VMOVU vmovdqu64 # define VMOVU vmovdqu64
# define VMOVA vmovdqa64 # define VMOVA vmovdqa64
# define VZEROUPPER
# define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ # define MEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \
vmovd d, %xmm0; \
movq r, %rax; \ movq r, %rax; \
vpbroadcastb %xmm0, %xmm0; \ vpbroadcastb d, %VEC0
vpbroadcastq %xmm0, %zmm0
# define WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \ # define WMEMSET_VDUP_TO_VEC0_AND_SET_RETURN(d, r) \
vmovd d, %xmm0; \
movq r, %rax; \ movq r, %rax; \
vpbroadcastd %xmm0, %xmm0; \ vpbroadcastd d, %VEC0
vpbroadcastq %xmm0, %zmm0
# define SECTION(p) p##.avx512 # define SECTION(p) p##.evex512
# define MEMSET_SYMBOL(p,s) p##_avx512_##s # define MEMSET_SYMBOL(p,s) p##_avx512_##s
# define WMEMSET_SYMBOL(p,s) p##_avx512_##s # define WMEMSET_SYMBOL(p,s) p##_avx512_##s