AuroraMiddleware/glibc
1
0
Fork 0
mirror of https://sourceware.org/git/glibc.git synced 2024-11-09 23:00:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

x86-64: Use ZMM16-ZMM31 in AVX512 memmove family functions

Browse Source 
Update ifunc-memmove.h to select the function optimized with AVX512
instructions using ZMM16-ZMM31 registers to avoid RTM abort with usable
AVX512VL since VZEROUPPER isn't needed at function exit.
This commit is contained in:
H.J. Lu 2021-03-07 09:45:23 -08:00
parent 4e2d8f3527
commit e4fda46310
3 changed files with 42 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
sysdeps/x86_64/multiarch/ifunc-impl-list.c
View File

@ -83,10 +83,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
CPU_FEATURE_USABLE (AVX512F), CPU_FEATURE_USABLE (AVX512F),
__memmove_chk_avx512_no_vzeroupper) __memmove_chk_avx512_no_vzeroupper)
IFUNC_IMPL_ADD (array, i, __memmove_chk, IFUNC_IMPL_ADD (array, i, __memmove_chk,
CPU_FEATURE_USABLE (AVX512F), CPU_FEATURE_USABLE (AVX512VL),
__memmove_chk_avx512_unaligned) __memmove_chk_avx512_unaligned)
IFUNC_IMPL_ADD (array, i, __memmove_chk, IFUNC_IMPL_ADD (array, i, __memmove_chk,
CPU_FEATURE_USABLE (AVX512F), CPU_FEATURE_USABLE (AVX512VL),
__memmove_chk_avx512_unaligned_erms) __memmove_chk_avx512_unaligned_erms)
IFUNC_IMPL_ADD (array, i, __memmove_chk, IFUNC_IMPL_ADD (array, i, __memmove_chk,
CPU_FEATURE_USABLE (AVX), CPU_FEATURE_USABLE (AVX),
@ -148,10 +148,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
CPU_FEATURE_USABLE (AVX512F), CPU_FEATURE_USABLE (AVX512F),
__memmove_avx512_no_vzeroupper) __memmove_avx512_no_vzeroupper)
IFUNC_IMPL_ADD (array, i, memmove, IFUNC_IMPL_ADD (array, i, memmove,
CPU_FEATURE_USABLE (AVX512F), CPU_FEATURE_USABLE (AVX512VL),
__memmove_avx512_unaligned) __memmove_avx512_unaligned)
IFUNC_IMPL_ADD (array, i, memmove, IFUNC_IMPL_ADD (array, i, memmove,
CPU_FEATURE_USABLE (AVX512F), CPU_FEATURE_USABLE (AVX512VL),
__memmove_avx512_unaligned_erms) __memmove_avx512_unaligned_erms)
IFUNC_IMPL_ADD (array, i, memmove, CPU_FEATURE_USABLE (SSSE3), IFUNC_IMPL_ADD (array, i, memmove, CPU_FEATURE_USABLE (SSSE3),
__memmove_ssse3_back) __memmove_ssse3_back)
@ -733,10 +733,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
CPU_FEATURE_USABLE (AVX512F), CPU_FEATURE_USABLE (AVX512F),
__memcpy_chk_avx512_no_vzeroupper) __memcpy_chk_avx512_no_vzeroupper)
IFUNC_IMPL_ADD (array, i, __memcpy_chk, IFUNC_IMPL_ADD (array, i, __memcpy_chk,
CPU_FEATURE_USABLE (AVX512F), CPU_FEATURE_USABLE (AVX512VL),
__memcpy_chk_avx512_unaligned) __memcpy_chk_avx512_unaligned)
IFUNC_IMPL_ADD (array, i, __memcpy_chk, IFUNC_IMPL_ADD (array, i, __memcpy_chk,
CPU_FEATURE_USABLE (AVX512F), CPU_FEATURE_USABLE (AVX512VL),
__memcpy_chk_avx512_unaligned_erms) __memcpy_chk_avx512_unaligned_erms)
IFUNC_IMPL_ADD (array, i, __memcpy_chk, IFUNC_IMPL_ADD (array, i, __memcpy_chk,
CPU_FEATURE_USABLE (AVX), CPU_FEATURE_USABLE (AVX),
@ -802,10 +802,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
CPU_FEATURE_USABLE (AVX512F), CPU_FEATURE_USABLE (AVX512F),
__memcpy_avx512_no_vzeroupper) __memcpy_avx512_no_vzeroupper)
IFUNC_IMPL_ADD (array, i, memcpy, IFUNC_IMPL_ADD (array, i, memcpy,
CPU_FEATURE_USABLE (AVX512F), CPU_FEATURE_USABLE (AVX512VL),
__memcpy_avx512_unaligned) __memcpy_avx512_unaligned)
IFUNC_IMPL_ADD (array, i, memcpy, IFUNC_IMPL_ADD (array, i, memcpy,
CPU_FEATURE_USABLE (AVX512F), CPU_FEATURE_USABLE (AVX512VL),
__memcpy_avx512_unaligned_erms) __memcpy_avx512_unaligned_erms)
IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_sse2_unaligned) IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_sse2_unaligned)
IFUNC_IMPL_ADD (array, i, memcpy, 1, IFUNC_IMPL_ADD (array, i, memcpy, 1,
@ -819,10 +819,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
CPU_FEATURE_USABLE (AVX512F), CPU_FEATURE_USABLE (AVX512F),
__mempcpy_chk_avx512_no_vzeroupper) __mempcpy_chk_avx512_no_vzeroupper)
IFUNC_IMPL_ADD (array, i, __mempcpy_chk, IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
CPU_FEATURE_USABLE (AVX512F), CPU_FEATURE_USABLE (AVX512VL),
__mempcpy_chk_avx512_unaligned) __mempcpy_chk_avx512_unaligned)
IFUNC_IMPL_ADD (array, i, __mempcpy_chk, IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
CPU_FEATURE_USABLE (AVX512F), CPU_FEATURE_USABLE (AVX512VL),
__mempcpy_chk_avx512_unaligned_erms) __mempcpy_chk_avx512_unaligned_erms)
IFUNC_IMPL_ADD (array, i, __mempcpy_chk, IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
CPU_FEATURE_USABLE (AVX), CPU_FEATURE_USABLE (AVX),
@ -864,10 +864,10 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
CPU_FEATURE_USABLE (AVX512F), CPU_FEATURE_USABLE (AVX512F),
__mempcpy_avx512_no_vzeroupper) __mempcpy_avx512_no_vzeroupper)
IFUNC_IMPL_ADD (array, i, mempcpy, IFUNC_IMPL_ADD (array, i, mempcpy,
CPU_FEATURE_USABLE (AVX512F), CPU_FEATURE_USABLE (AVX512VL),
__mempcpy_avx512_unaligned) __mempcpy_avx512_unaligned)
IFUNC_IMPL_ADD (array, i, mempcpy, IFUNC_IMPL_ADD (array, i, mempcpy,
CPU_FEATURE_USABLE (AVX512F), CPU_FEATURE_USABLE (AVX512VL),
__mempcpy_avx512_unaligned_erms) __mempcpy_avx512_unaligned_erms)
IFUNC_IMPL_ADD (array, i, mempcpy, IFUNC_IMPL_ADD (array, i, mempcpy,
CPU_FEATURE_USABLE (AVX), CPU_FEATURE_USABLE (AVX),

12
sysdeps/x86_64/multiarch/ifunc-memmove.h
View File

@ -56,13 +56,15 @@ IFUNC_SELECTOR (void)
if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F) if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F)
&& !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512)) && !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512))
{ {
if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL))
return OPTIMIZE (avx512_no_vzeroupper); {
if (CPU_FEATURE_USABLE_P (cpu_features, ERMS))
return OPTIMIZE (avx512_unaligned_erms);
if (CPU_FEATURE_USABLE_P (cpu_features, ERMS)) return OPTIMIZE (avx512_unaligned);
return OPTIMIZE (avx512_unaligned_erms); }
return OPTIMIZE (avx512_unaligned); return OPTIMIZE (avx512_no_vzeroupper);
} }
if (CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) if (CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))

25
sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S
View File

@ -1,11 +1,32 @@
#if IS_IN (libc) #if IS_IN (libc)
# define VEC_SIZE 64 # define VEC_SIZE 64
# define VEC(i) zmm##i # define XMM0 xmm16
# define XMM1 xmm17
# define YMM0 ymm16
# define YMM1 ymm17
# define VEC0 zmm16
# define VEC1 zmm17
# define VEC2 zmm18
# define VEC3 zmm19
# define VEC4 zmm20
# define VEC5 zmm21
# define VEC6 zmm22
# define VEC7 zmm23
# define VEC8 zmm24
# define VEC9 zmm25
# define VEC10 zmm26
# define VEC11 zmm27
# define VEC12 zmm28
# define VEC13 zmm29
# define VEC14 zmm30
# define VEC15 zmm31
# define VEC(i) VEC##i
# define VMOVNT vmovntdq # define VMOVNT vmovntdq
# define VMOVU vmovdqu64 # define VMOVU vmovdqu64
# define VMOVA vmovdqa64 # define VMOVA vmovdqa64
# define VZEROUPPER
# define SECTION(p) p##.avx512 # define SECTION(p) p##.evex512
# define MEMMOVE_SYMBOL(p,s) p##_avx512_##s # define MEMMOVE_SYMBOL(p,s) p##_avx512_##s
# include "memmove-vec-unaligned-erms.S" # include "memmove-vec-unaligned-erms.S"