mirror of
https://sourceware.org/git/glibc.git
synced 2024-12-25 20:21:07 +00:00
935971ba6b
Optimize x86-64 memcmp/wmemcmp with AVX2. It uses vector compare as much as possible. It is as fast as SSE4 memcmp for size <= 16 bytes and up to 2X faster for size > 16 bytes on Haswell and Skylake. Select AVX2 memcmp/wmemcmp on AVX2 machines where vzeroupper is preferred and AVX unaligned load is fast. NB: It uses TZCNT instead of BSF since TZCNT produces the same result as BSF for non-zero input. TZCNT is faster than BSF and is executed as BSF if machine doesn't support TZCNT. Key features: 1. For size from 2 to 7 bytes, load as big endian with movbe and bswap to avoid branches. 2. Use overlapping compare to avoid branch. 3. Use vector compare when size >= 4 bytes for memcmp or size >= 8 bytes for wmemcmp. 4. If size is 8 * VEC_SIZE or less, unroll the loop. 5. Compare 4 * VEC_SIZE at a time with the aligned first memory area. 6. Use 2 vector compares when size is 2 * VEC_SIZE or less. 7. Use 4 vector compares when size is 4 * VEC_SIZE or less. 8. Use 8 vector compares when size is 8 * VEC_SIZE or less. * sysdeps/x86/cpu-features.h (index_cpu_MOVBE): New. * sysdeps/x86_64/multiarch/Makefile (sysdep_routines): Add memcmp-avx2 and wmemcmp-avx2. * sysdeps/x86_64/multiarch/ifunc-impl-list.c (__libc_ifunc_impl_list): Test __memcmp_avx2 and __wmemcmp_avx2. * sysdeps/x86_64/multiarch/memcmp-avx2.S: New file. * sysdeps/x86_64/multiarch/wmemcmp-avx2.S: Likewise. * sysdeps/x86_64/multiarch/memcmp.S: Use __memcmp_avx2 on AVX 2 machines if AVX unaligned load is fast and vzeroupper is preferred. * sysdeps/x86_64/multiarch/wmemcmp.S: Use __wmemcmp_avx2 on AVX 2 machines if AVX unaligned load is fast and vzeroupper is preferred.
42 lines
1.2 KiB
Makefile
42 lines
1.2 KiB
Makefile
ifeq ($(subdir),csu)
|
|
tests += test-multiarch
|
|
endif
|
|
|
|
ifeq ($(subdir),string)
|
|
|
|
sysdep_routines += strncat-c stpncpy-c strncpy-c strcmp-ssse3 \
|
|
strcmp-sse2-unaligned strncmp-ssse3 \
|
|
memcmp-avx2-movbe \
|
|
memcmp-sse4 memcpy-ssse3 \
|
|
memmove-ssse3 \
|
|
memcpy-ssse3-back \
|
|
memmove-ssse3-back \
|
|
memmove-avx512-no-vzeroupper strcasecmp_l-ssse3 \
|
|
strncase_l-ssse3 strcat-ssse3 strncat-ssse3\
|
|
strcpy-ssse3 strncpy-ssse3 stpcpy-ssse3 stpncpy-ssse3 \
|
|
strcpy-sse2-unaligned strncpy-sse2-unaligned \
|
|
stpcpy-sse2-unaligned stpncpy-sse2-unaligned \
|
|
strcat-sse2-unaligned strncat-sse2-unaligned \
|
|
strchr-sse2-no-bsf memcmp-ssse3 strstr-sse2-unaligned \
|
|
strcspn-c strpbrk-c strspn-c varshift \
|
|
memset-avx512-no-vzeroupper \
|
|
memmove-avx-unaligned-erms \
|
|
memmove-avx512-unaligned-erms \
|
|
memset-avx2-unaligned-erms \
|
|
memset-avx512-unaligned-erms
|
|
CFLAGS-varshift.c += -msse4
|
|
CFLAGS-strcspn-c.c += -msse4
|
|
CFLAGS-strpbrk-c.c += -msse4
|
|
CFLAGS-strspn-c.c += -msse4
|
|
endif
|
|
|
|
ifeq ($(subdir),wcsmbs)
|
|
sysdep_routines += wmemcmp-sse4 wmemcmp-ssse3 wmemcmp-c \
|
|
wmemcmp-avx2-movbe \
|
|
wcscpy-ssse3 wcscpy-c
|
|
endif
|
|
|
|
ifeq ($(subdir),debug)
|
|
sysdep_routines += wmemset_chk-nonshared
|
|
endif
|