mirror of
https://sourceware.org/git/glibc.git
synced 2025-01-13 04:30:07 +00:00
x86_64: Add support for __memcmpeq using sse2, avx2, and evex
No bug. This commit adds support for __memcmpeq to be implemented seperately from memcmp. Support is added for versions optimized with sse2, avx2, and evex.
This commit is contained in:
parent
cf3acd774f
commit
cf4fd28ea4
@ -50,5 +50,8 @@
|
||||
'__<symbol>_<variant>' as the optimized implementation and
|
||||
'<symbol>_ifunc_selector' as the IFUNC selector. */
|
||||
#define REDIRECT_NAME EVALUATOR1 (__redirect, SYMBOL_NAME)
|
||||
#define OPTIMIZE(name) EVALUATOR2 (SYMBOL_NAME, name)
|
||||
#define IFUNC_SELECTOR EVALUATOR1 (SYMBOL_NAME, ifunc_selector)
|
||||
#define OPTIMIZE1(name) EVALUATOR1 (SYMBOL_NAME, name)
|
||||
#define OPTIMIZE2(name) EVALUATOR2 (SYMBOL_NAME, name)
|
||||
/* Default is to use OPTIMIZE2. */
|
||||
#define OPTIMIZE(name) OPTIMIZE2(name)
|
||||
|
@ -356,9 +356,10 @@ L(ATR32res):
|
||||
.p2align 4,, 4
|
||||
END(memcmp)
|
||||
|
||||
#undef bcmp
|
||||
#ifdef USE_AS_MEMCMPEQ
|
||||
libc_hidden_def (memcmp)
|
||||
#else
|
||||
# undef bcmp
|
||||
weak_alias (memcmp, bcmp)
|
||||
#undef __memcmpeq
|
||||
strong_alias (memcmp, __memcmpeq)
|
||||
libc_hidden_builtin_def (memcmp)
|
||||
libc_hidden_def (__memcmpeq)
|
||||
#endif
|
||||
|
@ -7,7 +7,9 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c \
|
||||
memchr-sse2 rawmemchr-sse2 memchr-avx2 rawmemchr-avx2 \
|
||||
memrchr-sse2 memrchr-avx2 \
|
||||
memcmp-sse2 \
|
||||
memcmpeq-sse2 \
|
||||
memcmp-avx2-movbe \
|
||||
memcmpeq-avx2 \
|
||||
memcmp-sse4 memcpy-ssse3 \
|
||||
memmove-ssse3 \
|
||||
memcpy-ssse3-back \
|
||||
@ -42,6 +44,7 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c \
|
||||
memset-avx512-unaligned-erms \
|
||||
memchr-avx2-rtm \
|
||||
memcmp-avx2-movbe-rtm \
|
||||
memcmpeq-avx2-rtm \
|
||||
memmove-avx-unaligned-erms-rtm \
|
||||
memrchr-avx2-rtm \
|
||||
memset-avx2-unaligned-erms-rtm \
|
||||
@ -61,6 +64,7 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c \
|
||||
strrchr-avx2-rtm \
|
||||
memchr-evex \
|
||||
memcmp-evex-movbe \
|
||||
memcmpeq-evex \
|
||||
memmove-evex-unaligned-erms \
|
||||
memrchr-evex \
|
||||
memset-evex-unaligned-erms \
|
||||
|
@ -38,6 +38,27 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
||||
|
||||
size_t i = 0;
|
||||
|
||||
/* Support sysdeps/x86_64/multiarch/memcmpeq.c. */
|
||||
IFUNC_IMPL (i, name, __memcmpeq,
|
||||
IFUNC_IMPL_ADD (array, i, __memcmpeq,
|
||||
(CPU_FEATURE_USABLE (AVX2)
|
||||
&& CPU_FEATURE_USABLE (MOVBE)
|
||||
&& CPU_FEATURE_USABLE (BMI2)),
|
||||
__memcmpeq_avx2)
|
||||
IFUNC_IMPL_ADD (array, i, __memcmpeq,
|
||||
(CPU_FEATURE_USABLE (AVX2)
|
||||
&& CPU_FEATURE_USABLE (BMI2)
|
||||
&& CPU_FEATURE_USABLE (MOVBE)
|
||||
&& CPU_FEATURE_USABLE (RTM)),
|
||||
__memcmpeq_avx2_rtm)
|
||||
IFUNC_IMPL_ADD (array, i, __memcmpeq,
|
||||
(CPU_FEATURE_USABLE (AVX512VL)
|
||||
&& CPU_FEATURE_USABLE (AVX512BW)
|
||||
&& CPU_FEATURE_USABLE (MOVBE)
|
||||
&& CPU_FEATURE_USABLE (BMI2)),
|
||||
__memcmpeq_evex)
|
||||
IFUNC_IMPL_ADD (array, i, __memcmpeq, 1, __memcmpeq_sse2))
|
||||
|
||||
/* Support sysdeps/x86_64/multiarch/memchr.c. */
|
||||
IFUNC_IMPL (i, name, memchr,
|
||||
IFUNC_IMPL_ADD (array, i, memchr,
|
||||
|
49
sysdeps/x86_64/multiarch/ifunc-memcmpeq.h
Normal file
49
sysdeps/x86_64/multiarch/ifunc-memcmpeq.h
Normal file
@ -0,0 +1,49 @@
|
||||
/* Common definition for __memcmpeq ifunc selections.
|
||||
All versions must be listed in ifunc-impl-list.c.
|
||||
Copyright (C) 2017-2021 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
# include <init-arch.h>
|
||||
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE1 (sse2) attribute_hidden;
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE1 (avx2) attribute_hidden;
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE1 (avx2_rtm) attribute_hidden;
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE1 (evex) attribute_hidden;
|
||||
|
||||
static inline void *
|
||||
IFUNC_SELECTOR (void)
|
||||
{
|
||||
const struct cpu_features* cpu_features = __get_cpu_features ();
|
||||
|
||||
if (CPU_FEATURE_USABLE_P (cpu_features, AVX2)
|
||||
&& CPU_FEATURE_USABLE_P (cpu_features, BMI2)
|
||||
&& CPU_FEATURE_USABLE_P (cpu_features, MOVBE)
|
||||
&& CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
|
||||
{
|
||||
if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
|
||||
&& CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
|
||||
return OPTIMIZE1 (evex);
|
||||
|
||||
if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
|
||||
return OPTIMIZE1 (avx2_rtm);
|
||||
|
||||
if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER))
|
||||
return OPTIMIZE1 (avx2);
|
||||
}
|
||||
|
||||
return OPTIMIZE1 (sse2);
|
||||
}
|
@ -17,7 +17,9 @@
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#if IS_IN (libc)
|
||||
# define memcmp __memcmp_sse2
|
||||
# ifndef memcmp
|
||||
# define memcmp __memcmp_sse2
|
||||
# endif
|
||||
|
||||
# ifdef SHARED
|
||||
# undef libc_hidden_builtin_def
|
||||
|
@ -29,9 +29,6 @@
|
||||
libc_ifunc_redirected (__redirect_memcmp, memcmp, IFUNC_SELECTOR ());
|
||||
# undef bcmp
|
||||
weak_alias (memcmp, bcmp)
|
||||
# undef __memcmpeq
|
||||
strong_alias (memcmp, __memcmpeq)
|
||||
libc_hidden_def (__memcmpeq)
|
||||
|
||||
# ifdef SHARED
|
||||
__hidden_ver1 (memcmp, __GI_memcmp, __redirect_memcmp)
|
||||
|
12
sysdeps/x86_64/multiarch/memcmpeq-avx2-rtm.S
Normal file
12
sysdeps/x86_64/multiarch/memcmpeq-avx2-rtm.S
Normal file
@ -0,0 +1,12 @@
|
||||
#ifndef MEMCMP
|
||||
# define MEMCMP __memcmpeq_avx2_rtm
|
||||
#endif
|
||||
|
||||
#define ZERO_UPPER_VEC_REGISTERS_RETURN \
|
||||
ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST
|
||||
|
||||
#define VZEROUPPER_RETURN jmp L(return_vzeroupper)
|
||||
|
||||
#define SECTION(p) p##.avx.rtm
|
||||
|
||||
#include "memcmpeq-avx2.S"
|
23
sysdeps/x86_64/multiarch/memcmpeq-avx2.S
Normal file
23
sysdeps/x86_64/multiarch/memcmpeq-avx2.S
Normal file
@ -0,0 +1,23 @@
|
||||
/* __memcmpeq optimized with AVX2.
|
||||
Copyright (C) 2017-2021 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#ifndef MEMCMP
|
||||
# define MEMCMP __memcmpeq_avx2
|
||||
#endif
|
||||
|
||||
#include "memcmp-avx2-movbe.S"
|
23
sysdeps/x86_64/multiarch/memcmpeq-evex.S
Normal file
23
sysdeps/x86_64/multiarch/memcmpeq-evex.S
Normal file
@ -0,0 +1,23 @@
|
||||
/* __memcmpeq optimized with EVEX.
|
||||
Copyright (C) 2017-2021 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#ifndef MEMCMP
|
||||
# define MEMCMP __memcmpeq_evex
|
||||
#endif
|
||||
|
||||
#include "memcmp-evex-movbe.S"
|
23
sysdeps/x86_64/multiarch/memcmpeq-sse2.S
Normal file
23
sysdeps/x86_64/multiarch/memcmpeq-sse2.S
Normal file
@ -0,0 +1,23 @@
|
||||
/* __memcmpeq optimized with SSE2.
|
||||
Copyright (C) 2017-2021 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#ifndef memcmp
|
||||
# define memcmp __memcmpeq_sse2
|
||||
#endif
|
||||
#define USE_AS_MEMCMPEQ 1
|
||||
#include "memcmp-sse2.S"
|
35
sysdeps/x86_64/multiarch/memcmpeq.c
Normal file
35
sysdeps/x86_64/multiarch/memcmpeq.c
Normal file
@ -0,0 +1,35 @@
|
||||
/* Multiple versions of __memcmpeq.
|
||||
All versions must be listed in ifunc-impl-list.c.
|
||||
Copyright (C) 2017-2021 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
/* Define multiple versions only for the definition in libc. */
|
||||
#if IS_IN (libc)
|
||||
# define __memcmpeq __redirect___memcmpeq
|
||||
# include <string.h>
|
||||
# undef __memcmpeq
|
||||
|
||||
# define SYMBOL_NAME __memcmpeq
|
||||
# include "ifunc-memcmpeq.h"
|
||||
|
||||
libc_ifunc_redirected (__redirect___memcmpeq, __memcmpeq, IFUNC_SELECTOR ());
|
||||
|
||||
# ifdef SHARED
|
||||
__hidden_ver1 (__memcmpeq, __GI___memcmpeq, __redirect___memcmpeq)
|
||||
__attribute__ ((visibility ("hidden"))) __attribute_copy__ (__memcmpeq);
|
||||
# endif
|
||||
#endif
|
Loading…
Reference in New Issue
Block a user