x86_64: Add support for __memcmpeq using sse2, avx2, and evex

No bug. This commit adds support for __memcmpeq to be implemented
seperately from memcmp. Support is added for versions optimized with
sse2, avx2, and evex.
This commit is contained in:
Noah Goldstein 2021-10-26 19:43:18 -05:00
parent cf3acd774f
commit cf4fd28ea4
12 changed files with 202 additions and 9 deletions

View File

@ -50,5 +50,8 @@
'__<symbol>_<variant>' as the optimized implementation and
'<symbol>_ifunc_selector' as the IFUNC selector. */
#define REDIRECT_NAME EVALUATOR1 (__redirect, SYMBOL_NAME)
#define OPTIMIZE(name) EVALUATOR2 (SYMBOL_NAME, name)
#define IFUNC_SELECTOR EVALUATOR1 (SYMBOL_NAME, ifunc_selector)
#define OPTIMIZE1(name) EVALUATOR1 (SYMBOL_NAME, name)
#define OPTIMIZE2(name) EVALUATOR2 (SYMBOL_NAME, name)
/* Default is to use OPTIMIZE2. */
#define OPTIMIZE(name) OPTIMIZE2(name)

View File

@ -356,9 +356,10 @@ L(ATR32res):
.p2align 4,, 4
END(memcmp)
#undef bcmp
#ifdef USE_AS_MEMCMPEQ
libc_hidden_def (memcmp)
#else
# undef bcmp
weak_alias (memcmp, bcmp)
#undef __memcmpeq
strong_alias (memcmp, __memcmpeq)
libc_hidden_builtin_def (memcmp)
libc_hidden_def (__memcmpeq)
#endif

View File

@ -7,7 +7,9 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c \
memchr-sse2 rawmemchr-sse2 memchr-avx2 rawmemchr-avx2 \
memrchr-sse2 memrchr-avx2 \
memcmp-sse2 \
memcmpeq-sse2 \
memcmp-avx2-movbe \
memcmpeq-avx2 \
memcmp-sse4 memcpy-ssse3 \
memmove-ssse3 \
memcpy-ssse3-back \
@ -42,6 +44,7 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c \
memset-avx512-unaligned-erms \
memchr-avx2-rtm \
memcmp-avx2-movbe-rtm \
memcmpeq-avx2-rtm \
memmove-avx-unaligned-erms-rtm \
memrchr-avx2-rtm \
memset-avx2-unaligned-erms-rtm \
@ -61,6 +64,7 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c \
strrchr-avx2-rtm \
memchr-evex \
memcmp-evex-movbe \
memcmpeq-evex \
memmove-evex-unaligned-erms \
memrchr-evex \
memset-evex-unaligned-erms \

View File

@ -38,6 +38,27 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
size_t i = 0;
/* Support sysdeps/x86_64/multiarch/memcmpeq.c. */
IFUNC_IMPL (i, name, __memcmpeq,
IFUNC_IMPL_ADD (array, i, __memcmpeq,
(CPU_FEATURE_USABLE (AVX2)
&& CPU_FEATURE_USABLE (MOVBE)
&& CPU_FEATURE_USABLE (BMI2)),
__memcmpeq_avx2)
IFUNC_IMPL_ADD (array, i, __memcmpeq,
(CPU_FEATURE_USABLE (AVX2)
&& CPU_FEATURE_USABLE (BMI2)
&& CPU_FEATURE_USABLE (MOVBE)
&& CPU_FEATURE_USABLE (RTM)),
__memcmpeq_avx2_rtm)
IFUNC_IMPL_ADD (array, i, __memcmpeq,
(CPU_FEATURE_USABLE (AVX512VL)
&& CPU_FEATURE_USABLE (AVX512BW)
&& CPU_FEATURE_USABLE (MOVBE)
&& CPU_FEATURE_USABLE (BMI2)),
__memcmpeq_evex)
IFUNC_IMPL_ADD (array, i, __memcmpeq, 1, __memcmpeq_sse2))
/* Support sysdeps/x86_64/multiarch/memchr.c. */
IFUNC_IMPL (i, name, memchr,
IFUNC_IMPL_ADD (array, i, memchr,

View File

@ -0,0 +1,49 @@
/* Common definition for __memcmpeq ifunc selections.
All versions must be listed in ifunc-impl-list.c.
Copyright (C) 2017-2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
# include <init-arch.h>
extern __typeof (REDIRECT_NAME) OPTIMIZE1 (sse2) attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE1 (avx2) attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE1 (avx2_rtm) attribute_hidden;
extern __typeof (REDIRECT_NAME) OPTIMIZE1 (evex) attribute_hidden;
static inline void *
IFUNC_SELECTOR (void)
{
const struct cpu_features* cpu_features = __get_cpu_features ();
if (CPU_FEATURE_USABLE_P (cpu_features, AVX2)
&& CPU_FEATURE_USABLE_P (cpu_features, BMI2)
&& CPU_FEATURE_USABLE_P (cpu_features, MOVBE)
&& CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
{
if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
&& CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
return OPTIMIZE1 (evex);
if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
return OPTIMIZE1 (avx2_rtm);
if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER))
return OPTIMIZE1 (avx2);
}
return OPTIMIZE1 (sse2);
}

View File

@ -17,7 +17,9 @@
<https://www.gnu.org/licenses/>. */
#if IS_IN (libc)
# define memcmp __memcmp_sse2
# ifndef memcmp
# define memcmp __memcmp_sse2
# endif
# ifdef SHARED
# undef libc_hidden_builtin_def

View File

@ -29,9 +29,6 @@
libc_ifunc_redirected (__redirect_memcmp, memcmp, IFUNC_SELECTOR ());
# undef bcmp
weak_alias (memcmp, bcmp)
# undef __memcmpeq
strong_alias (memcmp, __memcmpeq)
libc_hidden_def (__memcmpeq)
# ifdef SHARED
__hidden_ver1 (memcmp, __GI_memcmp, __redirect_memcmp)

View File

@ -0,0 +1,12 @@
#ifndef MEMCMP
# define MEMCMP __memcmpeq_avx2_rtm
#endif
#define ZERO_UPPER_VEC_REGISTERS_RETURN \
ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST
#define VZEROUPPER_RETURN jmp L(return_vzeroupper)
#define SECTION(p) p##.avx.rtm
#include "memcmpeq-avx2.S"

View File

@ -0,0 +1,23 @@
/* __memcmpeq optimized with AVX2.
Copyright (C) 2017-2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#ifndef MEMCMP
# define MEMCMP __memcmpeq_avx2
#endif
#include "memcmp-avx2-movbe.S"

View File

@ -0,0 +1,23 @@
/* __memcmpeq optimized with EVEX.
Copyright (C) 2017-2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#ifndef MEMCMP
# define MEMCMP __memcmpeq_evex
#endif
#include "memcmp-evex-movbe.S"

View File

@ -0,0 +1,23 @@
/* __memcmpeq optimized with SSE2.
Copyright (C) 2017-2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#ifndef memcmp
# define memcmp __memcmpeq_sse2
#endif
#define USE_AS_MEMCMPEQ 1
#include "memcmp-sse2.S"

View File

@ -0,0 +1,35 @@
/* Multiple versions of __memcmpeq.
All versions must be listed in ifunc-impl-list.c.
Copyright (C) 2017-2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
/* Define multiple versions only for the definition in libc. */
#if IS_IN (libc)
# define __memcmpeq __redirect___memcmpeq
# include <string.h>
# undef __memcmpeq
# define SYMBOL_NAME __memcmpeq
# include "ifunc-memcmpeq.h"
libc_ifunc_redirected (__redirect___memcmpeq, __memcmpeq, IFUNC_SELECTOR ());
# ifdef SHARED
__hidden_ver1 (__memcmpeq, __GI___memcmpeq, __redirect___memcmpeq)
__attribute__ ((visibility ("hidden"))) __attribute_copy__ (__memcmpeq);
# endif
#endif