glibc/sysdeps/x86_64/multiarch/strncpy-or-cat-overflow-def.h

81 lines
2.3 KiB
C
Raw Normal View History

x86: Optimize and shrink st{r|p}{n}{cat|cpy}-evex functions Optimizations are: 1. Use more overlapping stores to avoid branches. 2. Reduce how unrolled the aligning copies are (this is more of a code-size save, its a negative for some sizes in terms of perf). 3. Improve the loop a bit (similiar to what we do in strlen with 2x vpminu + kortest instead of 3x vpminu + kmov + test). 4. For st{r|p}n{cat|cpy} re-order the branches to minimize the number that are taken. Performance Changes: Times are from N = 10 runs of the benchmark suite and are reported as geometric mean of all ratios of New Implementation / Old Implementation. stpcpy-evex -> 0.922 strcat-evex -> 0.985 strcpy-evex -> 0.880 strncpy-evex -> 0.831 stpncpy-evex -> 0.780 strncat-evex -> 0.958 Code Size Changes: function -> Bytes New / Bytes Old -> Ratio strcat-evex -> 819 / 1874 -> 0.437 strcpy-evex -> 700 / 1074 -> 0.652 stpcpy-evex -> 735 / 1094 -> 0.672 strncpy-evex -> 1397 / 2611 -> 0.535 stpncpy-evex -> 1489 / 2691 -> 0.553 strncat-evex -> 1184 / 2832 -> 0.418 Notes: 1. Because of the significant difference between the implementations they are split into three files. strcpy-evex.S -> strcpy, stpcpy, strcat strncpy-evex.S -> strncpy strncat-evex.S > strncat I couldn't find a way to merge them without making the ifdefs incredibly difficult to follow. 2. All implementations can be made evex512 by including "x86-evex512-vecs.h" at the top. 3. All implementations have an optional define: `USE_EVEX_MASKED_STORE` Setting to one uses evex-masked stores for handling short strings. This saves code size and branches. It's disabled for all implementations are the moment as there are some serious drawbacks to masked stores in certain cases, but that may be fixed on future architectures. Full check passes on x86-64 and build succeeds for all ISA levels w/ and w/o multiarch.
2022-11-09 01:38:38 +00:00
/* Helper for getting proper name of overflow fallback function for
{wc|st}{p|r|s}n{cat|cpy}
All versions must be listed in ifunc-impl-list.c.
Copyright (C) 2022-2024 Free Software Foundation, Inc.
x86: Optimize and shrink st{r|p}{n}{cat|cpy}-evex functions Optimizations are: 1. Use more overlapping stores to avoid branches. 2. Reduce how unrolled the aligning copies are (this is more of a code-size save, its a negative for some sizes in terms of perf). 3. Improve the loop a bit (similiar to what we do in strlen with 2x vpminu + kortest instead of 3x vpminu + kmov + test). 4. For st{r|p}n{cat|cpy} re-order the branches to minimize the number that are taken. Performance Changes: Times are from N = 10 runs of the benchmark suite and are reported as geometric mean of all ratios of New Implementation / Old Implementation. stpcpy-evex -> 0.922 strcat-evex -> 0.985 strcpy-evex -> 0.880 strncpy-evex -> 0.831 stpncpy-evex -> 0.780 strncat-evex -> 0.958 Code Size Changes: function -> Bytes New / Bytes Old -> Ratio strcat-evex -> 819 / 1874 -> 0.437 strcpy-evex -> 700 / 1074 -> 0.652 stpcpy-evex -> 735 / 1094 -> 0.672 strncpy-evex -> 1397 / 2611 -> 0.535 stpncpy-evex -> 1489 / 2691 -> 0.553 strncat-evex -> 1184 / 2832 -> 0.418 Notes: 1. Because of the significant difference between the implementations they are split into three files. strcpy-evex.S -> strcpy, stpcpy, strcat strncpy-evex.S -> strncpy strncat-evex.S > strncat I couldn't find a way to merge them without making the ifdefs incredibly difficult to follow. 2. All implementations can be made evex512 by including "x86-evex512-vecs.h" at the top. 3. All implementations have an optional define: `USE_EVEX_MASKED_STORE` Setting to one uses evex-masked stores for handling short strings. This saves code size and branches. It's disabled for all implementations are the moment as there are some serious drawbacks to masked stores in certain cases, but that may be fixed on future architectures. Full check passes on x86-64 and build succeeds for all ISA levels w/ and w/o multiarch.
2022-11-09 01:38:38 +00:00
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#ifndef _STRNCPY_OR_CAT_OVERFLOW_DEF_H_
#define _STRNCPY_OR_CAT_OVERFLOW_DEF_H_ 1
#if defined USE_MULTIARCH && IS_IN(libc)
# define UNDERSCORES __
# ifdef USE_WITH_SSE2
# define ISA_EXT _sse2
# elif defined USE_WITH_AVX2
# ifdef USE_WITH_RTM
# define ISA_EXT _avx2_rtm
# else
# define ISA_EXT _avx2
# endif
# elif defined USE_WITH_EVEX256
# define ISA_EXT _evex
# elif defined USE_WITH_EVEX512
# define ISA_EXT _evex512
# endif
#else
# define UNDERSCORES
# define ISA_EXT
#endif
#ifdef USE_AS_WCSCPY
# define STRCPY_PREFIX wc
# define STRCAT_PREFIX wcs
# ifdef USE_AS_STPCPY
# define STRCPY_POSTFIX pcpy
# else
# define STRCPY_POSTFIX scpy
# endif
#else
# define STRCPY_PREFIX st
# define STRCAT_PREFIX str
# ifdef USE_AS_STPCPY
# define STRCPY_POSTFIX pcpy
# else
# define STRCPY_POSTFIX rcpy
# endif
#endif
#define STRCAT_POSTFIX cat
#define PRIMITIVE_OF_NAMER(underscores, prefix, postfix, ext) \
underscores##prefix##postfix##ext
#define OF_NAMER(...) PRIMITIVE_OF_NAMER (__VA_ARGS__)
#ifndef OVERFLOW_STRCPY
# define OVERFLOW_STRCPY \
OF_NAMER (UNDERSCORES, STRCPY_PREFIX, STRCPY_POSTFIX, ISA_EXT)
#endif
#ifndef OVERFLOW_STRCAT
# define OVERFLOW_STRCAT \
OF_NAMER (UNDERSCORES, STRCAT_PREFIX, STRCAT_POSTFIX, ISA_EXT)
#endif
#endif