mirror of
https://sourceware.org/git/glibc.git
synced 2024-12-29 05:51:10 +00:00
c69f960b01
The changes for these functions are different than the others because
the best implementation (sse4_2) requires the generic
implementation as a fallback to be built as well.
Changes are:
1. Add non-multiarch functions for str{c|p}{brk|spn}.c to statically
select the best implementation based on the configured ISA build
level.
2. Add stubs for str{c|p}{brk|spn}-generic and varshift.c to in the
sysdeps/x86_64 directory so that the the sse4 implementation will
have all of its dependencies for the non-multiarch / rtld build
when ISA level >= 2.
3. Add new multiarch/rtld-strcspn.c that just include the
non-multiarch strcspn.c which will in turn select the best
implementation based on the compiled ISA level.
4. Refactor the ifunc selector and ifunc implementation list to use
the ISA level aware wrapper macros that allow functions below the
compiled ISA level (with a guranteed replacement) to be skipped.
Tested with and without multiarch on x86_64 for ISA levels:
{generic, x86-64-v2, x86-64-v3, x86-64-v4}
And m32 with and without multiarch.
140 lines
4.0 KiB
C
140 lines
4.0 KiB
C
/* strspn with SSE4.2 intrinsics
|
|
Copyright (C) 2009-2022 Free Software Foundation, Inc.
|
|
This file is part of the GNU C Library.
|
|
|
|
The GNU C Library is free software; you can redistribute it and/or
|
|
modify it under the terms of the GNU Lesser General Public
|
|
License as published by the Free Software Foundation; either
|
|
version 2.1 of the License, or (at your option) any later version.
|
|
|
|
The GNU C Library is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
Lesser General Public License for more details.
|
|
|
|
You should have received a copy of the GNU Lesser General Public
|
|
License along with the GNU C Library; if not, see
|
|
<https://www.gnu.org/licenses/>. */
|
|
|
|
#include <nmmintrin.h>
|
|
#include <string.h>
|
|
#include "varshift.h"
|
|
|
|
/* We use 0x12:
|
|
_SIDD_SBYTE_OPS
|
|
| _SIDD_CMP_EQUAL_ANY
|
|
| _SIDD_NEGATIVE_POLARITY
|
|
| _SIDD_LEAST_SIGNIFICANT
|
|
on pcmpistri to compare xmm/mem128
|
|
|
|
0 1 2 3 4 5 6 7 8 9 A B C D E F
|
|
X X X X X X X X X X X X X X X X
|
|
|
|
against xmm
|
|
|
|
0 1 2 3 4 5 6 7 8 9 A B C D E F
|
|
A A A A A A A A A A A A A A A A
|
|
|
|
to find out if the first 16byte data element has any non-A byte and
|
|
the offset of the first byte. There are 2 cases:
|
|
|
|
1. The first 16byte data element has the non-A byte, including
|
|
EOS, at the offset X.
|
|
2. The first 16byte data element is valid and doesn't have the non-A
|
|
byte.
|
|
|
|
Here is the table of ECX, CFlag, ZFlag and SFlag for 2 cases:
|
|
|
|
case ECX CFlag ZFlag SFlag
|
|
1 X 1 0/1 0
|
|
2 16 0 0 0
|
|
|
|
We exit from the loop for case 1. */
|
|
|
|
extern size_t __strspn_generic (const char *, const char *) attribute_hidden;
|
|
|
|
#ifndef STRSPN
|
|
# define STRSPN __strspn_sse42
|
|
#endif
|
|
|
|
size_t
|
|
__attribute__ ((section (".text.sse4.2")))
|
|
STRSPN (const char *s, const char *a)
|
|
{
|
|
if (*a == 0)
|
|
return 0;
|
|
|
|
const char *aligned;
|
|
__m128i mask, maskz, zero;
|
|
unsigned int maskz_bits;
|
|
unsigned int offset = (int) ((size_t) a & 15);
|
|
zero = _mm_set1_epi8 (0);
|
|
if (offset != 0)
|
|
{
|
|
/* Load masks. */
|
|
aligned = (const char *) ((size_t) a & -16L);
|
|
__m128i mask0 = _mm_load_si128 ((__m128i *) aligned);
|
|
maskz = _mm_cmpeq_epi8 (mask0, zero);
|
|
|
|
/* Find where the NULL terminator is. */
|
|
maskz_bits = _mm_movemask_epi8 (maskz) >> offset;
|
|
if (maskz_bits != 0)
|
|
{
|
|
mask = __m128i_shift_right (mask0, offset);
|
|
offset = (unsigned int) ((size_t) s & 15);
|
|
if (offset)
|
|
goto start_unaligned;
|
|
|
|
aligned = s;
|
|
goto start_loop;
|
|
}
|
|
}
|
|
|
|
/* A is aligned. */
|
|
mask = _mm_loadu_si128 ((__m128i *) a);
|
|
|
|
/* Find where the NULL terminator is. */
|
|
maskz = _mm_cmpeq_epi8 (mask, zero);
|
|
maskz_bits = _mm_movemask_epi8 (maskz);
|
|
if (maskz_bits == 0)
|
|
{
|
|
/* There is no NULL terminator. Don't use SSE4.2 if the length
|
|
of A > 16. */
|
|
if (a[16] != 0)
|
|
return __strspn_generic (s, a);
|
|
}
|
|
aligned = s;
|
|
offset = (unsigned int) ((size_t) s & 15);
|
|
|
|
if (offset != 0)
|
|
{
|
|
start_unaligned:
|
|
/* Check partial string. */
|
|
aligned = (const char *) ((size_t) s & -16L);
|
|
__m128i value = _mm_load_si128 ((__m128i *) aligned);
|
|
__m128i adj_value = __m128i_shift_right (value, offset);
|
|
|
|
unsigned int length = _mm_cmpistri (mask, adj_value, 0x12);
|
|
/* No need to check CFlag since it is always 1. */
|
|
if (length < 16 - offset)
|
|
return length;
|
|
/* Find where the NULL terminator is. */
|
|
maskz = _mm_cmpeq_epi8 (value, zero);
|
|
maskz_bits = _mm_movemask_epi8 (maskz) >> offset;
|
|
if (maskz_bits != 0)
|
|
return length;
|
|
aligned += 16;
|
|
}
|
|
|
|
start_loop:
|
|
while (1)
|
|
{
|
|
__m128i value = _mm_load_si128 ((__m128i *) aligned);
|
|
unsigned int index = _mm_cmpistri (mask, value, 0x12);
|
|
unsigned int cflag = _mm_cmpistrc (mask, value, 0x12);
|
|
if (cflag)
|
|
return (size_t) (aligned + index - s);
|
|
aligned += 16;
|
|
}
|
|
}
|