mirror of
https://sourceware.org/git/glibc.git
synced 2024-11-26 06:50:07 +00:00
69717709ec
Size Optimizations:
1. Condence hot path for better cache-locality.
- This is most impact for strchrnul where the logic strings with
len <= VEC_SIZE or with a match in the first VEC no fits entirely
in the first cache line.
2. Reuse common targets in first 4x VEC and after the loop.
3. Don't align targets so aggressively if it doesn't change the number
of fetch blocks it will require and put more care in avoiding the
case where targets unnecessarily split cache lines.
4. Align the loop better for DSB/LSD
5. Use more code-size efficient instructions.
- tzcnt ... -> bsf ...
- vpcmpb $0 ... -> vpcmpeq ...
6. Align labels less aggressively, especially if it doesn't save fetch
blocks / causes the basic-block to span extra cache-lines.
Code Size Changes:
strchr-evex.S : -63 bytes
strchrnul-evex.S: -48 bytes
Net perf changes:
Reported as geometric mean of all improvements / regressions from N=10
runs of the benchtests. Value as New Time / Old Time so < 1.0 is
improvement and 1.0 is regression.
strchr-evex.S (Fixed) : 0.971
strchr-evex.S (Rand) : 0.932
strchrnul-evex.S : 0.965
Full results attached in email.
Full check passes on x86-64.
518 lines
14 KiB
ArmAsm
518 lines
14 KiB
ArmAsm
/* strchr/strchrnul optimized with 256-bit EVEX instructions.
|
|
Copyright (C) 2021-2022 Free Software Foundation, Inc.
|
|
This file is part of the GNU C Library.
|
|
|
|
The GNU C Library is free software; you can redistribute it and/or
|
|
modify it under the terms of the GNU Lesser General Public
|
|
License as published by the Free Software Foundation; either
|
|
version 2.1 of the License, or (at your option) any later version.
|
|
|
|
The GNU C Library is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
Lesser General Public License for more details.
|
|
|
|
You should have received a copy of the GNU Lesser General Public
|
|
License along with the GNU C Library; if not, see
|
|
<https://www.gnu.org/licenses/>. */
|
|
|
|
#include <isa-level.h>
|
|
|
|
#if ISA_SHOULD_BUILD (4)
|
|
|
|
# include <sysdep.h>
|
|
|
|
# ifndef STRCHR
|
|
# define STRCHR __strchr_evex
|
|
# endif
|
|
|
|
# ifndef VEC_SIZE
|
|
# include "x86-evex256-vecs.h"
|
|
# endif
|
|
|
|
# ifdef USE_AS_WCSCHR
|
|
# define VPBROADCAST vpbroadcastd
|
|
# define VPCMP vpcmpd
|
|
# define VPCMPEQ vpcmpeqd
|
|
# define VPTESTN vptestnmd
|
|
# define VPTEST vptestmd
|
|
# define VPMINU vpminud
|
|
# define CHAR_REG esi
|
|
# define SHIFT_REG rcx
|
|
# define CHAR_SIZE 4
|
|
|
|
# define USE_WIDE_CHAR
|
|
# else
|
|
# define VPBROADCAST vpbroadcastb
|
|
# define VPCMP vpcmpb
|
|
# define VPCMPEQ vpcmpeqb
|
|
# define VPTESTN vptestnmb
|
|
# define VPTEST vptestmb
|
|
# define VPMINU vpminub
|
|
# define CHAR_REG sil
|
|
# define SHIFT_REG rdi
|
|
# define CHAR_SIZE 1
|
|
# endif
|
|
|
|
# include "reg-macros.h"
|
|
|
|
# if VEC_SIZE == 64
|
|
# define MASK_GPR rcx
|
|
# define LOOP_REG rax
|
|
|
|
# define COND_MASK(k_reg) {%k_reg}
|
|
# else
|
|
# define MASK_GPR rax
|
|
# define LOOP_REG rdi
|
|
|
|
# define COND_MASK(k_reg)
|
|
# endif
|
|
|
|
# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE)
|
|
|
|
|
|
# if CHAR_PER_VEC == 64
|
|
# define LAST_VEC_OFFSET (VEC_SIZE * 3)
|
|
# define TESTZ(reg) incq %VGPR_SZ(reg, 64)
|
|
# else
|
|
|
|
# if CHAR_PER_VEC == 32
|
|
# define TESTZ(reg) incl %VGPR_SZ(reg, 32)
|
|
# elif CHAR_PER_VEC == 16
|
|
# define TESTZ(reg) incw %VGPR_SZ(reg, 16)
|
|
# else
|
|
# define TESTZ(reg) incb %VGPR_SZ(reg, 8)
|
|
# endif
|
|
|
|
# define LAST_VEC_OFFSET (VEC_SIZE * 2)
|
|
# endif
|
|
|
|
# define VMATCH VMM(0)
|
|
|
|
# define PAGE_SIZE 4096
|
|
|
|
.section SECTION(.text), "ax", @progbits
|
|
ENTRY_P2ALIGN (STRCHR, 6)
|
|
/* Broadcast CHAR to VEC_0. */
|
|
VPBROADCAST %esi, %VMATCH
|
|
movl %edi, %eax
|
|
andl $(PAGE_SIZE - 1), %eax
|
|
/* Check if we cross page boundary with one vector load.
|
|
Otherwise it is safe to use an unaligned load. */
|
|
cmpl $(PAGE_SIZE - VEC_SIZE), %eax
|
|
ja L(cross_page_boundary)
|
|
|
|
|
|
/* Check the first VEC_SIZE bytes. Search for both CHAR and the
|
|
null bytes. */
|
|
VMOVU (%rdi), %VMM(1)
|
|
/* Leaves only CHARS matching esi as 0. */
|
|
vpxorq %VMM(1), %VMATCH, %VMM(2)
|
|
VPMINU %VMM(2), %VMM(1), %VMM(2)
|
|
/* Each bit in K0 represents a CHAR or a null byte in VEC_1. */
|
|
VPTESTN %VMM(2), %VMM(2), %k0
|
|
KMOV %k0, %VRAX
|
|
# if VEC_SIZE == 64 && defined USE_AS_STRCHRNUL
|
|
/* If VEC_SIZE == 64 && STRCHRNUL use bsf to test condition so
|
|
that all logic for match/null in first VEC first in 1x cache
|
|
lines. This has a slight cost to larger sizes. */
|
|
bsf %VRAX, %VRAX
|
|
jz L(aligned_more)
|
|
# else
|
|
test %VRAX, %VRAX
|
|
jz L(aligned_more)
|
|
bsf %VRAX, %VRAX
|
|
# endif
|
|
# ifndef USE_AS_STRCHRNUL
|
|
/* Found CHAR or the null byte. */
|
|
cmp (%rdi, %rax, CHAR_SIZE), %CHAR_REG
|
|
/* NB: Use a branch instead of cmovcc here. The expectation is
|
|
that with strchr the user will branch based on input being
|
|
null. Since this branch will be 100% predictive of the user
|
|
branch a branch miss here should save what otherwise would
|
|
be branch miss in the user code. Otherwise using a branch 1)
|
|
saves code size and 2) is faster in highly predictable
|
|
environments. */
|
|
jne L(zero)
|
|
# endif
|
|
# ifdef USE_AS_WCSCHR
|
|
/* NB: Multiply wchar_t count by 4 to get the number of bytes.
|
|
*/
|
|
leaq (%rdi, %rax, CHAR_SIZE), %rax
|
|
# else
|
|
addq %rdi, %rax
|
|
# endif
|
|
ret
|
|
|
|
# ifndef USE_AS_STRCHRNUL
|
|
L(zero):
|
|
xorl %eax, %eax
|
|
ret
|
|
# endif
|
|
|
|
.p2align 4,, 2
|
|
L(first_vec_x3):
|
|
subq $-(VEC_SIZE * 2), %rdi
|
|
# if VEC_SIZE == 32
|
|
/* Reuse L(first_vec_x3) for last VEC2 only for VEC_SIZE == 32.
|
|
For VEC_SIZE == 64 the registers don't match. */
|
|
L(last_vec_x2):
|
|
# endif
|
|
L(first_vec_x1):
|
|
/* Use bsf here to save 1-byte keeping keeping the block in 1x
|
|
fetch block. eax guranteed non-zero. */
|
|
bsf %VRCX, %VRCX
|
|
# ifndef USE_AS_STRCHRNUL
|
|
/* Found CHAR or the null byte. */
|
|
cmp (VEC_SIZE)(%rdi, %rcx, CHAR_SIZE), %CHAR_REG
|
|
jne L(zero)
|
|
# endif
|
|
/* NB: Multiply sizeof char type (1 or 4) to get the number of
|
|
bytes. */
|
|
leaq (VEC_SIZE)(%rdi, %rcx, CHAR_SIZE), %rax
|
|
ret
|
|
|
|
.p2align 4,, 2
|
|
L(first_vec_x4):
|
|
subq $-(VEC_SIZE * 2), %rdi
|
|
L(first_vec_x2):
|
|
# ifndef USE_AS_STRCHRNUL
|
|
/* Check to see if first match was CHAR (k0) or null (k1). */
|
|
KMOV %k0, %VRAX
|
|
tzcnt %VRAX, %VRAX
|
|
KMOV %k1, %VRCX
|
|
/* bzhil will not be 0 if first match was null. */
|
|
bzhi %VRAX, %VRCX, %VRCX
|
|
jne L(zero)
|
|
# else
|
|
/* Combine CHAR and null matches. */
|
|
KOR %k0, %k1, %k0
|
|
KMOV %k0, %VRAX
|
|
bsf %VRAX, %VRAX
|
|
# endif
|
|
/* NB: Multiply sizeof char type (1 or 4) to get the number of
|
|
bytes. */
|
|
leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax
|
|
ret
|
|
|
|
# ifdef USE_AS_STRCHRNUL
|
|
/* We use this as a hook to get imm8 encoding for the jmp to
|
|
L(page_cross_boundary). This allows the hot case of a
|
|
match/null-term in first VEC to fit entirely in 1 cache
|
|
line. */
|
|
L(cross_page_boundary):
|
|
jmp L(cross_page_boundary_real)
|
|
# endif
|
|
|
|
.p2align 4
|
|
L(aligned_more):
|
|
L(cross_page_continue):
|
|
/* Align data to VEC_SIZE. */
|
|
andq $-VEC_SIZE, %rdi
|
|
|
|
/* Check the next 4 * VEC_SIZE. Only one VEC_SIZE at a time
|
|
since data is only aligned to VEC_SIZE. Use two alternating
|
|
methods for checking VEC to balance latency and port
|
|
contention. */
|
|
|
|
/* Method(1) with 8c latency:
|
|
For VEC_SIZE == 32:
|
|
p0 * 1.83, p1 * 0.83, p5 * 1.33
|
|
For VEC_SIZE == 64:
|
|
p0 * 2.50, p1 * 0.00, p5 * 1.50 */
|
|
VMOVA (VEC_SIZE)(%rdi), %VMM(1)
|
|
/* Leaves only CHARS matching esi as 0. */
|
|
vpxorq %VMM(1), %VMATCH, %VMM(2)
|
|
VPMINU %VMM(2), %VMM(1), %VMM(2)
|
|
/* Each bit in K0 represents a CHAR or a null byte in VEC_1. */
|
|
VPTESTN %VMM(2), %VMM(2), %k0
|
|
KMOV %k0, %VRCX
|
|
test %VRCX, %VRCX
|
|
jnz L(first_vec_x1)
|
|
|
|
/* Method(2) with 6c latency:
|
|
For VEC_SIZE == 32:
|
|
p0 * 1.00, p1 * 0.00, p5 * 2.00
|
|
For VEC_SIZE == 64:
|
|
p0 * 1.00, p1 * 0.00, p5 * 2.00 */
|
|
VMOVA (VEC_SIZE * 2)(%rdi), %VMM(1)
|
|
/* Each bit in K0 represents a CHAR in VEC_1. */
|
|
VPCMPEQ %VMM(1), %VMATCH, %k0
|
|
/* Each bit in K1 represents a CHAR in VEC_1. */
|
|
VPTESTN %VMM(1), %VMM(1), %k1
|
|
KORTEST %k0, %k1
|
|
jnz L(first_vec_x2)
|
|
|
|
/* By swapping between Method 1/2 we get more fair port
|
|
distrubition and better throughput. */
|
|
|
|
VMOVA (VEC_SIZE * 3)(%rdi), %VMM(1)
|
|
/* Leaves only CHARS matching esi as 0. */
|
|
vpxorq %VMM(1), %VMATCH, %VMM(2)
|
|
VPMINU %VMM(2), %VMM(1), %VMM(2)
|
|
/* Each bit in K0 represents a CHAR or a null byte in VEC_1. */
|
|
VPTESTN %VMM(2), %VMM(2), %k0
|
|
KMOV %k0, %VRCX
|
|
test %VRCX, %VRCX
|
|
jnz L(first_vec_x3)
|
|
|
|
VMOVA (VEC_SIZE * 4)(%rdi), %VMM(1)
|
|
/* Each bit in K0 represents a CHAR in VEC_1. */
|
|
VPCMPEQ %VMM(1), %VMATCH, %k0
|
|
/* Each bit in K1 represents a CHAR in VEC_1. */
|
|
VPTESTN %VMM(1), %VMM(1), %k1
|
|
KORTEST %k0, %k1
|
|
jnz L(first_vec_x4)
|
|
|
|
/* Align data to VEC_SIZE * 4 for the loop. */
|
|
# if VEC_SIZE == 64
|
|
/* Use rax for the loop reg as it allows to the loop to fit in
|
|
exactly 2-cache-lines. (more efficient imm32 + gpr
|
|
encoding). */
|
|
leaq (VEC_SIZE)(%rdi), %rax
|
|
/* No partial register stalls on evex512 processors. */
|
|
xorb %al, %al
|
|
# else
|
|
/* For VEC_SIZE == 32 continue using rdi for loop reg so we can
|
|
reuse more code and save space. */
|
|
addq $VEC_SIZE, %rdi
|
|
andq $-(VEC_SIZE * 4), %rdi
|
|
# endif
|
|
.p2align 4
|
|
L(loop_4x_vec):
|
|
/* Check 4x VEC at a time. No penalty for imm32 offset with evex
|
|
encoding (if offset % VEC_SIZE == 0). */
|
|
VMOVA (VEC_SIZE * 4)(%LOOP_REG), %VMM(1)
|
|
VMOVA (VEC_SIZE * 5)(%LOOP_REG), %VMM(2)
|
|
VMOVA (VEC_SIZE * 6)(%LOOP_REG), %VMM(3)
|
|
VMOVA (VEC_SIZE * 7)(%LOOP_REG), %VMM(4)
|
|
|
|
/* Collect bits where VEC_1 does NOT match esi. This is later
|
|
use to mask of results (getting not matches allows us to
|
|
save an instruction on combining). */
|
|
VPCMP $4, %VMATCH, %VMM(1), %k1
|
|
|
|
/* Two methods for loop depending on VEC_SIZE. This is because
|
|
with zmm registers VPMINU can only run on p0 (as opposed to
|
|
p0/p1 for ymm) so it is less prefered. */
|
|
# if VEC_SIZE == 32
|
|
/* For VEC_2 and VEC_3 use xor to set the CHARs matching esi to
|
|
zero. */
|
|
vpxorq %VMM(2), %VMATCH, %VMM(6)
|
|
vpxorq %VMM(3), %VMATCH, %VMM(7)
|
|
|
|
/* Find non-matches in VEC_4 while combining with non-matches
|
|
from VEC_1. NB: Try and use masked predicate execution on
|
|
instructions that have mask result as it has no latency
|
|
penalty. */
|
|
VPCMP $4, %VMATCH, %VMM(4), %k4{%k1}
|
|
|
|
/* Combined zeros from VEC_1 / VEC_2 (search for null term). */
|
|
VPMINU %VMM(1), %VMM(2), %VMM(2)
|
|
|
|
/* Use min to select all zeros from either xor or end of
|
|
string). */
|
|
VPMINU %VMM(3), %VMM(7), %VMM(3)
|
|
VPMINU %VMM(2), %VMM(6), %VMM(2)
|
|
|
|
/* Combined zeros from VEC_2 / VEC_3 (search for null term). */
|
|
VPMINU %VMM(3), %VMM(4), %VMM(4)
|
|
|
|
/* Combined zeros from VEC_2 / VEC_4 (this has all null term and
|
|
esi matches for VEC_2 / VEC_3). */
|
|
VPMINU %VMM(2), %VMM(4), %VMM(4)
|
|
# else
|
|
/* Collect non-matches for VEC_2. */
|
|
VPCMP $4, %VMM(2), %VMATCH, %k2
|
|
|
|
/* Combined zeros from VEC_1 / VEC_2 (search for null term). */
|
|
VPMINU %VMM(1), %VMM(2), %VMM(2)
|
|
|
|
/* Find non-matches in VEC_3/VEC_4 while combining with non-
|
|
matches from VEC_1/VEC_2 respectively. */
|
|
VPCMP $4, %VMM(3), %VMATCH, %k3{%k1}
|
|
VPCMP $4, %VMM(4), %VMATCH, %k4{%k2}
|
|
|
|
/* Finish combining zeros in all VECs. */
|
|
VPMINU %VMM(3), %VMM(4), %VMM(4)
|
|
|
|
/* Combine in esi matches for VEC_3 (if there was a match with
|
|
esi, the corresponding bit in %k3 is zero so the
|
|
VPMINU_MASKZ will have a zero in the result). NB: This make
|
|
the VPMINU 3c latency. The only way to avoid it is to
|
|
createa a 12c dependency chain on all the `VPCMP $4, ...`
|
|
which has higher total latency. */
|
|
VPMINU %VMM(2), %VMM(4), %VMM(4){%k3}{z}
|
|
# endif
|
|
VPTEST %VMM(4), %VMM(4), %k0{%k4}
|
|
KMOV %k0, %VRDX
|
|
subq $-(VEC_SIZE * 4), %LOOP_REG
|
|
|
|
/* TESTZ is inc using the proper register width depending on
|
|
CHAR_PER_VEC. An esi match or null-term match leaves a zero-
|
|
bit in rdx so inc won't overflow and won't be zero. */
|
|
TESTZ (rdx)
|
|
jz L(loop_4x_vec)
|
|
|
|
VPTEST %VMM(1), %VMM(1), %k0{%k1}
|
|
KMOV %k0, %VGPR(MASK_GPR)
|
|
TESTZ (MASK_GPR)
|
|
# if VEC_SIZE == 32
|
|
/* We can reuse the return code in page_cross logic for VEC_SIZE
|
|
== 32. */
|
|
jnz L(last_vec_x1_vec_size32)
|
|
# else
|
|
jnz L(last_vec_x1_vec_size64)
|
|
# endif
|
|
|
|
|
|
/* COND_MASK integates the esi matches for VEC_SIZE == 64. For
|
|
VEC_SIZE == 32 they are already integrated. */
|
|
VPTEST %VMM(2), %VMM(2), %k0 COND_MASK(k2)
|
|
KMOV %k0, %VRCX
|
|
TESTZ (rcx)
|
|
jnz L(last_vec_x2)
|
|
|
|
VPTEST %VMM(3), %VMM(3), %k0 COND_MASK(k3)
|
|
KMOV %k0, %VRCX
|
|
# if CHAR_PER_VEC == 64
|
|
TESTZ (rcx)
|
|
jnz L(last_vec_x3)
|
|
# else
|
|
salq $CHAR_PER_VEC, %rdx
|
|
TESTZ (rcx)
|
|
orq %rcx, %rdx
|
|
# endif
|
|
|
|
bsfq %rdx, %rdx
|
|
|
|
# ifndef USE_AS_STRCHRNUL
|
|
/* Check if match was CHAR or null. */
|
|
cmp (LAST_VEC_OFFSET)(%LOOP_REG, %rdx, CHAR_SIZE), %CHAR_REG
|
|
jne L(zero_end)
|
|
# endif
|
|
/* NB: Multiply sizeof char type (1 or 4) to get the number of
|
|
bytes. */
|
|
leaq (LAST_VEC_OFFSET)(%LOOP_REG, %rdx, CHAR_SIZE), %rax
|
|
ret
|
|
|
|
# ifndef USE_AS_STRCHRNUL
|
|
L(zero_end):
|
|
xorl %eax, %eax
|
|
ret
|
|
# endif
|
|
|
|
|
|
/* Seperate return label for last VEC1 because for VEC_SIZE ==
|
|
32 we can reuse return code in L(page_cross) but VEC_SIZE ==
|
|
64 has mismatched registers. */
|
|
# if VEC_SIZE == 64
|
|
.p2align 4,, 8
|
|
L(last_vec_x1_vec_size64):
|
|
bsf %VRCX, %VRCX
|
|
# ifndef USE_AS_STRCHRNUL
|
|
/* Check if match was null. */
|
|
cmp (%rax, %rcx, CHAR_SIZE), %CHAR_REG
|
|
jne L(zero_end)
|
|
# endif
|
|
# ifdef USE_AS_WCSCHR
|
|
/* NB: Multiply wchar_t count by 4 to get the number of bytes.
|
|
*/
|
|
leaq (%rax, %rcx, CHAR_SIZE), %rax
|
|
# else
|
|
addq %rcx, %rax
|
|
# endif
|
|
ret
|
|
|
|
/* Since we can't combine the last 2x matches for CHAR_PER_VEC
|
|
== 64 we need return label for last VEC3. */
|
|
# if CHAR_PER_VEC == 64
|
|
.p2align 4,, 8
|
|
L(last_vec_x3):
|
|
addq $VEC_SIZE, %LOOP_REG
|
|
# endif
|
|
|
|
/* Duplicate L(last_vec_x2) for VEC_SIZE == 64 because we can't
|
|
reuse L(first_vec_x3) due to register mismatch. */
|
|
L(last_vec_x2):
|
|
bsf %VGPR(MASK_GPR), %VGPR(MASK_GPR)
|
|
# ifndef USE_AS_STRCHRNUL
|
|
/* Check if match was null. */
|
|
cmp (VEC_SIZE * 1)(%LOOP_REG, %MASK_GPR, CHAR_SIZE), %CHAR_REG
|
|
jne L(zero_end)
|
|
# endif
|
|
/* NB: Multiply sizeof char type (1 or 4) to get the number of
|
|
bytes. */
|
|
leaq (VEC_SIZE * 1)(%LOOP_REG, %MASK_GPR, CHAR_SIZE), %rax
|
|
ret
|
|
# endif
|
|
|
|
/* Cold case for crossing page with first load. */
|
|
.p2align 4,, 10
|
|
# ifndef USE_AS_STRCHRNUL
|
|
L(cross_page_boundary):
|
|
# endif
|
|
L(cross_page_boundary_real):
|
|
/* Align rdi. */
|
|
xorq %rdi, %rax
|
|
VMOVA (PAGE_SIZE - VEC_SIZE)(%rax), %VMM(1)
|
|
/* Use high latency method of getting matches to save code size.
|
|
*/
|
|
|
|
/* K1 has 1s where VEC(1) does NOT match esi. */
|
|
VPCMP $4, %VMM(1), %VMATCH, %k1
|
|
/* K0 has ones where K1 is 1 (non-match with esi), and non-zero
|
|
(null). */
|
|
VPTEST %VMM(1), %VMM(1), %k0{%k1}
|
|
KMOV %k0, %VRAX
|
|
/* Remove the leading bits. */
|
|
# ifdef USE_AS_WCSCHR
|
|
movl %edi, %VGPR_SZ(SHIFT_REG, 32)
|
|
/* NB: Divide shift count by 4 since each bit in K1 represent 4
|
|
bytes. */
|
|
sarl $2, %VGPR_SZ(SHIFT_REG, 32)
|
|
andl $(CHAR_PER_VEC - 1), %VGPR_SZ(SHIFT_REG, 32)
|
|
|
|
/* if wcsrchr we need to reverse matches as we can't rely on
|
|
signed shift to bring in ones. There is not sarx for
|
|
gpr8/16. Also not we can't use inc here as the lower bits
|
|
represent matches out of range so we can't rely on overflow.
|
|
*/
|
|
xorl $((1 << CHAR_PER_VEC)- 1), %eax
|
|
# endif
|
|
/* Use arithmatic shift so that leading 1s are filled in. */
|
|
sarx %VGPR(SHIFT_REG), %VRAX, %VRAX
|
|
/* If eax is all ones then no matches for esi or NULL. */
|
|
|
|
# ifdef USE_AS_WCSCHR
|
|
test %VRAX, %VRAX
|
|
# else
|
|
inc %VRAX
|
|
# endif
|
|
jz L(cross_page_continue)
|
|
|
|
.p2align 4,, 10
|
|
L(last_vec_x1_vec_size32):
|
|
bsf %VRAX, %VRAX
|
|
# ifdef USE_AS_WCSCHR
|
|
/* NB: Multiply wchar_t count by 4 to get the number of bytes.
|
|
*/
|
|
leaq (%rdi, %rax, CHAR_SIZE), %rax
|
|
# else
|
|
addq %rdi, %rax
|
|
# endif
|
|
# ifndef USE_AS_STRCHRNUL
|
|
/* Check to see if match was CHAR or null. */
|
|
cmp (%rax), %CHAR_REG
|
|
jne L(zero_end_0)
|
|
# endif
|
|
ret
|
|
# ifndef USE_AS_STRCHRNUL
|
|
L(zero_end_0):
|
|
xorl %eax, %eax
|
|
ret
|
|
# endif
|
|
|
|
END (STRCHR)
|
|
#endif
|