glibc/sysdeps/powerpc/powerpc64/le/power9/strlen.S

214 lines
5.2 KiB
ArmAsm
Raw Normal View History

/* Optimized strlen implementation for PowerPC64/POWER9.
Copyright (C) 2020-2022 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#include <sysdep.h>
#ifndef STRLEN
# define STRLEN __strlen
# define DEFINE_STRLEN_HIDDEN_DEF 1
#endif
/* Implements the function
int [r3] strlen (const void *s [r3])
The implementation can load bytes past a matching byte, but only
up to the next 64B boundary, so it never crosses a page. */
.machine power9
ENTRY_TOCLESS (STRLEN, 4)
CALL_MCOUNT 2
vspltisb v18,0
vspltisb v19,-1
neg r5,r3
rldicl r9,r5,0,60 /* How many bytes to get source 16B aligned? */
/* Align data and fill bytes not loaded with non matching char. */
lvx v0,0,r3
lvsr v1,0,r3
vperm v0,v19,v0,v1
vcmpequb. v6,v0,v18
beq cr6,L(aligned)
vctzlsbb r3,v6
blr
/* Test 64B 16B at a time. The 64B vector loop is optimized for
longer strings. Likewise, we check a multiple of 64B to avoid
breaking the alignment calculation below. */
L(aligned):
add r4,r3,r9
rldicl. r5,r4,60,62 /* Determine the number of 48B loops needed for
alignment to 64B. And test for zero. */
lxv v0+32,0(r4)
vcmpequb. v6,v0,v18
bne cr6,L(tail1)
lxv v0+32,16(r4)
vcmpequb. v6,v0,v18
bne cr6,L(tail2)
lxv v0+32,32(r4)
vcmpequb. v6,v0,v18
bne cr6,L(tail3)
lxv v0+32,48(r4)
vcmpequb. v6,v0,v18
bne cr6,L(tail4)
addi r4,r4,64
/* Speculatively generate a fake 16B aligned address to generate the
vector byte constant 0,1,..,15 using lvsl during reduction. */
li r0,0
/* Skip the alignment if already 64B aligned. */
beq L(loop_64b)
mtctr r5
/* Test 48B per iteration until 64B aligned. */
.p2align 5
L(loop):
lxv v0+32,0(r4)
vcmpequb. v6,v0,v18
bne cr6,L(tail1)
lxv v0+32,16(r4)
vcmpequb. v6,v0,v18
bne cr6,L(tail2)
lxv v0+32,32(r4)
vcmpequb. v6,v0,v18
bne cr6,L(tail3)
addi r4,r4,48
bdnz L(loop)
.p2align 5
L(loop_64b):
lxv v1+32,0(r4) /* Load 4 quadwords. */
lxv v2+32,16(r4)
lxv v3+32,32(r4)
lxv v4+32,48(r4)
vminub v5,v1,v2 /* Compare and merge into one VR for speed. */
vminub v6,v3,v4
vminub v7,v5,v6
vcmpequb. v7,v7,v18 /* Check for NULLs. */
addi r4,r4,64 /* Adjust address for the next iteration. */
bne cr6,L(vmx_zero)
lxv v1+32,0(r4) /* Load 4 quadwords. */
lxv v2+32,16(r4)
lxv v3+32,32(r4)
lxv v4+32,48(r4)
vminub v5,v1,v2 /* Compare and merge into one VR for speed. */
vminub v6,v3,v4
vminub v7,v5,v6
vcmpequb. v7,v7,v18 /* Check for NULLs. */
addi r4,r4,64 /* Adjust address for the next iteration. */
bne cr6,L(vmx_zero)
lxv v1+32,0(r4) /* Load 4 quadwords. */
lxv v2+32,16(r4)
lxv v3+32,32(r4)
lxv v4+32,48(r4)
vminub v5,v1,v2 /* Compare and merge into one VR for speed. */
vminub v6,v3,v4
vminub v7,v5,v6
vcmpequb. v7,v7,v18 /* Check for NULLs. */
addi r4,r4,64 /* Adjust address for the next iteration. */
beq cr6,L(loop_64b)
L(vmx_zero):
/* OK, we found a null byte. Let's look for it in the current 64-byte
block and mark it in its corresponding VR. */
vcmpequb v1,v1,v18
vcmpequb v2,v2,v18
vcmpequb v3,v3,v18
vcmpequb v4,v4,v18
/* We will now 'compress' the result into a single doubleword, so it
can be moved to a GPR for the final calculation. First, we
generate an appropriate mask for vbpermq, so we can permute bits into
the first halfword. */
vspltisb v10,3
lvsl v11,0,r0
vslb v10,v11,v10
/* Permute the first bit of each byte into bits 48-63. */
vbpermq v1,v1,v10
vbpermq v2,v2,v10
vbpermq v3,v3,v10
vbpermq v4,v4,v10
/* Shift each component into its correct position for merging. */
vsldoi v2,v2,v2,2
vsldoi v3,v3,v3,4
vsldoi v4,v4,v4,6
/* Merge the results and move to a GPR. */
vor v1,v2,v1
vor v2,v3,v4
vor v4,v1,v2
mfvrd r10,v4
/* Adjust address to the begninning of the current 64-byte block. */
addi r4,r4,-64
cnttzd r0,r10 /* Count trailing zeros before the match. */
subf r5,r3,r4
add r3,r5,r0 /* Compute final length. */
blr
L(tail1):
vctzlsbb r0,v6
add r4,r4,r0
subf r3,r3,r4
blr
L(tail2):
vctzlsbb r0,v6
add r4,r4,r0
addi r4,r4,16
subf r3,r3,r4
blr
L(tail3):
vctzlsbb r0,v6
add r4,r4,r0
addi r4,r4,32
subf r3,r3,r4
blr
L(tail4):
vctzlsbb r0,v6
add r4,r4,r0
addi r4,r4,48
subf r3,r3,r4
blr
END (STRLEN)
#ifdef DEFINE_STRLEN_HIDDEN_DEF
weak_alias (__strlen, strlen)
libc_hidden_builtin_def (strlen)
#endif