AuroraMiddleware/glibc
1
0
Fork 0
mirror of https://sourceware.org/git/glibc.git synced 2024-11-25 14:30:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

x86: Move strlen SSE2 implementation to multiarch/strlen-sse2.S

Browse Source 
This commit doesn't affect libc.so.6, its just housekeeping to prepare
for adding explicit ISA level support.

Tested build on x86_64 and x86_32 with/without multiarch.
This commit is contained in:
Noah Goldstein 2022-07-12 12:29:01 -07:00
parent 60a583ec60
commit 58e6cd4bcb
9 changed files with 306 additions and 286 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
sysdeps/x86_64/multiarch/rtld-strlen.S Normal file
View File

@ -0,0 +1,18 @@
/* Copyright (C) 2022 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#include "../strlen.S"

18
sysdeps/x86_64/multiarch/rtld-strnlen.S Normal file
View File

@ -0,0 +1,18 @@
/* Copyright (C) 2022 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#include "../strnlen.S"

260
sysdeps/x86_64/multiarch/strlen-sse2.S
View File

@ -16,8 +16,260 @@
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#if IS_IN (libc)
# define strlen __strlen_sse2
#endif
#if IS_IN (libc) || defined STRLEN
#include "strlen-vec.S"
# ifndef STRLEN
# define STRLEN __strlen_sse2
# endif
# include <sysdep.h>
# ifdef AS_WCSLEN
# define PMINU pminud
# define PCMPEQ pcmpeqd
# define SHIFT_RETURN shrq $2, %rax
# else
# define PMINU pminub
# define PCMPEQ pcmpeqb
# define SHIFT_RETURN
# endif
# ifndef SECTION
# define SECTION(p) p
# endif
/* Long lived register in strlen(s), strnlen(s, n) are:
%xmm3 - zero
%rdi - s
%r10 (s+n) & (~(64-1))
%r11 s+n
*/
.section SECTION(.text),"ax",@progbits
ENTRY(STRLEN)
/* Test 64 bytes from %rax for zero. Save result as bitmask in %rdx. */
# define FIND_ZERO \
PCMPEQ (%rax), %xmm0; \
PCMPEQ 16(%rax), %xmm1; \
PCMPEQ 32(%rax), %xmm2; \
PCMPEQ 48(%rax), %xmm3; \
pmovmskb %xmm0, %esi; \
pmovmskb %xmm1, %edx; \
pmovmskb %xmm2, %r8d; \
pmovmskb %xmm3, %ecx; \
salq $16, %rdx; \
salq $16, %rcx; \
orq %rsi, %rdx; \
orq %r8, %rcx; \
salq $32, %rcx; \
orq %rcx, %rdx;
# ifdef AS_STRNLEN
/* Do not read anything when n==0. */
test %RSI_LP, %RSI_LP
jne L(n_nonzero)
xor %rax, %rax
ret
L(n_nonzero):
# ifdef AS_WCSLEN
/* Check for overflow from maxlen * sizeof(wchar_t). If it would
overflow the only way this program doesn't have undefined behavior
is if there is a null terminator in valid memory so wcslen will
suffice. */
mov %RSI_LP, %R10_LP
sar $62, %R10_LP
jnz __wcslen_sse4_1
sal $2, %RSI_LP
# endif
/* Initialize long lived registers. */
add %RDI_LP, %RSI_LP
mov %RSI_LP, %R10_LP
and $-64, %R10_LP
mov %RSI_LP, %R11_LP
# endif
pxor %xmm0, %xmm0
pxor %xmm1, %xmm1
pxor %xmm2, %xmm2
pxor %xmm3, %xmm3
movq %rdi, %rax
movq %rdi, %rcx
andq $4095, %rcx
/* Offsets 4032-4047 will be aligned into 4032 thus fit into page. */
cmpq $4047, %rcx
/* We cannot unify this branching as it would be ~6 cycles slower. */
ja L(cross_page)
# ifdef AS_STRNLEN
/* Test if end is among first 64 bytes. */
# define STRNLEN_PROLOG \
mov %r11, %rsi; \
subq %rax, %rsi; \
andq $-64, %rax; \
testq $-64, %rsi; \
je L(strnlen_ret)
# else
# define STRNLEN_PROLOG andq $-64, %rax;
# endif
/* Ignore bits in mask that come before start of string. */
# define PROLOG(lab) \
movq %rdi, %rcx; \
xorq %rax, %rcx; \
STRNLEN_PROLOG; \
sarq %cl, %rdx; \
test %rdx, %rdx; \
je L(lab); \
bsfq %rdx, %rax; \
SHIFT_RETURN; \
ret
# ifdef AS_STRNLEN
andq $-16, %rax
FIND_ZERO
# else
/* Test first 16 bytes unaligned. */
movdqu (%rax), %xmm4
PCMPEQ %xmm0, %xmm4
pmovmskb %xmm4, %edx
test %edx, %edx
je L(next48_bytes)
bsf %edx, %eax /* If eax is zeroed 16bit bsf can be used. */
SHIFT_RETURN
ret
L(next48_bytes):
/* Same as FIND_ZERO except we do not check first 16 bytes. */
andq $-16, %rax
PCMPEQ 16(%rax), %xmm1
PCMPEQ 32(%rax), %xmm2
PCMPEQ 48(%rax), %xmm3
pmovmskb %xmm1, %edx
pmovmskb %xmm2, %r8d
pmovmskb %xmm3, %ecx
salq $16, %rdx
salq $16, %rcx
orq %r8, %rcx
salq $32, %rcx
orq %rcx, %rdx
# endif
/* When no zero byte is found xmm1-3 are zero so we do not have to
zero them. */
PROLOG(loop)
.p2align 4
L(cross_page):
andq $-64, %rax
FIND_ZERO
PROLOG(loop_init)
# ifdef AS_STRNLEN
/* We must do this check to correctly handle strnlen (s, -1). */
L(strnlen_ret):
bts %rsi, %rdx
sarq %cl, %rdx
test %rdx, %rdx
je L(loop_init)
bsfq %rdx, %rax
SHIFT_RETURN
ret
# endif
.p2align 4
L(loop_init):
pxor %xmm1, %xmm1
pxor %xmm2, %xmm2
pxor %xmm3, %xmm3
# ifdef AS_STRNLEN
.p2align 4
L(loop):
addq $64, %rax
cmpq %rax, %r10
je L(exit_end)
movdqa (%rax), %xmm0
PMINU 16(%rax), %xmm0
PMINU 32(%rax), %xmm0
PMINU 48(%rax), %xmm0
PCMPEQ %xmm3, %xmm0
pmovmskb %xmm0, %edx
testl %edx, %edx
jne L(exit)
jmp L(loop)
.p2align 4
L(exit_end):
cmp %rax, %r11
je L(first) /* Do not read when end is at page boundary. */
pxor %xmm0, %xmm0
FIND_ZERO
L(first):
bts %r11, %rdx
bsfq %rdx, %rdx
addq %rdx, %rax
subq %rdi, %rax
SHIFT_RETURN
ret
.p2align 4
L(exit):
pxor %xmm0, %xmm0
FIND_ZERO
bsfq %rdx, %rdx
addq %rdx, %rax
subq %rdi, %rax
SHIFT_RETURN
ret
# else
/* Main loop. Unrolled twice to improve L2 cache performance on core2. */
.p2align 4
L(loop):
movdqa 64(%rax), %xmm0
PMINU 80(%rax), %xmm0
PMINU 96(%rax), %xmm0
PMINU 112(%rax), %xmm0
PCMPEQ %xmm3, %xmm0
pmovmskb %xmm0, %edx
testl %edx, %edx
jne L(exit64)
subq $-128, %rax
movdqa (%rax), %xmm0
PMINU 16(%rax), %xmm0
PMINU 32(%rax), %xmm0
PMINU 48(%rax), %xmm0
PCMPEQ %xmm3, %xmm0
pmovmskb %xmm0, %edx
testl %edx, %edx
jne L(exit0)
jmp L(loop)
.p2align 4
L(exit64):
addq $64, %rax
L(exit0):
pxor %xmm0, %xmm0
FIND_ZERO
bsfq %rdx, %rdx
addq %rdx, %rax
subq %rdi, %rax
SHIFT_RETURN
ret
# endif
END(STRLEN)
#endif

267
sysdeps/x86_64/multiarch/strlen-vec.S
View File

@ -1,267 +0,0 @@
/* SSE2 version of strlen and SSE4.1 version of wcslen.
Copyright (C) 2012-2022 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#include <sysdep.h>
#ifdef AS_WCSLEN
# define PMINU pminud
# define PCMPEQ pcmpeqd
# define SHIFT_RETURN shrq $2, %rax
#else
# define PMINU pminub
# define PCMPEQ pcmpeqb
# define SHIFT_RETURN
#endif
#ifndef SECTION
# define SECTION(p) p
#endif
/* Long lived register in strlen(s), strnlen(s, n) are:
%xmm3 - zero
%rdi - s
%r10 (s+n) & (~(64-1))
%r11 s+n
*/
.section SECTION(.text),"ax",@progbits
ENTRY(strlen)
/* Test 64 bytes from %rax for zero. Save result as bitmask in %rdx. */
#define FIND_ZERO \
PCMPEQ (%rax), %xmm0; \
PCMPEQ 16(%rax), %xmm1; \
PCMPEQ 32(%rax), %xmm2; \
PCMPEQ 48(%rax), %xmm3; \
pmovmskb %xmm0, %esi; \
pmovmskb %xmm1, %edx; \
pmovmskb %xmm2, %r8d; \
pmovmskb %xmm3, %ecx; \
salq $16, %rdx; \
salq $16, %rcx; \
orq %rsi, %rdx; \
orq %r8, %rcx; \
salq $32, %rcx; \
orq %rcx, %rdx;
#ifdef AS_STRNLEN
/* Do not read anything when n==0. */
test %RSI_LP, %RSI_LP
jne L(n_nonzero)
xor %rax, %rax
ret
L(n_nonzero):
# ifdef AS_WCSLEN
/* Check for overflow from maxlen * sizeof(wchar_t). If it would
overflow the only way this program doesn't have undefined behavior
is if there is a null terminator in valid memory so wcslen will
suffice. */
mov %RSI_LP, %R10_LP
sar $62, %R10_LP
jnz __wcslen_sse4_1
sal $2, %RSI_LP
# endif
/* Initialize long lived registers. */
add %RDI_LP, %RSI_LP
mov %RSI_LP, %R10_LP
and $-64, %R10_LP
mov %RSI_LP, %R11_LP
#endif
pxor %xmm0, %xmm0
pxor %xmm1, %xmm1
pxor %xmm2, %xmm2
pxor %xmm3, %xmm3
movq %rdi, %rax
movq %rdi, %rcx
andq $4095, %rcx
/* Offsets 4032-4047 will be aligned into 4032 thus fit into page. */
cmpq $4047, %rcx
/* We cannot unify this branching as it would be ~6 cycles slower. */
ja L(cross_page)
#ifdef AS_STRNLEN
/* Test if end is among first 64 bytes. */
# define STRNLEN_PROLOG \
mov %r11, %rsi; \
subq %rax, %rsi; \
andq $-64, %rax; \
testq $-64, %rsi; \
je L(strnlen_ret)
#else
# define STRNLEN_PROLOG andq $-64, %rax;
#endif
/* Ignore bits in mask that come before start of string. */
#define PROLOG(lab) \
movq %rdi, %rcx; \
xorq %rax, %rcx; \
STRNLEN_PROLOG; \
sarq %cl, %rdx; \
test %rdx, %rdx; \
je L(lab); \
bsfq %rdx, %rax; \
SHIFT_RETURN; \
ret
#ifdef AS_STRNLEN
andq $-16, %rax
FIND_ZERO
#else
/* Test first 16 bytes unaligned. */
movdqu (%rax), %xmm4
PCMPEQ %xmm0, %xmm4
pmovmskb %xmm4, %edx
test %edx, %edx
je L(next48_bytes)
bsf %edx, %eax /* If eax is zeroed 16bit bsf can be used. */
SHIFT_RETURN
ret
L(next48_bytes):
/* Same as FIND_ZERO except we do not check first 16 bytes. */
andq $-16, %rax
PCMPEQ 16(%rax), %xmm1
PCMPEQ 32(%rax), %xmm2
PCMPEQ 48(%rax), %xmm3
pmovmskb %xmm1, %edx
pmovmskb %xmm2, %r8d
pmovmskb %xmm3, %ecx
salq $16, %rdx
salq $16, %rcx
orq %r8, %rcx
salq $32, %rcx
orq %rcx, %rdx
#endif
/* When no zero byte is found xmm1-3 are zero so we do not have to
zero them. */
PROLOG(loop)
.p2align 4
L(cross_page):
andq $-64, %rax
FIND_ZERO
PROLOG(loop_init)
#ifdef AS_STRNLEN
/* We must do this check to correctly handle strnlen (s, -1). */
L(strnlen_ret):
bts %rsi, %rdx
sarq %cl, %rdx
test %rdx, %rdx
je L(loop_init)
bsfq %rdx, %rax
SHIFT_RETURN
ret
#endif
.p2align 4
L(loop_init):
pxor %xmm1, %xmm1
pxor %xmm2, %xmm2
pxor %xmm3, %xmm3
#ifdef AS_STRNLEN
.p2align 4
L(loop):
addq $64, %rax
cmpq %rax, %r10
je L(exit_end)
movdqa (%rax), %xmm0
PMINU 16(%rax), %xmm0
PMINU 32(%rax), %xmm0
PMINU 48(%rax), %xmm0
PCMPEQ %xmm3, %xmm0
pmovmskb %xmm0, %edx
testl %edx, %edx
jne L(exit)
jmp L(loop)
.p2align 4
L(exit_end):
cmp %rax, %r11
je L(first) /* Do not read when end is at page boundary. */
pxor %xmm0, %xmm0
FIND_ZERO
L(first):
bts %r11, %rdx
bsfq %rdx, %rdx
addq %rdx, %rax
subq %rdi, %rax
SHIFT_RETURN
ret
.p2align 4
L(exit):
pxor %xmm0, %xmm0
FIND_ZERO
bsfq %rdx, %rdx
addq %rdx, %rax
subq %rdi, %rax
SHIFT_RETURN
ret
#else
/* Main loop. Unrolled twice to improve L2 cache performance on core2. */
.p2align 4
L(loop):
movdqa 64(%rax), %xmm0
PMINU 80(%rax), %xmm0
PMINU 96(%rax), %xmm0
PMINU 112(%rax), %xmm0
PCMPEQ %xmm3, %xmm0
pmovmskb %xmm0, %edx
testl %edx, %edx
jne L(exit64)
subq $-128, %rax
movdqa (%rax), %xmm0
PMINU 16(%rax), %xmm0
PMINU 32(%rax), %xmm0
PMINU 48(%rax), %xmm0
PCMPEQ %xmm3, %xmm0
pmovmskb %xmm0, %edx
testl %edx, %edx
jne L(exit0)
jmp L(loop)
.p2align 4
L(exit64):
addq $64, %rax
L(exit0):
pxor %xmm0, %xmm0
FIND_ZERO
bsfq %rdx, %rdx
addq %rdx, %rax
subq %rdi, %rax
SHIFT_RETURN
ret
#endif
END(strlen)

12
sysdeps/x86_64/multiarch/strnlen-sse2.S
View File

@ -17,12 +17,10 @@
<https://www.gnu.org/licenses/>. */
#if IS_IN (libc)
# define __strnlen __strnlen_sse2
# undef weak_alias
# define weak_alias(__strnlen, strnlen)
# undef libc_hidden_builtin_def
# define libc_hidden_builtin_def(strnlen)
# ifndef STRLEN
# define STRLEN __strnlen_sse2
# endif
#endif
#include "../strnlen.S"
#define AS_STRNLEN
#include "strlen-sse2.S"

4
sysdeps/x86_64/multiarch/wcslen-sse4_1.S
View File

@ -1,5 +1,5 @@
#define AS_WCSLEN
#define strlen __wcslen_sse4_1
#define STRLEN __wcslen_sse4_1
#define SECTION(p) p##.sse4.1
#include "strlen-vec.S"
#include "strlen-sse2.S"

4
sysdeps/x86_64/multiarch/wcsnlen-sse4_1.S
View File

@ -1,6 +1,6 @@
#define AS_WCSLEN
#define AS_STRNLEN
#define strlen __wcsnlen_sse4_1
#define STRLEN __wcsnlen_sse4_1
#define SECTION(p) p##.sse4.1
#include "strlen-vec.S"
#include "strlen-sse2.S"

3
sysdeps/x86_64/strlen.S
View File

@ -16,6 +16,7 @@
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#include "multiarch/strlen-vec.S"
#define STRLEN strlen
#include "multiarch/strlen-sse2.S"
libc_hidden_builtin_def (strlen)

6
sysdeps/x86_64/strnlen.S
View File

@ -1,6 +1,6 @@
#define AS_STRNLEN
#define strlen __strnlen
#include "strlen.S"
#define STRLEN __strnlen
#include "multiarch/strnlen-sse2.S"
libc_hidden_def (__strnlen)
weak_alias (__strnlen, strnlen);
libc_hidden_builtin_def (strnlen)