mirror of
https://sourceware.org/git/glibc.git
synced 2024-11-24 14:00:30 +00:00
LoongArch: Add ifunc support for memrchr{lsx, lasx}
According to glibc memrchr microbenchmark, this implementation could reduce the runtime as following: Name Percent of rutime reduced memrchr-lasx 20%-83% memrchr-lsx 20%-64%
This commit is contained in:
dengjianbo
caiyinyu
parent
60bcb9acbf
commit
55e84dc6ed
@ -27,5 +27,8 @@ sysdep_routines += \
|
||||
memchr-aligned \
|
||||
memchr-lsx \
|
||||
memchr-lasx \
|
||||
memrchr-generic \
|
||||
memrchr-lsx \
|
||||
memrchr-lasx \
|
||||
# sysdep_routines
|
||||
endif
|
||||
|
||||
@ -109,5 +109,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
||||
#endif
|
||||
IFUNC_IMPL_ADD (array, i, memchr, 1, __memchr_aligned)
|
||||
)
|
||||
|
||||
IFUNC_IMPL (i, name, memrchr,
|
||||
#if !defined __loongarch_soft_float
|
||||
IFUNC_IMPL_ADD (array, i, memrchr, SUPPORT_LASX, __memrchr_lasx)
|
||||
IFUNC_IMPL_ADD (array, i, memrchr, SUPPORT_LSX, __memrchr_lsx)
|
||||
#endif
|
||||
IFUNC_IMPL_ADD (array, i, memrchr, 1, __memrchr_generic)
|
||||
)
|
||||
return i;
|
||||
}
|
||||
|
||||
40
sysdeps/loongarch/lp64/multiarch/ifunc-memrchr.h
Normal file
40
sysdeps/loongarch/lp64/multiarch/ifunc-memrchr.h
Normal file
@ -0,0 +1,40 @@
|
||||
/* Common definition for memrchr implementation.
|
||||
All versions must be listed in ifunc-impl-list.c.
|
||||
Copyright (C) 2023 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <ldsodefs.h>
|
||||
#include <ifunc-init.h>
|
||||
|
||||
#if !defined __loongarch_soft_float
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE (lasx) attribute_hidden;
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE (lsx) attribute_hidden;
|
||||
#endif
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE (generic) attribute_hidden;
|
||||
|
||||
static inline void *
|
||||
IFUNC_SELECTOR (void)
|
||||
{
|
||||
#if !defined __loongarch_soft_float
|
||||
if (SUPPORT_LASX)
|
||||
return OPTIMIZE (lasx);
|
||||
else if (SUPPORT_LSX)
|
||||
return OPTIMIZE (lsx);
|
||||
else
|
||||
#endif
|
||||
return OPTIMIZE (generic);
|
||||
}
|
||||
23
sysdeps/loongarch/lp64/multiarch/memrchr-generic.c
Normal file
23
sysdeps/loongarch/lp64/multiarch/memrchr-generic.c
Normal file
@ -0,0 +1,23 @@
|
||||
/* Generic implementation of memrchr.
|
||||
Copyright (C) 2023 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library. If not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#if IS_IN (libc)
|
||||
# define MEMRCHR __memrchr_generic
|
||||
#endif
|
||||
|
||||
#include <string/memrchr.c>
|
||||
123
sysdeps/loongarch/lp64/multiarch/memrchr-lasx.S
Normal file
123
sysdeps/loongarch/lp64/multiarch/memrchr-lasx.S
Normal file
@ -0,0 +1,123 @@
|
||||
/* Optimized memrchr implementation using LoongArch LASX instructions.
|
||||
Copyright (C) 2023 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library. If not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <sysdep.h>
|
||||
#include <sys/regdef.h>
|
||||
#include <sys/asm.h>
|
||||
|
||||
#if IS_IN (libc) && !defined __loongarch_soft_float
|
||||
|
||||
#ifndef MEMRCHR
|
||||
# define MEMRCHR __memrchr_lasx
|
||||
#endif
|
||||
|
||||
LEAF(MEMRCHR, 6)
|
||||
beqz a2, L(ret0)
|
||||
addi.d a2, a2, -1
|
||||
add.d a3, a0, a2
|
||||
andi t1, a3, 0x3f
|
||||
|
||||
bstrins.d a3, zero, 5, 0
|
||||
addi.d t1, t1, 1
|
||||
xvld xr0, a3, 0
|
||||
xvld xr1, a3, 32
|
||||
|
||||
sub.d t2, zero, t1
|
||||
li.d t3, -1
|
||||
xvreplgr2vr.b xr2, a1
|
||||
andi t4, a0, 0x3f
|
||||
|
||||
srl.d t2, t3, t2
|
||||
xvseq.b xr0, xr0, xr2
|
||||
xvseq.b xr1, xr1, xr2
|
||||
xvmsknz.b xr0, xr0
|
||||
|
||||
|
||||
xvmsknz.b xr1, xr1
|
||||
xvpickve.w xr3, xr0, 4
|
||||
xvpickve.w xr4, xr1, 4
|
||||
vilvl.h vr0, vr3, vr0
|
||||
|
||||
vilvl.h vr1, vr4, vr1
|
||||
vilvl.w vr0, vr1, vr0
|
||||
movfr2gr.d t0, fa0
|
||||
and t0, t0, t2
|
||||
|
||||
bltu a2, t1, L(end)
|
||||
bnez t0, L(found)
|
||||
bstrins.d a0, zero, 5, 0
|
||||
L(loop):
|
||||
xvld xr0, a3, -64
|
||||
|
||||
xvld xr1, a3, -32
|
||||
addi.d a3, a3, -64
|
||||
xvseq.b xr0, xr0, xr2
|
||||
xvseq.b xr1, xr1, xr2
|
||||
|
||||
|
||||
beq a0, a3, L(out)
|
||||
xvmax.bu xr3, xr0, xr1
|
||||
xvseteqz.v fcc0, xr3
|
||||
bcnez fcc0, L(loop)
|
||||
|
||||
xvmsknz.b xr0, xr0
|
||||
xvmsknz.b xr1, xr1
|
||||
xvpickve.w xr3, xr0, 4
|
||||
xvpickve.w xr4, xr1, 4
|
||||
|
||||
vilvl.h vr0, vr3, vr0
|
||||
vilvl.h vr1, vr4, vr1
|
||||
vilvl.w vr0, vr1, vr0
|
||||
movfr2gr.d t0, fa0
|
||||
|
||||
L(found):
|
||||
addi.d a0, a3, 63
|
||||
clz.d t1, t0
|
||||
sub.d a0, a0, t1
|
||||
jr ra
|
||||
|
||||
|
||||
L(out):
|
||||
xvmsknz.b xr0, xr0
|
||||
xvmsknz.b xr1, xr1
|
||||
xvpickve.w xr3, xr0, 4
|
||||
xvpickve.w xr4, xr1, 4
|
||||
|
||||
vilvl.h vr0, vr3, vr0
|
||||
vilvl.h vr1, vr4, vr1
|
||||
vilvl.w vr0, vr1, vr0
|
||||
movfr2gr.d t0, fa0
|
||||
|
||||
L(end):
|
||||
sll.d t2, t3, t4
|
||||
and t0, t0, t2
|
||||
addi.d a0, a3, 63
|
||||
clz.d t1, t0
|
||||
|
||||
sub.d a0, a0, t1
|
||||
maskeqz a0, a0, t0
|
||||
jr ra
|
||||
L(ret0):
|
||||
move a0, zero
|
||||
|
||||
|
||||
jr ra
|
||||
END(MEMRCHR)
|
||||
|
||||
libc_hidden_builtin_def (MEMRCHR)
|
||||
#endif
|
||||
105
sysdeps/loongarch/lp64/multiarch/memrchr-lsx.S
Normal file
105
sysdeps/loongarch/lp64/multiarch/memrchr-lsx.S
Normal file
@ -0,0 +1,105 @@
|
||||
/* Optimized memrchr implementation using LoongArch LSX instructions.
|
||||
Copyright (C) 2023 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library. If not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <sysdep.h>
|
||||
#include <sys/regdef.h>
|
||||
#include <sys/asm.h>
|
||||
|
||||
#if IS_IN (libc) && !defined __loongarch_soft_float
|
||||
|
||||
# define MEMRCHR __memrchr_lsx
|
||||
|
||||
LEAF(MEMRCHR, 6)
|
||||
beqz a2, L(ret0)
|
||||
addi.d a2, a2, -1
|
||||
add.d a3, a0, a2
|
||||
andi t1, a3, 0x1f
|
||||
|
||||
bstrins.d a3, zero, 4, 0
|
||||
addi.d t1, t1, 1
|
||||
vld vr0, a3, 0
|
||||
vld vr1, a3, 16
|
||||
|
||||
sub.d t2, zero, t1
|
||||
li.d t3, -1
|
||||
vreplgr2vr.b vr2, a1
|
||||
andi t4, a0, 0x1f
|
||||
|
||||
srl.d t2, t3, t2
|
||||
vseq.b vr0, vr0, vr2
|
||||
vseq.b vr1, vr1, vr2
|
||||
vmsknz.b vr0, vr0
|
||||
|
||||
|
||||
vmsknz.b vr1, vr1
|
||||
vilvl.h vr0, vr1, vr0
|
||||
movfr2gr.s t0, fa0
|
||||
and t0, t0, t2
|
||||
|
||||
bltu a2, t1, L(end)
|
||||
bnez t0, L(found)
|
||||
bstrins.d a0, zero, 4, 0
|
||||
L(loop):
|
||||
vld vr0, a3, -32
|
||||
|
||||
vld vr1, a3, -16
|
||||
addi.d a3, a3, -32
|
||||
vseq.b vr0, vr0, vr2
|
||||
vseq.b vr1, vr1, vr2
|
||||
|
||||
beq a0, a3, L(out)
|
||||
vmax.bu vr3, vr0, vr1
|
||||
vseteqz.v fcc0, vr3
|
||||
bcnez fcc0, L(loop)
|
||||
|
||||
|
||||
vmsknz.b vr0, vr0
|
||||
vmsknz.b vr1, vr1
|
||||
vilvl.h vr0, vr1, vr0
|
||||
movfr2gr.s t0, fa0
|
||||
|
||||
L(found):
|
||||
addi.d a0, a3, 31
|
||||
clz.w t1, t0
|
||||
sub.d a0, a0, t1
|
||||
jr ra
|
||||
|
||||
L(out):
|
||||
vmsknz.b vr0, vr0
|
||||
vmsknz.b vr1, vr1
|
||||
vilvl.h vr0, vr1, vr0
|
||||
movfr2gr.s t0, fa0
|
||||
|
||||
L(end):
|
||||
sll.d t2, t3, t4
|
||||
and t0, t0, t2
|
||||
addi.d a0, a3, 31
|
||||
clz.w t1, t0
|
||||
|
||||
|
||||
sub.d a0, a0, t1
|
||||
maskeqz a0, a0, t0
|
||||
jr ra
|
||||
L(ret0):
|
||||
move a0, zero
|
||||
|
||||
jr ra
|
||||
END(MEMRCHR)
|
||||
|
||||
libc_hidden_builtin_def (MEMRCHR)
|
||||
#endif
|
||||
33
sysdeps/loongarch/lp64/multiarch/memrchr.c
Normal file
33
sysdeps/loongarch/lp64/multiarch/memrchr.c
Normal file
@ -0,0 +1,33 @@
|
||||
/* Multiple versions of memrchr.
|
||||
All versions must be listed in ifunc-impl-list.c.
|
||||
Copyright (C) 2023 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
/* Define multiple versions only for the definition in libc. */
|
||||
#if IS_IN (libc)
|
||||
# define memrchr __redirect_memrchr
|
||||
# include <string.h>
|
||||
# undef memrchr
|
||||
|
||||
# define SYMBOL_NAME memrchr
|
||||
# include "ifunc-memrchr.h"
|
||||
|
||||
libc_ifunc_redirected (__redirect_memrchr, __memrchr, IFUNC_SELECTOR ());
|
||||
libc_hidden_def (__memrchr)
|
||||
weak_alias (__memrchr, memrchr)
|
||||
|
||||
#endif
|
||||
Loading…
Reference in New Issue
Block a user