mirror of
https://sourceware.org/git/glibc.git
synced 2024-12-11 22:00:08 +00:00
AArch64: Add support for MOPS memcpy/memmove/memset
Add support for MOPS in cpu_features and INIT_ARCH. Add ifuncs using MOPS for
memcpy, memmove and memset (use .inst for now so it works with all binutils
versions without needing complex configure and conditional compilation).
Reviewed-by: Szabolcs Nagy <szabolcs.nagy@arm.com>
(cherry picked from commit 2bd0017988
)
This commit is contained in:
parent
cb0012f076
commit
3b79e57c1c
@ -5,14 +5,17 @@ sysdep_routines += \
|
||||
memcpy_a64fx \
|
||||
memcpy_falkor \
|
||||
memcpy_generic \
|
||||
memcpy_mops \
|
||||
memcpy_sve \
|
||||
memcpy_thunderx \
|
||||
memcpy_thunderx2 \
|
||||
memmove_mops \
|
||||
memset_a64fx \
|
||||
memset_emag \
|
||||
memset_falkor \
|
||||
memset_generic \
|
||||
memset_kunpeng \
|
||||
memset_mops \
|
||||
strlen_asimd \
|
||||
strlen_mte \
|
||||
# sysdep_routines
|
||||
|
@ -46,6 +46,7 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
||||
IFUNC_IMPL_ADD (array, i, memcpy, sve, __memcpy_a64fx)
|
||||
IFUNC_IMPL_ADD (array, i, memcpy, sve, __memcpy_sve)
|
||||
#endif
|
||||
IFUNC_IMPL_ADD (array, i, memcpy, mops, __memcpy_mops)
|
||||
IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_generic))
|
||||
IFUNC_IMPL (i, name, memmove,
|
||||
IFUNC_IMPL_ADD (array, i, memmove, 1, __memmove_thunderx)
|
||||
@ -55,6 +56,7 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
||||
IFUNC_IMPL_ADD (array, i, memmove, sve, __memmove_a64fx)
|
||||
IFUNC_IMPL_ADD (array, i, memmove, sve, __memmove_sve)
|
||||
#endif
|
||||
IFUNC_IMPL_ADD (array, i, memmove, mops, __memmove_mops)
|
||||
IFUNC_IMPL_ADD (array, i, memmove, 1, __memmove_generic))
|
||||
IFUNC_IMPL (i, name, memset,
|
||||
/* Enable this on non-falkor processors too so that other cores
|
||||
@ -65,6 +67,7 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
||||
#if HAVE_AARCH64_SVE_ASM
|
||||
IFUNC_IMPL_ADD (array, i, memset, sve && zva_size == 256, __memset_a64fx)
|
||||
#endif
|
||||
IFUNC_IMPL_ADD (array, i, memset, mops, __memset_mops)
|
||||
IFUNC_IMPL_ADD (array, i, memset, 1, __memset_generic))
|
||||
IFUNC_IMPL (i, name, memchr,
|
||||
IFUNC_IMPL_ADD (array, i, memchr, !mte, __memchr_nosimd)
|
||||
|
@ -35,4 +35,6 @@
|
||||
bool __attribute__((unused)) mte = \
|
||||
MTE_ENABLED (); \
|
||||
bool __attribute__((unused)) sve = \
|
||||
GLRO(dl_aarch64_cpu_features).sve;
|
||||
GLRO(dl_aarch64_cpu_features).sve; \
|
||||
bool __attribute__((unused)) mops = \
|
||||
GLRO(dl_aarch64_cpu_features).mops;
|
||||
|
@ -34,12 +34,16 @@ extern __typeof (__redirect_memcpy) __memcpy_thunderx2 attribute_hidden;
|
||||
extern __typeof (__redirect_memcpy) __memcpy_falkor attribute_hidden;
|
||||
extern __typeof (__redirect_memcpy) __memcpy_a64fx attribute_hidden;
|
||||
extern __typeof (__redirect_memcpy) __memcpy_sve attribute_hidden;
|
||||
extern __typeof (__redirect_memcpy) __memcpy_mops attribute_hidden;
|
||||
|
||||
static inline __typeof (__redirect_memcpy) *
|
||||
select_memcpy_ifunc (void)
|
||||
{
|
||||
INIT_ARCH ();
|
||||
|
||||
if (mops)
|
||||
return __memcpy_mops;
|
||||
|
||||
if (sve && HAVE_AARCH64_SVE_ASM)
|
||||
{
|
||||
if (IS_A64FX (midr))
|
||||
|
39
sysdeps/aarch64/multiarch/memcpy_mops.S
Normal file
39
sysdeps/aarch64/multiarch/memcpy_mops.S
Normal file
@ -0,0 +1,39 @@
|
||||
/* Optimized memcpy for MOPS.
|
||||
Copyright (C) 2023 Free Software Foundation, Inc.
|
||||
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library. If not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <sysdep.h>
|
||||
|
||||
/* Assumptions:
|
||||
*
|
||||
* AArch64, MOPS.
|
||||
*
|
||||
*/
|
||||
|
||||
ENTRY (__memcpy_mops)
|
||||
PTR_ARG (0)
|
||||
PTR_ARG (1)
|
||||
SIZE_ARG (2)
|
||||
|
||||
mov x3, x0
|
||||
.inst 0x19010443 /* cpyfp [x3]!, [x1]!, x2! */
|
||||
.inst 0x19410443 /* cpyfm [x3]!, [x1]!, x2! */
|
||||
.inst 0x19810443 /* cpyfe [x3]!, [x1]!, x2! */
|
||||
ret
|
||||
|
||||
END (__memcpy_mops)
|
@ -34,12 +34,16 @@ extern __typeof (__redirect_memmove) __memmove_thunderx2 attribute_hidden;
|
||||
extern __typeof (__redirect_memmove) __memmove_falkor attribute_hidden;
|
||||
extern __typeof (__redirect_memmove) __memmove_a64fx attribute_hidden;
|
||||
extern __typeof (__redirect_memmove) __memmove_sve attribute_hidden;
|
||||
extern __typeof (__redirect_memmove) __memmove_mops attribute_hidden;
|
||||
|
||||
static inline __typeof (__redirect_memmove) *
|
||||
select_memmove_ifunc (void)
|
||||
{
|
||||
INIT_ARCH ();
|
||||
|
||||
if (mops)
|
||||
return __memmove_mops;
|
||||
|
||||
if (sve && HAVE_AARCH64_SVE_ASM)
|
||||
{
|
||||
if (IS_A64FX (midr))
|
||||
|
39
sysdeps/aarch64/multiarch/memmove_mops.S
Normal file
39
sysdeps/aarch64/multiarch/memmove_mops.S
Normal file
@ -0,0 +1,39 @@
|
||||
/* Optimized memmove for MOPS.
|
||||
Copyright (C) 2023 Free Software Foundation, Inc.
|
||||
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library. If not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <sysdep.h>
|
||||
|
||||
/* Assumptions:
|
||||
*
|
||||
* AArch64, MOPS.
|
||||
*
|
||||
*/
|
||||
|
||||
ENTRY (__memmove_mops)
|
||||
PTR_ARG (0)
|
||||
PTR_ARG (1)
|
||||
SIZE_ARG (2)
|
||||
|
||||
mov x3, x0
|
||||
.inst 0x1d010443 /* cpyp [x3]!, [x1]!, x2! */
|
||||
.inst 0x1d410443 /* cpym [x3]!, [x1]!, x2! */
|
||||
.inst 0x1d810443 /* cpye [x3]!, [x1]!, x2! */
|
||||
ret
|
||||
|
||||
END (__memmove_mops)
|
@ -33,12 +33,16 @@ extern __typeof (__redirect_memset) __memset_emag attribute_hidden;
|
||||
extern __typeof (__redirect_memset) __memset_kunpeng attribute_hidden;
|
||||
extern __typeof (__redirect_memset) __memset_a64fx attribute_hidden;
|
||||
extern __typeof (__redirect_memset) __memset_generic attribute_hidden;
|
||||
extern __typeof (__redirect_memset) __memset_mops attribute_hidden;
|
||||
|
||||
static inline __typeof (__redirect_memset) *
|
||||
select_memset_ifunc (void)
|
||||
{
|
||||
INIT_ARCH ();
|
||||
|
||||
if (mops)
|
||||
return __memset_mops;
|
||||
|
||||
if (sve && HAVE_AARCH64_SVE_ASM)
|
||||
{
|
||||
if (IS_A64FX (midr) && zva_size == 256)
|
||||
|
38
sysdeps/aarch64/multiarch/memset_mops.S
Normal file
38
sysdeps/aarch64/multiarch/memset_mops.S
Normal file
@ -0,0 +1,38 @@
|
||||
/* Optimized memset for MOPS.
|
||||
Copyright (C) 2023 Free Software Foundation, Inc.
|
||||
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library. If not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <sysdep.h>
|
||||
|
||||
/* Assumptions:
|
||||
*
|
||||
* AArch64, MOPS.
|
||||
*
|
||||
*/
|
||||
|
||||
ENTRY (__memset_mops)
|
||||
PTR_ARG (0)
|
||||
SIZE_ARG (2)
|
||||
|
||||
mov x3, x0
|
||||
.inst 0x19c10443 /* setp [x3]!, x2!, x1 */
|
||||
.inst 0x19c14443 /* setm [x3]!, x2!, x1 */
|
||||
.inst 0x19c18443 /* sete [x3]!, x2!, x1 */
|
||||
ret
|
||||
|
||||
END (__memset_mops)
|
@ -120,4 +120,7 @@ init_cpu_features (struct cpu_features *cpu_features)
|
||||
|
||||
/* Check if SVE is supported. */
|
||||
cpu_features->sve = GLRO (dl_hwcap) & HWCAP_SVE;
|
||||
|
||||
/* Check if MOPS is supported. */
|
||||
cpu_features->mops = GLRO (dl_hwcap2) & HWCAP2_MOPS;
|
||||
}
|
||||
|
@ -76,6 +76,7 @@ struct cpu_features
|
||||
/* Currently, the GLIBC memory tagging tunable only defines 8 bits. */
|
||||
uint8_t mte_state;
|
||||
bool sve;
|
||||
bool mops;
|
||||
};
|
||||
|
||||
#endif /* _CPU_FEATURES_AARCH64_H */
|
||||
|
Loading…
Reference in New Issue
Block a user