2002-09-20 23:46:03 +00:00
|
|
|
/* Assembly macros for 32-bit PowerPC.
|
2023-01-06 21:08:04 +00:00
|
|
|
Copyright (C) 1999-2023 Free Software Foundation, Inc.
|
2002-09-20 23:46:03 +00:00
|
|
|
This file is part of the GNU C Library.
|
|
|
|
|
|
|
|
The GNU C Library is free software; you can redistribute it and/or
|
|
|
|
modify it under the terms of the GNU Lesser General Public
|
|
|
|
License as published by the Free Software Foundation; either
|
|
|
|
version 2.1 of the License, or (at your option) any later version.
|
|
|
|
|
|
|
|
The GNU C Library is distributed in the hope that it will be useful,
|
|
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
Lesser General Public License for more details.
|
|
|
|
|
|
|
|
You should have received a copy of the GNU Lesser General Public
|
2012-02-09 23:18:22 +00:00
|
|
|
License along with the GNU C Library; if not, see
|
Prefer https to http for gnu.org and fsf.org URLs
Also, change sources.redhat.com to sourceware.org.
This patch was automatically generated by running the following shell
script, which uses GNU sed, and which avoids modifying files imported
from upstream:
sed -ri '
s,(http|ftp)(://(.*\.)?(gnu|fsf|sourceware)\.org($|[^.]|\.[^a-z])),https\2,g
s,(http|ftp)(://(.*\.)?)sources\.redhat\.com($|[^.]|\.[^a-z]),https\2sourceware.org\4,g
' \
$(find $(git ls-files) -prune -type f \
! -name '*.po' \
! -name 'ChangeLog*' \
! -path COPYING ! -path COPYING.LIB \
! -path manual/fdl-1.3.texi ! -path manual/lgpl-2.1.texi \
! -path manual/texinfo.tex ! -path scripts/config.guess \
! -path scripts/config.sub ! -path scripts/install-sh \
! -path scripts/mkinstalldirs ! -path scripts/move-if-change \
! -path INSTALL ! -path locale/programs/charmap-kw.h \
! -path po/libc.pot ! -path sysdeps/gnu/errlist.c \
! '(' -name configure \
-execdir test -f configure.ac -o -f configure.in ';' ')' \
! '(' -name preconfigure \
-execdir test -f preconfigure.ac ';' ')' \
-print)
and then by running 'make dist-prepare' to regenerate files built
from the altered files, and then executing the following to cleanup:
chmod a+x sysdeps/unix/sysv/linux/riscv/configure
# Omit irrelevant whitespace and comment-only changes,
# perhaps from a slightly-different Autoconf version.
git checkout -f \
sysdeps/csky/configure \
sysdeps/hppa/configure \
sysdeps/riscv/configure \
sysdeps/unix/sysv/linux/csky/configure
# Omit changes that caused a pre-commit check to fail like this:
# remote: *** error: sysdeps/powerpc/powerpc64/ppc-mcount.S: trailing lines
git checkout -f \
sysdeps/powerpc/powerpc64/ppc-mcount.S \
sysdeps/unix/sysv/linux/s390/s390-64/syscall.S
# Omit change that caused a pre-commit check to fail like this:
# remote: *** error: sysdeps/sparc/sparc64/multiarch/memcpy-ultra3.S: last line does not end in newline
git checkout -f sysdeps/sparc/sparc64/multiarch/memcpy-ultra3.S
2019-09-07 05:40:42 +00:00
|
|
|
<https://www.gnu.org/licenses/>. */
|
2002-09-20 23:46:03 +00:00
|
|
|
|
|
|
|
#include <sysdeps/powerpc/sysdep.h>
|
|
|
|
|
2004-05-26 04:47:00 +00:00
|
|
|
#ifdef __ASSEMBLER__
|
|
|
|
|
2002-09-20 23:46:03 +00:00
|
|
|
/* If compiled for profiling, call `_mcount' at the start of each
|
|
|
|
function. */
|
|
|
|
#ifdef PROF
|
|
|
|
/* The mcount code relies on a the return address being on the stack
|
|
|
|
to locate our caller and so it can restore it; so store one just
|
|
|
|
for its benefit. */
|
2005-06-17 23:11:35 +00:00
|
|
|
# define CALL_MCOUNT \
|
2004-12-22 20:10:10 +00:00
|
|
|
mflr r0; \
|
2006-01-04 20:05:05 +00:00
|
|
|
stw r0,4(r1); \
|
2012-08-02 19:04:29 +00:00
|
|
|
cfi_offset (lr, 4); \
|
2004-12-22 20:10:10 +00:00
|
|
|
bl JUMPTARGET(_mcount);
|
2002-09-20 23:46:03 +00:00
|
|
|
#else /* PROF */
|
2003-02-20 20:30:02 +00:00
|
|
|
# define CALL_MCOUNT /* Do nothing. */
|
2002-09-20 23:46:03 +00:00
|
|
|
#endif /* PROF */
|
|
|
|
|
|
|
|
#define ENTRY(name) \
|
2012-08-02 19:04:29 +00:00
|
|
|
.globl C_SYMBOL_NAME(name); \
|
|
|
|
.type C_SYMBOL_NAME(name),@function; \
|
2002-09-20 23:46:03 +00:00
|
|
|
.align ALIGNARG(2); \
|
|
|
|
C_LABEL(name) \
|
2006-01-04 20:05:05 +00:00
|
|
|
cfi_startproc; \
|
2002-09-20 23:46:03 +00:00
|
|
|
CALL_MCOUNT
|
|
|
|
|
PowerPC64 ENTRY_TOCLESS
A number of functions in the sysdeps/powerpc/powerpc64/ tree don't use
or change r2, yet declare a global entry that sets up r2. This patch
fixes that problem, and consolidates the ENTRY and EALIGN macros.
* sysdeps/powerpc/powerpc64/sysdep.h: Formatting.
(NOPS, ENTRY_3): New macros.
(ENTRY): Rewrite.
(ENTRY_TOCLESS): Define.
(EALIGN, EALIGN_W_0, EALIGN_W_1, EALIGN_W_2, EALIGN_W_4, EALIGN_W_5,
EALIGN_W_6, EALIGN_W_7, EALIGN_W_8): Delete.
* sysdeps/powerpc/powerpc64/a2/memcpy.S: Replace EALIGN with ENTRY.
* sysdeps/powerpc/powerpc64/dl-trampoline.S: Likewise.
* sysdeps/powerpc/powerpc64/fpu/s_ceil.S: Likewise.
* sysdeps/powerpc/powerpc64/fpu/s_ceilf.S: Likewise.
* sysdeps/powerpc/powerpc64/fpu/s_floor.S: Likewise.
* sysdeps/powerpc/powerpc64/fpu/s_floorf.S: Likewise.
* sysdeps/powerpc/powerpc64/fpu/s_nearbyint.S: Likewise.
* sysdeps/powerpc/powerpc64/fpu/s_nearbyintf.S: Likewise.
* sysdeps/powerpc/powerpc64/fpu/s_rint.S: Likewise.
* sysdeps/powerpc/powerpc64/fpu/s_rintf.S: Likewise.
* sysdeps/powerpc/powerpc64/fpu/s_round.S: Likewise.
* sysdeps/powerpc/powerpc64/fpu/s_roundf.S: Likewise.
* sysdeps/powerpc/powerpc64/fpu/s_trunc.S: Likewise.
* sysdeps/powerpc/powerpc64/fpu/s_truncf.S: Likewise.
* sysdeps/powerpc/powerpc64/memset.S: Likewise.
* sysdeps/powerpc/powerpc64/power7/fpu/s_finite.S: Likewise.
* sysdeps/powerpc/powerpc64/power7/fpu/s_isinf.S: Likewise.
* sysdeps/powerpc/powerpc64/power7/fpu/s_isnan.S: Likewise.
* sysdeps/powerpc/powerpc64/power7/strstr.S: Likewise.
* sysdeps/powerpc/powerpc64/power8/fpu/e_expf.S: Likewise.
* sysdeps/powerpc/powerpc64/power8/fpu/s_cosf.S: Likewise.
* sysdeps/powerpc/powerpc64/power8/fpu/s_sinf.S: Likewise.
* sysdeps/powerpc/powerpc64/power8/strcasestr.S: Likewise.
* sysdeps/powerpc/powerpc64/addmul_1.S: Use ENTRY_TOCLESS.
* sysdeps/powerpc/powerpc64/cell/memcpy.S: Likewise.
* sysdeps/powerpc/powerpc64/fpu/s_copysign.S: Likewise.
* sysdeps/powerpc/powerpc64/fpu/s_copysignl.S: Likewise.
* sysdeps/powerpc/powerpc64/fpu/s_fabsl.S: Likewise.
* sysdeps/powerpc/powerpc64/fpu/s_isnan.S: Likewise.
* sysdeps/powerpc/powerpc64/fpu/s_llrint.S: Likewise.
* sysdeps/powerpc/powerpc64/fpu/s_llrintf.S: Likewise.
* sysdeps/powerpc/powerpc64/lshift.S: Likewise.
* sysdeps/powerpc/powerpc64/memcpy.S: Likewise.
* sysdeps/powerpc/powerpc64/mul_1.S: Likewise.
* sysdeps/powerpc/powerpc64/power4/memcmp.S: Likewise.
* sysdeps/powerpc/powerpc64/power4/memcpy.S: Likewise.
* sysdeps/powerpc/powerpc64/power4/memset.S: Likewise.
* sysdeps/powerpc/powerpc64/power4/strncmp.S: Likewise.
* sysdeps/powerpc/powerpc64/power5+/fpu/s_ceil.S: Likewise.
* sysdeps/powerpc/powerpc64/power5+/fpu/s_ceilf.S: Likewise.
* sysdeps/powerpc/powerpc64/power5+/fpu/s_floor.S: Likewise.
* sysdeps/powerpc/powerpc64/power5+/fpu/s_floorf.S: Likewise.
* sysdeps/powerpc/powerpc64/power5+/fpu/s_llround.S: Likewise.
* sysdeps/powerpc/powerpc64/power5+/fpu/s_round.S: Likewise.
* sysdeps/powerpc/powerpc64/power5+/fpu/s_roundf.S: Likewise.
* sysdeps/powerpc/powerpc64/power5+/fpu/s_trunc.S: Likewise.
* sysdeps/powerpc/powerpc64/power5+/fpu/s_truncf.S: Likewise.
* sysdeps/powerpc/powerpc64/power5/fpu/s_isnan.S: Likewise.
* sysdeps/powerpc/powerpc64/power6/fpu/s_copysign.S: Likewise.
* sysdeps/powerpc/powerpc64/power6/fpu/s_isnan.S: Likewise.
* sysdeps/powerpc/powerpc64/power6/memcpy.S: Likewise.
* sysdeps/powerpc/powerpc64/power6/memset.S: Likewise.
* sysdeps/powerpc/powerpc64/power6x/fpu/s_isnan.S: Likewise.
* sysdeps/powerpc/powerpc64/power6x/fpu/s_llrint.S: Likewise.
* sysdeps/powerpc/powerpc64/power6x/fpu/s_llround.S: Likewise.
* sysdeps/powerpc/powerpc64/power7/add_n.S: Likewise.
* sysdeps/powerpc/powerpc64/power7/memchr.S: Likewise.
* sysdeps/powerpc/powerpc64/power7/memcmp.S: Likewise.
* sysdeps/powerpc/powerpc64/power7/memcpy.S: Likewise.
* sysdeps/powerpc/powerpc64/power7/memmove.S: Likewise.
* sysdeps/powerpc/powerpc64/power7/mempcpy.S: Likewise.
* sysdeps/powerpc/powerpc64/power7/memrchr.S: Likewise.
* sysdeps/powerpc/powerpc64/power7/memset.S: Likewise.
* sysdeps/powerpc/powerpc64/power7/rawmemchr.S: Likewise.
* sysdeps/powerpc/powerpc64/power7/strcasecmp.S (strcasecmp_l):
Likewise.
* sysdeps/powerpc/powerpc64/power7/strchr.S: Likewise.
* sysdeps/powerpc/powerpc64/power7/strchrnul.S: Likewise.
* sysdeps/powerpc/powerpc64/power7/strcmp.S: Likewise.
* sysdeps/powerpc/powerpc64/power7/strlen.S: Likewise.
* sysdeps/powerpc/powerpc64/power7/strncmp.S: Likewise.
* sysdeps/powerpc/powerpc64/power7/strncpy.S: Likewise.
* sysdeps/powerpc/powerpc64/power7/strnlen.S: Likewise.
* sysdeps/powerpc/powerpc64/power7/strrchr.S: Likewise.
* sysdeps/powerpc/powerpc64/power8/fpu/s_finite.S: Likewise.
* sysdeps/powerpc/powerpc64/power8/fpu/s_isinf.S: Likewise.
* sysdeps/powerpc/powerpc64/power8/fpu/s_isnan.S: Likewise.
* sysdeps/powerpc/powerpc64/power8/fpu/s_llrint.S: Likewise.
* sysdeps/powerpc/powerpc64/power8/fpu/s_llround.S: Likewise.
* sysdeps/powerpc/powerpc64/power8/memcmp.S: Likewise.
* sysdeps/powerpc/powerpc64/power8/memset.S: Likewise.
* sysdeps/powerpc/powerpc64/power8/strchr.S: Likewise.
* sysdeps/powerpc/powerpc64/power8/strcmp.S: Likewise.
* sysdeps/powerpc/powerpc64/power8/strcpy.S: Likewise.
* sysdeps/powerpc/powerpc64/power8/strlen.S: Likewise.
* sysdeps/powerpc/powerpc64/power8/strncmp.S: Likewise.
* sysdeps/powerpc/powerpc64/power8/strncpy.S: Likewise.
* sysdeps/powerpc/powerpc64/power8/strnlen.S: Likewise.
* sysdeps/powerpc/powerpc64/power8/strrchr.S: Likewise.
* sysdeps/powerpc/powerpc64/power8/strspn.S: Likewise.
* sysdeps/powerpc/powerpc64/power9/strcmp.S: Likewise.
* sysdeps/powerpc/powerpc64/power9/strncmp.S: Likewise.
* sysdeps/powerpc/powerpc64/strchr.S: Likewise.
* sysdeps/powerpc/powerpc64/strcmp.S: Likewise.
* sysdeps/powerpc/powerpc64/strlen.S: Likewise.
* sysdeps/powerpc/powerpc64/strncmp.S: Likewise.
* sysdeps/powerpc/powerpc64/ppc-mcount.S: Store LR earlier. Don't
add nop when SHARED.
* sysdeps/powerpc/powerpc64/start.S: Fix comment.
* sysdeps/powerpc/powerpc64/multiarch/strrchr-power8.S (ENTRY): Don't
define.
(ENTRY_TOCLESS): Define.
* sysdeps/powerpc/powerpc32/sysdep.h (ENTRY_TOCLESS): Define.
* sysdeps/powerpc/fpu/s_fma.S: Use ENTRY_TOCLESS.
* sysdeps/powerpc/fpu/s_fmaf.S: Likewise.
2017-06-14 01:15:50 +00:00
|
|
|
#define ENTRY_TOCLESS(name) ENTRY(name)
|
|
|
|
|
2011-12-17 20:07:31 +00:00
|
|
|
/* helper macro for accessing the 32-bit powerpc GOT. */
|
|
|
|
|
2012-08-02 19:04:29 +00:00
|
|
|
#define SETUP_GOT_ACCESS(regname,GOT_LABEL) \
|
2011-12-17 20:07:31 +00:00
|
|
|
bcl 20,31,GOT_LABEL ; \
|
|
|
|
GOT_LABEL: ; \
|
|
|
|
mflr (regname)
|
|
|
|
|
2002-09-20 23:46:03 +00:00
|
|
|
#define EALIGN_W_0 /* No words to insert. */
|
|
|
|
#define EALIGN_W_1 nop
|
|
|
|
#define EALIGN_W_2 nop;nop
|
|
|
|
#define EALIGN_W_3 nop;nop;nop
|
|
|
|
#define EALIGN_W_4 EALIGN_W_3;nop
|
|
|
|
#define EALIGN_W_5 EALIGN_W_4;nop
|
|
|
|
#define EALIGN_W_6 EALIGN_W_5;nop
|
|
|
|
#define EALIGN_W_7 EALIGN_W_6;nop
|
|
|
|
|
|
|
|
/* EALIGN is like ENTRY, but does alignment to 'words'*4 bytes
|
|
|
|
past a 2^align boundary. */
|
|
|
|
#ifdef PROF
|
2003-02-20 20:30:02 +00:00
|
|
|
# define EALIGN(name, alignt, words) \
|
2012-08-02 19:04:29 +00:00
|
|
|
.globl C_SYMBOL_NAME(name); \
|
|
|
|
.type C_SYMBOL_NAME(name),@function; \
|
2002-09-20 23:46:03 +00:00
|
|
|
.align ALIGNARG(2); \
|
|
|
|
C_LABEL(name) \
|
2006-01-04 20:05:05 +00:00
|
|
|
cfi_startproc; \
|
2002-09-20 23:46:03 +00:00
|
|
|
CALL_MCOUNT \
|
|
|
|
b 0f; \
|
|
|
|
.align ALIGNARG(alignt); \
|
|
|
|
EALIGN_W_##words; \
|
|
|
|
0:
|
|
|
|
#else /* PROF */
|
2003-02-20 20:30:02 +00:00
|
|
|
# define EALIGN(name, alignt, words) \
|
2012-08-02 19:04:29 +00:00
|
|
|
.globl C_SYMBOL_NAME(name); \
|
|
|
|
.type C_SYMBOL_NAME(name),@function; \
|
2002-09-20 23:46:03 +00:00
|
|
|
.align ALIGNARG(alignt); \
|
|
|
|
EALIGN_W_##words; \
|
2006-01-04 20:05:05 +00:00
|
|
|
C_LABEL(name) \
|
|
|
|
cfi_startproc;
|
2002-09-20 23:46:03 +00:00
|
|
|
#endif
|
|
|
|
|
|
|
|
#undef END
|
|
|
|
#define END(name) \
|
2006-01-04 20:05:05 +00:00
|
|
|
cfi_endproc; \
|
2002-09-20 23:46:03 +00:00
|
|
|
ASM_SIZE_DIRECTIVE(name)
|
|
|
|
|
2012-08-02 19:04:29 +00:00
|
|
|
#define DO_CALL(syscall) \
|
2012-01-12 22:05:03 +00:00
|
|
|
li 0,syscall; \
|
powerpc: Runtime selection between sc and scv for syscalls
Linux kernel v5.9 added support for system calls using the scv
instruction for POWER9 and later. The new codepath provides better
performance (see below) if compared to using sc. For the
foreseeable future, both sc and scv mechanisms will co-exist, so this
patch enables glibc to do a runtime check and use scv when it is
available.
Before issuing the system call to the kernel, we check hwcap2 in the TCB
for PPC_FEATURE2_SCV to see if scv is supported by the kernel. If not,
we fallback to sc and keep the old behavior.
The kernel implements a different error return convention for scv, so
when returning from a system call we need to handle the return value
differently depending on the instruction we used to enter the kernel.
For syscalls implemented in ASM, entry and exit are implemented by
different macros (PSEUDO and PSEUDO_RET, resp.), which may be used in
sequence (e.g. for templated syscalls) or with other instructions in
between (e.g. clone). To avoid accessing the TCB a second time on
PSEUDO_RET to check which instruction we used, the value read from
hwcap2 is cached on a non-volatile register.
This is not needed when using INTERNAL_SYSCALL macro, since entry and
exit are bundled into the same inline asm directive.
The dynamic loader may issue syscalls before the TCB has been setup
so it always uses sc with no extra checks. For the static case, there
is no compile-time way to determine if we are inside startup code,
so we also check the value of the thread pointer before effectively
accessing the TCB. For such situations in which the availability of
scv cannot be determined, sc is always used.
Support for scv in syscalls implemented in their own ASM file (clone and
vfork) will be added later. For now simply use sc as before.
Average performance over 1M calls for each syscall "type":
- stat: C wrapper calling INTERNAL_SYSCALL
- getpid: templated ASM syscall
- syscall: call to gettid using syscall function
Standard:
stat : 1.573445 us / ~3619 cycles
getpid : 0.164986 us / ~379 cycles
syscall : 0.162743 us / ~374 cycles
With scv:
stat : 1.537049 us / ~3535 cycles <~ -84 cycles / -2.32%
getpid : 0.109923 us / ~253 cycles <~ -126 cycles / -33.25%
syscall : 0.116410 us / ~268 cycles <~ -106 cycles / -28.34%
Tested on powerpc, powerpc64, powerpc64le (with and without scv)
Tested-by: Lucas A. M. Magalhães <lamm@linux.ibm.com>
Reviewed-by: Tulio Magno Quites Machado Filho <tuliom@linux.ibm.com>
2020-12-03 17:15:27 +00:00
|
|
|
DO_CALL_SC
|
|
|
|
|
|
|
|
#define DO_CALL_SC \
|
|
|
|
sc
|
2002-09-20 23:46:03 +00:00
|
|
|
|
2003-01-09 20:30:21 +00:00
|
|
|
#undef JUMPTARGET
|
2002-09-20 23:46:03 +00:00
|
|
|
#ifdef PIC
|
2003-02-20 20:30:02 +00:00
|
|
|
# define JUMPTARGET(name) name##@plt
|
2002-09-20 23:46:03 +00:00
|
|
|
#else
|
2003-02-20 20:30:02 +00:00
|
|
|
# define JUMPTARGET(name) name
|
|
|
|
#endif
|
|
|
|
|
2013-09-04 15:25:42 +00:00
|
|
|
#if defined SHARED && defined PIC && !defined NO_HIDDEN
|
2003-02-20 20:30:02 +00:00
|
|
|
# undef HIDDEN_JUMPTARGET
|
|
|
|
# define HIDDEN_JUMPTARGET(name) __GI_##name##@local
|
2002-09-20 23:46:03 +00:00
|
|
|
#endif
|
|
|
|
|
powerpc: Runtime selection between sc and scv for syscalls
Linux kernel v5.9 added support for system calls using the scv
instruction for POWER9 and later. The new codepath provides better
performance (see below) if compared to using sc. For the
foreseeable future, both sc and scv mechanisms will co-exist, so this
patch enables glibc to do a runtime check and use scv when it is
available.
Before issuing the system call to the kernel, we check hwcap2 in the TCB
for PPC_FEATURE2_SCV to see if scv is supported by the kernel. If not,
we fallback to sc and keep the old behavior.
The kernel implements a different error return convention for scv, so
when returning from a system call we need to handle the return value
differently depending on the instruction we used to enter the kernel.
For syscalls implemented in ASM, entry and exit are implemented by
different macros (PSEUDO and PSEUDO_RET, resp.), which may be used in
sequence (e.g. for templated syscalls) or with other instructions in
between (e.g. clone). To avoid accessing the TCB a second time on
PSEUDO_RET to check which instruction we used, the value read from
hwcap2 is cached on a non-volatile register.
This is not needed when using INTERNAL_SYSCALL macro, since entry and
exit are bundled into the same inline asm directive.
The dynamic loader may issue syscalls before the TCB has been setup
so it always uses sc with no extra checks. For the static case, there
is no compile-time way to determine if we are inside startup code,
so we also check the value of the thread pointer before effectively
accessing the TCB. For such situations in which the availability of
scv cannot be determined, sc is always used.
Support for scv in syscalls implemented in their own ASM file (clone and
vfork) will be added later. For now simply use sc as before.
Average performance over 1M calls for each syscall "type":
- stat: C wrapper calling INTERNAL_SYSCALL
- getpid: templated ASM syscall
- syscall: call to gettid using syscall function
Standard:
stat : 1.573445 us / ~3619 cycles
getpid : 0.164986 us / ~379 cycles
syscall : 0.162743 us / ~374 cycles
With scv:
stat : 1.537049 us / ~3535 cycles <~ -84 cycles / -2.32%
getpid : 0.109923 us / ~253 cycles <~ -126 cycles / -33.25%
syscall : 0.116410 us / ~268 cycles <~ -106 cycles / -28.34%
Tested on powerpc, powerpc64, powerpc64le (with and without scv)
Tested-by: Lucas A. M. Magalhães <lamm@linux.ibm.com>
Reviewed-by: Tulio Magno Quites Machado Filho <tuliom@linux.ibm.com>
2020-12-03 17:15:27 +00:00
|
|
|
#define TAIL_CALL_SYSCALL_ERROR \
|
|
|
|
b __syscall_error@local
|
|
|
|
|
2002-09-20 23:46:03 +00:00
|
|
|
#define PSEUDO(name, syscall_name, args) \
|
|
|
|
.section ".text"; \
|
|
|
|
ENTRY (name) \
|
|
|
|
DO_CALL (SYS_ify (syscall_name));
|
|
|
|
|
powerpc: Runtime selection between sc and scv for syscalls
Linux kernel v5.9 added support for system calls using the scv
instruction for POWER9 and later. The new codepath provides better
performance (see below) if compared to using sc. For the
foreseeable future, both sc and scv mechanisms will co-exist, so this
patch enables glibc to do a runtime check and use scv when it is
available.
Before issuing the system call to the kernel, we check hwcap2 in the TCB
for PPC_FEATURE2_SCV to see if scv is supported by the kernel. If not,
we fallback to sc and keep the old behavior.
The kernel implements a different error return convention for scv, so
when returning from a system call we need to handle the return value
differently depending on the instruction we used to enter the kernel.
For syscalls implemented in ASM, entry and exit are implemented by
different macros (PSEUDO and PSEUDO_RET, resp.), which may be used in
sequence (e.g. for templated syscalls) or with other instructions in
between (e.g. clone). To avoid accessing the TCB a second time on
PSEUDO_RET to check which instruction we used, the value read from
hwcap2 is cached on a non-volatile register.
This is not needed when using INTERNAL_SYSCALL macro, since entry and
exit are bundled into the same inline asm directive.
The dynamic loader may issue syscalls before the TCB has been setup
so it always uses sc with no extra checks. For the static case, there
is no compile-time way to determine if we are inside startup code,
so we also check the value of the thread pointer before effectively
accessing the TCB. For such situations in which the availability of
scv cannot be determined, sc is always used.
Support for scv in syscalls implemented in their own ASM file (clone and
vfork) will be added later. For now simply use sc as before.
Average performance over 1M calls for each syscall "type":
- stat: C wrapper calling INTERNAL_SYSCALL
- getpid: templated ASM syscall
- syscall: call to gettid using syscall function
Standard:
stat : 1.573445 us / ~3619 cycles
getpid : 0.164986 us / ~379 cycles
syscall : 0.162743 us / ~374 cycles
With scv:
stat : 1.537049 us / ~3535 cycles <~ -84 cycles / -2.32%
getpid : 0.109923 us / ~253 cycles <~ -126 cycles / -33.25%
syscall : 0.116410 us / ~268 cycles <~ -106 cycles / -28.34%
Tested on powerpc, powerpc64, powerpc64le (with and without scv)
Tested-by: Lucas A. M. Magalhães <lamm@linux.ibm.com>
Reviewed-by: Tulio Magno Quites Machado Filho <tuliom@linux.ibm.com>
2020-12-03 17:15:27 +00:00
|
|
|
#define RET_SC \
|
|
|
|
bnslr+;
|
|
|
|
|
2002-09-20 23:46:03 +00:00
|
|
|
#define PSEUDO_RET \
|
powerpc: Runtime selection between sc and scv for syscalls
Linux kernel v5.9 added support for system calls using the scv
instruction for POWER9 and later. The new codepath provides better
performance (see below) if compared to using sc. For the
foreseeable future, both sc and scv mechanisms will co-exist, so this
patch enables glibc to do a runtime check and use scv when it is
available.
Before issuing the system call to the kernel, we check hwcap2 in the TCB
for PPC_FEATURE2_SCV to see if scv is supported by the kernel. If not,
we fallback to sc and keep the old behavior.
The kernel implements a different error return convention for scv, so
when returning from a system call we need to handle the return value
differently depending on the instruction we used to enter the kernel.
For syscalls implemented in ASM, entry and exit are implemented by
different macros (PSEUDO and PSEUDO_RET, resp.), which may be used in
sequence (e.g. for templated syscalls) or with other instructions in
between (e.g. clone). To avoid accessing the TCB a second time on
PSEUDO_RET to check which instruction we used, the value read from
hwcap2 is cached on a non-volatile register.
This is not needed when using INTERNAL_SYSCALL macro, since entry and
exit are bundled into the same inline asm directive.
The dynamic loader may issue syscalls before the TCB has been setup
so it always uses sc with no extra checks. For the static case, there
is no compile-time way to determine if we are inside startup code,
so we also check the value of the thread pointer before effectively
accessing the TCB. For such situations in which the availability of
scv cannot be determined, sc is always used.
Support for scv in syscalls implemented in their own ASM file (clone and
vfork) will be added later. For now simply use sc as before.
Average performance over 1M calls for each syscall "type":
- stat: C wrapper calling INTERNAL_SYSCALL
- getpid: templated ASM syscall
- syscall: call to gettid using syscall function
Standard:
stat : 1.573445 us / ~3619 cycles
getpid : 0.164986 us / ~379 cycles
syscall : 0.162743 us / ~374 cycles
With scv:
stat : 1.537049 us / ~3535 cycles <~ -84 cycles / -2.32%
getpid : 0.109923 us / ~253 cycles <~ -126 cycles / -33.25%
syscall : 0.116410 us / ~268 cycles <~ -106 cycles / -28.34%
Tested on powerpc, powerpc64, powerpc64le (with and without scv)
Tested-by: Lucas A. M. Magalhães <lamm@linux.ibm.com>
Reviewed-by: Tulio Magno Quites Machado Filho <tuliom@linux.ibm.com>
2020-12-03 17:15:27 +00:00
|
|
|
RET_SC; \
|
|
|
|
TAIL_CALL_SYSCALL_ERROR
|
2002-09-20 23:46:03 +00:00
|
|
|
#define ret PSEUDO_RET
|
|
|
|
|
|
|
|
#undef PSEUDO_END
|
|
|
|
#define PSEUDO_END(name) \
|
|
|
|
END (name)
|
|
|
|
|
2003-03-23 19:44:31 +00:00
|
|
|
#define PSEUDO_NOERRNO(name, syscall_name, args) \
|
|
|
|
.section ".text"; \
|
|
|
|
ENTRY (name) \
|
|
|
|
DO_CALL (SYS_ify (syscall_name));
|
|
|
|
|
|
|
|
#define PSEUDO_RET_NOERRNO \
|
|
|
|
blr
|
|
|
|
#define ret_NOERRNO PSEUDO_RET_NOERRNO
|
|
|
|
|
|
|
|
#undef PSEUDO_END_NOERRNO
|
|
|
|
#define PSEUDO_END_NOERRNO(name) \
|
|
|
|
END (name)
|
|
|
|
|
Update.
2003-08-16 Jakub Jelinek <jakub@redhat.com>
* sysdeps/powerpc/powerpc32/sysdep.h (PSEUDO_ERRVAL,
PSEUDO_RET_ERRVAL, ret_ERRVAL, PSEUDO_END_ERRVAL): Define.
* sysdeps/powerpc/powerpc64/sysdep.h (PSEUDO_ERRVAL,
PSEUDO_RET_ERRVAL, ret_ERRVAL, PSEUDO_END_ERRVAL): Define.
* sysdeps/unix/alpha/sysdep.h (PSEUDO_ERRVAL,
ret_ERRVAL, PSEUDO_END_ERRVAL): Define.
* sysdeps/unix/mips/sysdep.h (PSEUDO_ERRVAL,
ret_ERRVAL, PSEUDO_END_ERRVAL): Define.
* sysdeps/unix/sparc/sysdep.h (PSEUDO_ERRVAL,
ret_ERRVAL): Define.
* sysdeps/unix/sysv/linux/s390/s390-32/sysdep.h (PSEUDO_ERRVAL,
PSEUDO_END_ERRVAL, ret_ERRVAL): Define.
* sysdeps/unix/sysv/linux/s390/s390-64/sysdep.h (PSEUDO_ERRVAL,
PSEUDO_END_ERRVAL, ret_ERRVAL): Define.
* sysdeps/unix/sysv/linux/sparc/sparc32/sysdep.h (PSEUDO_ERRVAL):
Define.
* sysdeps/unix/sysv/linux/sparc/sparc64/sysdep.h (PSEUDO_ERRVAL,
PSEUDO_END_ERRVAL, ret_ERRVAL): Define.
* sysdeps/unix/sysdep.h (PSEUDO_END_ERRVAL): Define.
* sysdeps/unix/sysv/linux/syscalls.list (posix_fadvise64,
posix_fadvise64_64): Remove.
* sysdeps/unix/sysv/linux/alpha/syscalls.list (posix_fadvise64): Add
V flag.
* sysdeps/unix/sysv/linux/ia64/syscalls.list (posix_fadvise64):
Likewise.
* sysdeps/unix/sysv/linux/powerpc/powerpc64/syscalls.list
(posix_fadvise64): Likewise.
* sysdeps/unix/sysv/linux/s390/s390-64/syscalls.list
(posix_fadvise64): Likewise.
* sysdeps/unix/sysv/linux/sparc/sparc64/syscalls.list
(posix_fadvise64): Likewise.
* sysdeps/unix/sysv/linux/x86_64/syscalls.list (posix_fadvise64):
Likewise.
* sysdeps/unix/sysv/linux/i386/posix_fadvise64_64.S: Moved to...
* sysdeps/unix/sysv/linux/i386/posix_fadvise64.S: ...here.
(__posix_fadvise64_l64): Fix a typo in fadvise64 syscall invocation.
(__posix_fadvise64_l32): New function.
* sysdeps/unix/sysv/linux/i386/Makefile: Revert last change.
* sysdeps/unix/sysv/linux/powerpc/powerpc32/Versions (libc): Export
posix_fadvise64@@GLIBC_2.3.3.
* sysdeps/unix/sysv/linux/s390/s390-32/Versions (libc): Likewise.
* sysdeps/unix/sysv/linux/sparc/sparc32/Versions (libc): Likewise.
* sysdeps/unix/sysv/linux/posix_fadvise.c (posix_fadvise): Return
error value.
* sysdeps/unix/sysv/linux/posix_fadvise64.c: New file.
* sysdeps/unix/sysv/linux/sparc/sparc32/sysdep.h
(SYSCALL_ERROR_HANDLER): Use TLS errno/__libc_errno if USE___THREAD.
2003-08-15 Jakub Jelinek <jakub@redhat.com>
* sysdeps/sparc/sparc32/dl-machine.h (WEAKADDR): Remove.
(elf_machine_matches_host): Remove weak extern stuff.
Use GL(dl_hwcap) unconditionally and GL(dl_hwcap_mask) if SHARED.
(elf_machine_runtime_setup, sparc_fixup_plt): Remove weak extern
stuff. Use GL(dl_hwcap) unconditionally.
2003-08-17 00:37:19 +00:00
|
|
|
#define PSEUDO_ERRVAL(name, syscall_name, args) \
|
|
|
|
.section ".text"; \
|
|
|
|
ENTRY (name) \
|
|
|
|
DO_CALL (SYS_ify (syscall_name));
|
|
|
|
|
|
|
|
#define PSEUDO_RET_ERRVAL \
|
|
|
|
blr
|
|
|
|
#define ret_ERRVAL PSEUDO_RET_ERRVAL
|
|
|
|
|
|
|
|
#undef PSEUDO_END_ERRVAL
|
|
|
|
#define PSEUDO_END_ERRVAL(name) \
|
|
|
|
END (name)
|
|
|
|
|
2002-09-20 23:46:03 +00:00
|
|
|
/* Local labels stripped out by the linker. */
|
|
|
|
#undef L
|
|
|
|
#define L(x) .L##x
|
|
|
|
|
2012-01-12 22:05:03 +00:00
|
|
|
#define XGLUE(a,b) a##b
|
|
|
|
#define GLUE(a,b) XGLUE (a,b)
|
|
|
|
#define GENERATE_GOT_LABEL(name) GLUE (.got_label, name)
|
|
|
|
|
2002-09-20 23:46:03 +00:00
|
|
|
/* Label in text section. */
|
|
|
|
#define C_TEXT(name) name
|
|
|
|
|
2020-01-10 22:14:56 +00:00
|
|
|
/* Read the value of member from rtld_global_ro. */
|
|
|
|
#ifdef PIC
|
|
|
|
# ifdef SHARED
|
|
|
|
# if IS_IN (rtld)
|
|
|
|
/* Inside ld.so we use the local alias to avoid runtime GOT
|
|
|
|
relocations. */
|
|
|
|
# define __GLRO(rOUT, rGOT, member, offset) \
|
|
|
|
lwz rOUT,_rtld_local_ro@got(rGOT); \
|
|
|
|
lwz rOUT,offset(rOUT)
|
|
|
|
# else
|
|
|
|
# define __GLRO(rOUT, rGOT, member, offset) \
|
|
|
|
lwz rOUT,_rtld_global_ro@got(rGOT); \
|
|
|
|
lwz rOUT,offset(rOUT)
|
|
|
|
# endif
|
|
|
|
# else
|
|
|
|
# define __GLRO(rOUT, rGOT, member, offset) \
|
|
|
|
lwz rOUT,member@got(rGOT); \
|
|
|
|
lwz rOUT,0(rOUT)
|
|
|
|
# endif
|
|
|
|
#else
|
|
|
|
/* Position-dependent code does not require access to the GOT. */
|
|
|
|
# define __GLRO(rOUT, rGOT, member, offset) \
|
2020-08-03 16:07:19 +00:00
|
|
|
lis rOUT,(member)@ha; \
|
|
|
|
lwz rOUT,(member)@l(rOUT)
|
2020-01-10 22:14:56 +00:00
|
|
|
#endif /* PIC */
|
|
|
|
|
2002-09-20 23:46:03 +00:00
|
|
|
#endif /* __ASSEMBLER__ */
|