mirror of
https://sourceware.org/git/glibc.git
synced 2024-11-22 04:50:07 +00:00
b6a02c3606
1. Refactor files so that all implementations are in the multiarch
directory
- Moved the implementation portion of memmove sse2 from memmove.S
to multiarch/memmove-sse2.S
- The non-multiarch file now only includes one of the
implementations in the multiarch directory based on the compiled
ISA level (only used for non-multiarch builds. Otherwise we go
through the ifunc selector).
2. Add ISA level build guards to different implementations.
- I.e memmove-avx2-unaligned-erms.S which is ISA level 3 will only
build if compiled ISA level <= 3. Otherwise there is no reason
to include it as we will always use one of the ISA level 4
implementations (memmove-evex-unaligned-erms.S).
3. Add new multiarch/rtld-memmove.S that just include the
non-multiarch memmove.S which will in turn select the best
implementation based on the compiled ISA level.
4. Refactor the ifunc selector and ifunc implementation list to use
the ISA level aware wrapper macros that allow functions below the
compiled ISA level (with a guranteed replacement) to be skipped.
Tested with and without multiarch on x86_64 for ISA levels:
{generic, x86-64-v2, x86-64-v3, x86-64-v4}
And m32 with and without multiarch.
isa raising memmove
415 lines
9.0 KiB
ArmAsm
415 lines
9.0 KiB
ArmAsm
/* memmove/memcpy/mempcpy optimized for aligned access with SSSE3.
|
|
All versions must be listed in ifunc-impl-list.c.
|
|
Copyright (C) 2022 Free Software Foundation, Inc.
|
|
This file is part of the GNU C Library.
|
|
|
|
The GNU C Library is free software; you can redistribute it and/or
|
|
modify it under the terms of the GNU Lesser General Public
|
|
License as published by the Free Software Foundation; either
|
|
version 2.1 of the License, or (at your option) any later version.
|
|
|
|
The GNU C Library is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
Lesser General Public License for more details.
|
|
|
|
You should have received a copy of the GNU Lesser General Public
|
|
License along with the GNU C Library; if not, see
|
|
<https://www.gnu.org/licenses/>. */
|
|
|
|
|
|
#include <isa-level.h>
|
|
|
|
#if ISA_SHOULD_BUILD (2)
|
|
|
|
# include <sysdep.h>
|
|
# ifndef MEMMOVE
|
|
# define MEMMOVE __memmove_ssse3
|
|
# define MEMMOVE_CHK __memmove_chk_ssse3
|
|
# define MEMCPY __memcpy_ssse3
|
|
# define MEMCPY_CHK __memcpy_chk_ssse3
|
|
# define MEMPCPY __mempcpy_ssse3
|
|
# define MEMPCPY_CHK __mempcpy_chk_ssse3
|
|
# endif
|
|
|
|
.section .text.ssse3, "ax", @progbits
|
|
# if defined SHARED
|
|
ENTRY(MEMPCPY_CHK)
|
|
cmp %RDX_LP, %RCX_LP
|
|
jb HIDDEN_JUMPTARGET(__chk_fail)
|
|
END(MEMPCPY_CHK)
|
|
# endif
|
|
|
|
ENTRY(MEMPCPY)
|
|
mov %RDI_LP, %RAX_LP
|
|
add %RDX_LP, %RAX_LP
|
|
jmp L(start)
|
|
END(MEMPCPY)
|
|
|
|
# if defined SHARED
|
|
ENTRY(MEMMOVE_CHK)
|
|
cmp %RDX_LP, %RCX_LP
|
|
jb HIDDEN_JUMPTARGET(__chk_fail)
|
|
END(MEMMOVE_CHK)
|
|
# endif
|
|
|
|
ENTRY_P2ALIGN(MEMMOVE, 6)
|
|
# ifdef __ILP32__
|
|
/* Clear the upper 32 bits. */
|
|
movl %edx, %edx
|
|
# endif
|
|
movq %rdi, %rax
|
|
L(start):
|
|
cmpq $16, %rdx
|
|
jb L(copy_0_15)
|
|
|
|
/* These loads are always useful. */
|
|
movups 0(%rsi), %xmm0
|
|
movups -16(%rsi, %rdx), %xmm7
|
|
cmpq $32, %rdx
|
|
ja L(more_2x_vec)
|
|
|
|
movups %xmm0, 0(%rdi)
|
|
movups %xmm7, -16(%rdi, %rdx)
|
|
ret
|
|
|
|
.p2align 4,, 4
|
|
L(copy_0_15):
|
|
cmpl $4, %edx
|
|
jb L(copy_0_3)
|
|
cmpl $8, %edx
|
|
jb L(copy_4_7)
|
|
movq 0(%rsi), %rcx
|
|
movq -8(%rsi, %rdx), %rsi
|
|
movq %rcx, 0(%rdi)
|
|
movq %rsi, -8(%rdi, %rdx)
|
|
ret
|
|
|
|
.p2align 4,, 4
|
|
L(copy_4_7):
|
|
movl 0(%rsi), %ecx
|
|
movl -4(%rsi, %rdx), %esi
|
|
movl %ecx, 0(%rdi)
|
|
movl %esi, -4(%rdi, %rdx)
|
|
ret
|
|
|
|
.p2align 4,, 4
|
|
L(copy_0_3):
|
|
decl %edx
|
|
jl L(copy_0_0)
|
|
movb (%rsi), %cl
|
|
je L(copy_1_1)
|
|
|
|
movzwl -1(%rsi, %rdx), %esi
|
|
movw %si, -1(%rdi, %rdx)
|
|
L(copy_1_1):
|
|
movb %cl, (%rdi)
|
|
L(copy_0_0):
|
|
ret
|
|
|
|
.p2align 4,, 4
|
|
L(copy_4x_vec):
|
|
movups 16(%rsi), %xmm1
|
|
movups -32(%rsi, %rdx), %xmm2
|
|
|
|
movups %xmm0, 0(%rdi)
|
|
movups %xmm1, 16(%rdi)
|
|
movups %xmm2, -32(%rdi, %rdx)
|
|
movups %xmm7, -16(%rdi, %rdx)
|
|
L(nop):
|
|
ret
|
|
|
|
.p2align 4
|
|
L(more_2x_vec):
|
|
cmpq $64, %rdx
|
|
jbe L(copy_4x_vec)
|
|
|
|
/* We use rcx later to get alignr value. */
|
|
movq %rdi, %rcx
|
|
|
|
/* Backward copy for overlap + dst > src for memmove safety. */
|
|
subq %rsi, %rcx
|
|
cmpq %rdx, %rcx
|
|
jb L(copy_backward)
|
|
|
|
/* Load tail. */
|
|
|
|
/* -16(%rsi, %rdx) already loaded into xmm7. */
|
|
movups -32(%rsi, %rdx), %xmm8
|
|
movups -48(%rsi, %rdx), %xmm9
|
|
|
|
/* Get misalignment. */
|
|
andl $0xf, %ecx
|
|
|
|
movq %rsi, %r9
|
|
addq %rcx, %rsi
|
|
andq $-16, %rsi
|
|
/* Get first vec for `palignr`. */
|
|
movaps (%rsi), %xmm1
|
|
|
|
/* We have loaded (%rsi) so safe to do this store before the
|
|
loop. */
|
|
movups %xmm0, (%rdi)
|
|
|
|
# ifdef SHARED_CACHE_SIZE_HALF
|
|
cmp $SHARED_CACHE_SIZE_HALF, %RDX_LP
|
|
# else
|
|
cmp __x86_shared_cache_size_half(%rip), %rdx
|
|
# endif
|
|
ja L(large_memcpy)
|
|
|
|
leaq -64(%rdi, %rdx), %r8
|
|
andq $-16, %rdi
|
|
movl $48, %edx
|
|
|
|
leaq L(loop_fwd_start)(%rip), %r9
|
|
sall $6, %ecx
|
|
addq %r9, %rcx
|
|
jmp * %rcx
|
|
|
|
.p2align 4,, 8
|
|
L(copy_backward):
|
|
testq %rcx, %rcx
|
|
jz L(nop)
|
|
|
|
/* Preload tail. */
|
|
|
|
/* (%rsi) already loaded into xmm0. */
|
|
movups 16(%rsi), %xmm4
|
|
movups 32(%rsi), %xmm5
|
|
|
|
movq %rdi, %r8
|
|
subq %rdi, %rsi
|
|
leaq -49(%rdi, %rdx), %rdi
|
|
andq $-16, %rdi
|
|
addq %rdi, %rsi
|
|
andq $-16, %rsi
|
|
|
|
movaps 48(%rsi), %xmm6
|
|
|
|
|
|
leaq L(loop_bkwd_start)(%rip), %r9
|
|
andl $0xf, %ecx
|
|
sall $6, %ecx
|
|
addq %r9, %rcx
|
|
jmp * %rcx
|
|
|
|
.p2align 4,, 8
|
|
L(large_memcpy):
|
|
movups -64(%r9, %rdx), %xmm10
|
|
movups -80(%r9, %rdx), %xmm11
|
|
|
|
sall $5, %ecx
|
|
leal (%rcx, %rcx, 2), %r8d
|
|
leaq -96(%rdi, %rdx), %rcx
|
|
andq $-16, %rdi
|
|
leaq L(large_loop_fwd_start)(%rip), %rdx
|
|
addq %r8, %rdx
|
|
jmp * %rdx
|
|
|
|
|
|
/* Instead of a typical jump table all 16 loops are exactly
|
|
64-bytes in size. So, we can just jump to first loop + r8 *
|
|
64. Before modifying any loop ensure all their sizes match!
|
|
*/
|
|
.p2align 6
|
|
L(loop_fwd_start):
|
|
L(loop_fwd_0x0):
|
|
movaps 16(%rsi), %xmm1
|
|
movaps 32(%rsi), %xmm2
|
|
movaps 48(%rsi), %xmm3
|
|
movaps %xmm1, 16(%rdi)
|
|
movaps %xmm2, 32(%rdi)
|
|
movaps %xmm3, 48(%rdi)
|
|
addq %rdx, %rdi
|
|
addq %rdx, %rsi
|
|
cmpq %rdi, %r8
|
|
ja L(loop_fwd_0x0)
|
|
L(end_loop_fwd):
|
|
movups %xmm9, 16(%r8)
|
|
movups %xmm8, 32(%r8)
|
|
movups %xmm7, 48(%r8)
|
|
ret
|
|
|
|
/* Extactly 64 bytes if `jmp L(end_loop_fwd)` is long encoding.
|
|
60 bytes otherwise. */
|
|
# define ALIGNED_LOOP_FWD(align_by); \
|
|
.p2align 6; \
|
|
L(loop_fwd_ ## align_by): \
|
|
movaps 16(%rsi), %xmm0; \
|
|
movaps 32(%rsi), %xmm2; \
|
|
movaps 48(%rsi), %xmm3; \
|
|
movaps %xmm3, %xmm4; \
|
|
palignr $align_by, %xmm2, %xmm3; \
|
|
palignr $align_by, %xmm0, %xmm2; \
|
|
palignr $align_by, %xmm1, %xmm0; \
|
|
movaps %xmm4, %xmm1; \
|
|
movaps %xmm0, 16(%rdi); \
|
|
movaps %xmm2, 32(%rdi); \
|
|
movaps %xmm3, 48(%rdi); \
|
|
addq %rdx, %rdi; \
|
|
addq %rdx, %rsi; \
|
|
cmpq %rdi, %r8; \
|
|
ja L(loop_fwd_ ## align_by); \
|
|
jmp L(end_loop_fwd);
|
|
|
|
/* Must be in descending order. */
|
|
ALIGNED_LOOP_FWD (0xf)
|
|
ALIGNED_LOOP_FWD (0xe)
|
|
ALIGNED_LOOP_FWD (0xd)
|
|
ALIGNED_LOOP_FWD (0xc)
|
|
ALIGNED_LOOP_FWD (0xb)
|
|
ALIGNED_LOOP_FWD (0xa)
|
|
ALIGNED_LOOP_FWD (0x9)
|
|
ALIGNED_LOOP_FWD (0x8)
|
|
ALIGNED_LOOP_FWD (0x7)
|
|
ALIGNED_LOOP_FWD (0x6)
|
|
ALIGNED_LOOP_FWD (0x5)
|
|
ALIGNED_LOOP_FWD (0x4)
|
|
ALIGNED_LOOP_FWD (0x3)
|
|
ALIGNED_LOOP_FWD (0x2)
|
|
ALIGNED_LOOP_FWD (0x1)
|
|
|
|
.p2align 6
|
|
L(large_loop_fwd_start):
|
|
L(large_loop_fwd_0x0):
|
|
movaps 16(%rsi), %xmm1
|
|
movaps 32(%rsi), %xmm2
|
|
movaps 48(%rsi), %xmm3
|
|
movaps 64(%rsi), %xmm4
|
|
movaps 80(%rsi), %xmm5
|
|
movntps %xmm1, 16(%rdi)
|
|
movntps %xmm2, 32(%rdi)
|
|
movntps %xmm3, 48(%rdi)
|
|
movntps %xmm4, 64(%rdi)
|
|
movntps %xmm5, 80(%rdi)
|
|
addq $80, %rdi
|
|
addq $80, %rsi
|
|
cmpq %rdi, %rcx
|
|
ja L(large_loop_fwd_0x0)
|
|
|
|
/* Ensure no icache line split on tail. */
|
|
.p2align 4
|
|
L(end_large_loop_fwd):
|
|
sfence
|
|
movups %xmm11, 16(%rcx)
|
|
movups %xmm10, 32(%rcx)
|
|
movups %xmm9, 48(%rcx)
|
|
movups %xmm8, 64(%rcx)
|
|
movups %xmm7, 80(%rcx)
|
|
ret
|
|
|
|
|
|
/* Size > 64 bytes and <= 96 bytes. 32-byte align between ensure
|
|
96-byte spacing between each. */
|
|
# define ALIGNED_LARGE_LOOP_FWD(align_by); \
|
|
.p2align 5; \
|
|
L(large_loop_fwd_ ## align_by): \
|
|
movaps 16(%rsi), %xmm0; \
|
|
movaps 32(%rsi), %xmm2; \
|
|
movaps 48(%rsi), %xmm3; \
|
|
movaps 64(%rsi), %xmm4; \
|
|
movaps 80(%rsi), %xmm5; \
|
|
movaps %xmm5, %xmm6; \
|
|
palignr $align_by, %xmm4, %xmm5; \
|
|
palignr $align_by, %xmm3, %xmm4; \
|
|
palignr $align_by, %xmm2, %xmm3; \
|
|
palignr $align_by, %xmm0, %xmm2; \
|
|
palignr $align_by, %xmm1, %xmm0; \
|
|
movaps %xmm6, %xmm1; \
|
|
movntps %xmm0, 16(%rdi); \
|
|
movntps %xmm2, 32(%rdi); \
|
|
movntps %xmm3, 48(%rdi); \
|
|
movntps %xmm4, 64(%rdi); \
|
|
movntps %xmm5, 80(%rdi); \
|
|
addq $80, %rdi; \
|
|
addq $80, %rsi; \
|
|
cmpq %rdi, %rcx; \
|
|
ja L(large_loop_fwd_ ## align_by); \
|
|
jmp L(end_large_loop_fwd);
|
|
|
|
/* Must be in descending order. */
|
|
ALIGNED_LARGE_LOOP_FWD (0xf)
|
|
ALIGNED_LARGE_LOOP_FWD (0xe)
|
|
ALIGNED_LARGE_LOOP_FWD (0xd)
|
|
ALIGNED_LARGE_LOOP_FWD (0xc)
|
|
ALIGNED_LARGE_LOOP_FWD (0xb)
|
|
ALIGNED_LARGE_LOOP_FWD (0xa)
|
|
ALIGNED_LARGE_LOOP_FWD (0x9)
|
|
ALIGNED_LARGE_LOOP_FWD (0x8)
|
|
ALIGNED_LARGE_LOOP_FWD (0x7)
|
|
ALIGNED_LARGE_LOOP_FWD (0x6)
|
|
ALIGNED_LARGE_LOOP_FWD (0x5)
|
|
ALIGNED_LARGE_LOOP_FWD (0x4)
|
|
ALIGNED_LARGE_LOOP_FWD (0x3)
|
|
ALIGNED_LARGE_LOOP_FWD (0x2)
|
|
ALIGNED_LARGE_LOOP_FWD (0x1)
|
|
|
|
|
|
.p2align 6
|
|
L(loop_bkwd_start):
|
|
L(loop_bkwd_0x0):
|
|
movaps 32(%rsi), %xmm1
|
|
movaps 16(%rsi), %xmm2
|
|
movaps 0(%rsi), %xmm3
|
|
movaps %xmm1, 32(%rdi)
|
|
movaps %xmm2, 16(%rdi)
|
|
movaps %xmm3, 0(%rdi)
|
|
subq $48, %rdi
|
|
subq $48, %rsi
|
|
cmpq %rdi, %r8
|
|
jb L(loop_bkwd_0x0)
|
|
L(end_loop_bkwd):
|
|
movups %xmm7, -16(%r8, %rdx)
|
|
movups %xmm0, 0(%r8)
|
|
movups %xmm4, 16(%r8)
|
|
movups %xmm5, 32(%r8)
|
|
|
|
ret
|
|
|
|
|
|
/* Extactly 64 bytes if `jmp L(end_loop_bkwd)` is long encoding.
|
|
60 bytes otherwise. */
|
|
# define ALIGNED_LOOP_BKWD(align_by); \
|
|
.p2align 6; \
|
|
L(loop_bkwd_ ## align_by): \
|
|
movaps 32(%rsi), %xmm1; \
|
|
movaps 16(%rsi), %xmm2; \
|
|
movaps 0(%rsi), %xmm3; \
|
|
palignr $align_by, %xmm1, %xmm6; \
|
|
palignr $align_by, %xmm2, %xmm1; \
|
|
palignr $align_by, %xmm3, %xmm2; \
|
|
movaps %xmm6, 32(%rdi); \
|
|
movaps %xmm1, 16(%rdi); \
|
|
movaps %xmm2, 0(%rdi); \
|
|
subq $48, %rdi; \
|
|
subq $48, %rsi; \
|
|
movaps %xmm3, %xmm6; \
|
|
cmpq %rdi, %r8; \
|
|
jb L(loop_bkwd_ ## align_by); \
|
|
jmp L(end_loop_bkwd);
|
|
|
|
/* Must be in descending order. */
|
|
ALIGNED_LOOP_BKWD (0xf)
|
|
ALIGNED_LOOP_BKWD (0xe)
|
|
ALIGNED_LOOP_BKWD (0xd)
|
|
ALIGNED_LOOP_BKWD (0xc)
|
|
ALIGNED_LOOP_BKWD (0xb)
|
|
ALIGNED_LOOP_BKWD (0xa)
|
|
ALIGNED_LOOP_BKWD (0x9)
|
|
ALIGNED_LOOP_BKWD (0x8)
|
|
ALIGNED_LOOP_BKWD (0x7)
|
|
ALIGNED_LOOP_BKWD (0x6)
|
|
ALIGNED_LOOP_BKWD (0x5)
|
|
ALIGNED_LOOP_BKWD (0x4)
|
|
ALIGNED_LOOP_BKWD (0x3)
|
|
ALIGNED_LOOP_BKWD (0x2)
|
|
ALIGNED_LOOP_BKWD (0x1)
|
|
END(MEMMOVE)
|
|
|
|
strong_alias (MEMMOVE, MEMCPY)
|
|
# if defined SHARED
|
|
strong_alias (MEMMOVE_CHK, MEMCPY_CHK)
|
|
# endif
|
|
#endif
|