mirror of
https://sourceware.org/git/glibc.git
synced 2024-12-11 05:40:06 +00:00
f049f52dfe
Optimizations are:
1. Use more overlapping stores to avoid branches.
2. Reduce how unrolled the aligning copies are (this is more of a
code-size save, its a negative for some sizes in terms of
perf).
3. Improve the loop a bit (similiar to what we do in strlen with
2x vpminu + kortest instead of 3x vpminu + kmov + test).
4. For st{r|p}n{cat|cpy} re-order the branches to minimize the
number that are taken.
Performance Changes:
Times are from N = 10 runs of the benchmark suite and are
reported as geometric mean of all ratios of
New Implementation / Old Implementation.
stpcpy-evex -> 0.922
strcat-evex -> 0.985
strcpy-evex -> 0.880
strncpy-evex -> 0.831
stpncpy-evex -> 0.780
strncat-evex -> 0.958
Code Size Changes:
function -> Bytes New / Bytes Old -> Ratio
strcat-evex -> 819 / 1874 -> 0.437
strcpy-evex -> 700 / 1074 -> 0.652
stpcpy-evex -> 735 / 1094 -> 0.672
strncpy-evex -> 1397 / 2611 -> 0.535
stpncpy-evex -> 1489 / 2691 -> 0.553
strncat-evex -> 1184 / 2832 -> 0.418
Notes:
1. Because of the significant difference between the
implementations they are split into three files.
strcpy-evex.S -> strcpy, stpcpy, strcat
strncpy-evex.S -> strncpy
strncat-evex.S > strncat
I couldn't find a way to merge them without making the
ifdefs incredibly difficult to follow.
2. All implementations can be made evex512 by including
"x86-evex512-vecs.h" at the top.
3. All implementations have an optional define:
`USE_EVEX_MASKED_STORE`
Setting to one uses evex-masked stores for handling short
strings. This saves code size and branches. It's disabled
for all implementations are the moment as there are some
serious drawbacks to masked stores in certain cases, but
that may be fixed on future architectures.
Full check passes on x86-64 and build succeeds for all ISA levels w/
and w/o multiarch.
543 lines
12 KiB
ArmAsm
543 lines
12 KiB
ArmAsm
/* {wcs|wcp|str|stp}cpy with 256/512-bit EVEX instructions.
|
|
Copyright (C) 2021-2022 Free Software Foundation, Inc.
|
|
This file is part of the GNU C Library.
|
|
|
|
The GNU C Library is free software; you can redistribute it and/or
|
|
modify it under the terms of the GNU Lesser General Public
|
|
License as published by the Free Software Foundation; either
|
|
version 2.1 of the License, or (at your option) any later version.
|
|
|
|
The GNU C Library is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
Lesser General Public License for more details.
|
|
|
|
You should have received a copy of the GNU Lesser General Public
|
|
License along with the GNU C Library; if not, see
|
|
<https://www.gnu.org/licenses/>. */
|
|
|
|
#include <isa-level.h>
|
|
#if ISA_SHOULD_BUILD (4)
|
|
|
|
|
|
/* Use evex-masked stores for small sizes. Turned off at the
|
|
moment. */
|
|
# define USE_EVEX_MASKED_STORE 0
|
|
/* Use movsb in page cross case to save code size. */
|
|
# define USE_MOVSB_IN_PAGE_CROSS 1
|
|
|
|
# include <sysdep.h>
|
|
|
|
# ifndef VEC_SIZE
|
|
# include "x86-evex256-vecs.h"
|
|
# endif
|
|
|
|
# ifndef STRCPY
|
|
# define STRCPY __strcpy_evex
|
|
# endif
|
|
|
|
|
|
# ifdef USE_AS_WCSCPY
|
|
# define VMOVU_MASK vmovdqu32
|
|
# define VPMIN vpminud
|
|
# define VPTESTN vptestnmd
|
|
# define VPTEST vptestmd
|
|
# define VPCMPEQ vpcmpeqd
|
|
# define CHAR_SIZE 4
|
|
|
|
# define REP_MOVS rep movsd
|
|
|
|
# define USE_WIDE_CHAR
|
|
# else
|
|
# define VMOVU_MASK vmovdqu8
|
|
# define VPMIN vpminub
|
|
# define VPTESTN vptestnmb
|
|
# define VPTEST vptestmb
|
|
# define VPCMPEQ vpcmpeqb
|
|
# define CHAR_SIZE 1
|
|
|
|
# define REP_MOVS rep movsb
|
|
# endif
|
|
|
|
# include "reg-macros.h"
|
|
|
|
|
|
# ifdef USE_AS_STPCPY
|
|
# define END_REG rax
|
|
# else
|
|
# define END_REG rdi, %rdx, CHAR_SIZE
|
|
# endif
|
|
|
|
# ifdef USE_AS_STRCAT
|
|
# define PAGE_ALIGN_REG edx
|
|
# define PAGE_ALIGN_REG_64 rdx
|
|
# else
|
|
# define PAGE_ALIGN_REG eax
|
|
# define PAGE_ALIGN_REG_64 rax
|
|
# endif
|
|
|
|
# define VZERO VMM(7)
|
|
# define VZERO_128 VMM_128(7)
|
|
|
|
|
|
# define PAGE_SIZE 4096
|
|
# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE)
|
|
|
|
|
|
.section SECTION(.text), "ax", @progbits
|
|
ENTRY(STRCPY)
|
|
# ifdef USE_AS_STRCAT
|
|
movq %rdi, %rax
|
|
# include "strcat-strlen-evex.h.S"
|
|
# endif
|
|
|
|
movl %esi, %PAGE_ALIGN_REG
|
|
andl $(PAGE_SIZE - 1), %PAGE_ALIGN_REG
|
|
cmpl $(PAGE_SIZE - VEC_SIZE), %PAGE_ALIGN_REG
|
|
ja L(page_cross)
|
|
L(page_cross_continue):
|
|
VMOVU (%rsi), %VMM(0)
|
|
# if !defined USE_AS_STPCPY && !defined USE_AS_STRCAT
|
|
movq %rdi, %rax
|
|
# endif
|
|
|
|
|
|
/* Two short string implementations. One with traditional
|
|
branching approach and one with masked instructions (which
|
|
have potential for dramatically bad perf if dst splits a
|
|
page and is not in the TLB). */
|
|
# if USE_EVEX_MASKED_STORE
|
|
VPTEST %VMM(0), %VMM(0), %k0
|
|
KMOV %k0, %VRCX
|
|
# ifdef USE_AS_WCSCPY
|
|
subl $((1 << CHAR_PER_VEC)- 1), %VRCX
|
|
# else
|
|
inc %VRCX
|
|
# endif
|
|
jz L(more_1x_vec)
|
|
KMOV %VRCX, %k1
|
|
KXOR %k0, %k1, %k1
|
|
|
|
VMOVU_MASK %VMM(0), (%rdi){%k1}
|
|
|
|
# ifdef USE_AS_STPCPY
|
|
bsf %VRCX, %VRCX
|
|
leaq (%rdi, %rcx, CHAR_SIZE), %rax
|
|
# endif
|
|
ret
|
|
|
|
# else
|
|
VPTESTN %VMM(0), %VMM(0), %k0
|
|
KMOV %k0, %VRCX
|
|
test %VRCX, %VRCX
|
|
jz L(more_1x_vec)
|
|
|
|
xorl %edx, %edx
|
|
bsf %VRCX, %VRDX
|
|
# ifdef USE_AS_STPCPY
|
|
leaq (%rdi, %rdx, CHAR_SIZE), %rax
|
|
# endif
|
|
|
|
/* Use mask bits in rcx to detect which copy we need. If the low
|
|
mask is zero then there must be a bit set in the upper half.
|
|
I.e if rcx != 0 and ecx == 0, then match must be upper 32
|
|
bits so we use L(copy_32_63). */
|
|
# if VEC_SIZE == 64
|
|
# ifdef USE_AS_WCSCPY
|
|
testb %cl, %cl
|
|
# else
|
|
testl %ecx, %ecx
|
|
# endif
|
|
jz L(copy_32_63)
|
|
# endif
|
|
|
|
# ifdef USE_AS_WCSCPY
|
|
testb $0xf, %cl
|
|
# else
|
|
testw %cx, %cx
|
|
# endif
|
|
jz L(copy_16_31)
|
|
|
|
|
|
# ifdef USE_AS_WCSCPY
|
|
testb $0x3, %cl
|
|
# else
|
|
testb %cl, %cl
|
|
# endif
|
|
jz L(copy_8_15)
|
|
|
|
|
|
# ifdef USE_AS_WCSCPY
|
|
vmovd %VMM_128(0), (%rdi)
|
|
/* No need to copy, we know its zero. */
|
|
movl $0, (%END_REG)
|
|
|
|
ret
|
|
# else
|
|
|
|
testb $0x7, %cl
|
|
jz L(copy_4_7)
|
|
|
|
|
|
test %edx, %edx
|
|
jz L(set_null_term)
|
|
|
|
/* NB: make this `vmovw` if support for AVX512-FP16 is added.
|
|
*/
|
|
vmovd %VMM_128(0), %esi
|
|
movw %si, (%rdi)
|
|
|
|
.p2align 4,, 1
|
|
L(set_null_term):
|
|
/* No need to copy, we know its zero. */
|
|
movb $0, (%END_REG)
|
|
ret
|
|
# endif
|
|
|
|
# if VEC_SIZE == 64
|
|
.p2align 4,, 6
|
|
L(copy_32_63):
|
|
VMOVU -(32 - CHAR_SIZE)(%rsi, %rdx, CHAR_SIZE), %VMM_256(1)
|
|
VMOVU %VMM_256(0), (%rdi)
|
|
VMOVU %VMM_256(1), -(32 - CHAR_SIZE)(%END_REG)
|
|
ret
|
|
# endif
|
|
|
|
|
|
.p2align 4,, 6
|
|
L(copy_16_31):
|
|
/* Use xmm1 explicitly here as it won't require a `vzeroupper`
|
|
and will save code size. */
|
|
vmovdqu -(16 - CHAR_SIZE)(%rsi, %rdx, CHAR_SIZE), %xmm1
|
|
VMOVU %VMM_128(0), (%rdi)
|
|
vmovdqu %xmm1, -(16 - CHAR_SIZE)(%END_REG)
|
|
ret
|
|
|
|
.p2align 4,, 8
|
|
L(copy_8_15):
|
|
# ifdef USE_AS_WCSCPY
|
|
movl -(8 - CHAR_SIZE)(%rsi, %rdx, CHAR_SIZE), %ecx
|
|
# else
|
|
movq -(8 - CHAR_SIZE)(%rsi, %rdx, CHAR_SIZE), %rcx
|
|
# endif
|
|
vmovq %VMM_128(0), (%rdi)
|
|
movq %rcx, -(8 - CHAR_SIZE)(%END_REG)
|
|
ret
|
|
# endif
|
|
|
|
|
|
# ifndef USE_AS_WCSCPY
|
|
.p2align 4,, 12
|
|
L(copy_4_7):
|
|
movl -(4 - CHAR_SIZE)(%rsi, %rdx, CHAR_SIZE), %ecx
|
|
vmovd %VMM_128(0), (%rdi)
|
|
movl %ecx, -(4 - CHAR_SIZE)(%END_REG)
|
|
ret
|
|
# endif
|
|
|
|
|
|
.p2align 4,, 8
|
|
L(more_1x_vec):
|
|
# if defined USE_AS_STPCPY || defined USE_AS_STRCAT
|
|
VMOVU %VMM(0), (%rdi)
|
|
# endif
|
|
subq %rsi, %rdi
|
|
andq $-(VEC_SIZE), %rsi
|
|
addq %rsi, %rdi
|
|
VMOVA (VEC_SIZE * 1)(%rsi), %VMM(1)
|
|
|
|
/* Ideally we store after moves to minimize impact of potential
|
|
false-dependencies. */
|
|
# if !defined USE_AS_STPCPY && !defined USE_AS_STRCAT
|
|
VMOVU %VMM(0), (%rax)
|
|
# endif
|
|
|
|
VPTESTN %VMM(1), %VMM(1), %k0
|
|
KMOV %k0, %VRCX
|
|
test %VRCX, %VRCX
|
|
jnz L(ret_vec_x1)
|
|
|
|
VMOVA (VEC_SIZE * 2)(%rsi), %VMM(2)
|
|
VMOVU %VMM(1), VEC_SIZE(%rdi)
|
|
|
|
VPTESTN %VMM(2), %VMM(2), %k0
|
|
KMOV %k0, %VRCX
|
|
test %VRCX, %VRCX
|
|
jnz L(ret_vec_x2)
|
|
|
|
VMOVA (VEC_SIZE * 3)(%rsi), %VMM(3)
|
|
VMOVU %VMM(2), (VEC_SIZE * 2)(%rdi)
|
|
|
|
VPTESTN %VMM(3), %VMM(3), %k0
|
|
KMOV %k0, %VRDX
|
|
test %VRDX, %VRDX
|
|
jnz L(ret_vec_x3)
|
|
|
|
VMOVA (VEC_SIZE * 4)(%rsi), %VMM(4)
|
|
VMOVU %VMM(3), (VEC_SIZE * 3)(%rdi)
|
|
VPTESTN %VMM(4), %VMM(4), %k0
|
|
KMOV %k0, %VRCX
|
|
test %VRCX, %VRCX
|
|
jnz L(ret_vec_x4)
|
|
|
|
VMOVU %VMM(4), (VEC_SIZE * 4)(%rdi)
|
|
|
|
|
|
/* Align for 4x loop. */
|
|
subq %rsi, %rdi
|
|
|
|
/* + VEC_SIZE * 5 because we never added the original VEC_SIZE
|
|
we covered before aligning. */
|
|
subq $-(VEC_SIZE * 5), %rsi
|
|
andq $-(VEC_SIZE * 4), %rsi
|
|
|
|
|
|
/* Load first half of the loop before entry. */
|
|
VMOVA (VEC_SIZE * 0 + 0)(%rsi), %VMM(0)
|
|
VMOVA (VEC_SIZE * 1 + 0)(%rsi), %VMM(1)
|
|
VMOVA (VEC_SIZE * 2 + 0)(%rsi), %VMM(2)
|
|
VMOVA (VEC_SIZE * 3 + 0)(%rsi), %VMM(3)
|
|
|
|
VPMIN %VMM(0), %VMM(1), %VMM(4)
|
|
VPMIN %VMM(2), %VMM(3), %VMM(6)
|
|
VPTESTN %VMM(4), %VMM(4), %k2
|
|
VPTESTN %VMM(6), %VMM(6), %k4
|
|
KORTEST %k2, %k4
|
|
jnz L(loop_4x_done)
|
|
|
|
.p2align 4,, 11
|
|
L(loop_4x_vec):
|
|
|
|
VMOVU %VMM(0), (VEC_SIZE * 0 + 0)(%rdi, %rsi)
|
|
VMOVU %VMM(1), (VEC_SIZE * 1 + 0)(%rdi, %rsi)
|
|
VMOVU %VMM(2), (VEC_SIZE * 2 + 0)(%rdi, %rsi)
|
|
VMOVU %VMM(3), (VEC_SIZE * 3 + 0)(%rdi, %rsi)
|
|
|
|
subq $(VEC_SIZE * -4), %rsi
|
|
|
|
VMOVA (VEC_SIZE * 0 + 0)(%rsi), %VMM(0)
|
|
VMOVA (VEC_SIZE * 1 + 0)(%rsi), %VMM(1)
|
|
VMOVA (VEC_SIZE * 2 + 0)(%rsi), %VMM(2)
|
|
VMOVA (VEC_SIZE * 3 + 0)(%rsi), %VMM(3)
|
|
|
|
|
|
VPMIN %VMM(0), %VMM(1), %VMM(4)
|
|
VPMIN %VMM(2), %VMM(3), %VMM(6)
|
|
VPTESTN %VMM(4), %VMM(4), %k2
|
|
VPTESTN %VMM(6), %VMM(6), %k4
|
|
KORTEST %k2, %k4
|
|
jz L(loop_4x_vec)
|
|
|
|
L(loop_4x_done):
|
|
VPTESTN %VMM(0), %VMM(0), %k0
|
|
KMOV %k0, %VRCX
|
|
/* Restore rdi (%rdi). */
|
|
addq %rsi, %rdi
|
|
test %VRCX, %VRCX
|
|
jnz L(ret_vec_x0_end)
|
|
VMOVU %VMM(0), (VEC_SIZE * 0 + 0)(%rdi)
|
|
|
|
KMOV %k2, %VRCX
|
|
test %VRCX, %VRCX
|
|
jnz L(ret_vec_x1)
|
|
VMOVU %VMM(1), (VEC_SIZE * 1 + 0)(%rdi)
|
|
|
|
VPTESTN %VMM(2), %VMM(2), %k0
|
|
KMOV %k0, %VRCX
|
|
test %VRCX, %VRCX
|
|
jnz L(ret_vec_x2)
|
|
VMOVU %VMM(2), (VEC_SIZE * 2 + 0)(%rdi)
|
|
/* Place L(ret_vec_x4) here to save code size. We get a
|
|
meaningfuly benefit doing this for stpcpy. */
|
|
KMOV %k4, %VRDX
|
|
L(ret_vec_x3):
|
|
bsf %VRDX, %VRDX
|
|
VMOVU ((VEC_SIZE * 3)-(VEC_SIZE - CHAR_SIZE))(%rsi, %rdx, CHAR_SIZE), %VMM(0)
|
|
VMOVU %VMM(0), ((VEC_SIZE * 3 + 0)-(VEC_SIZE - CHAR_SIZE))(%rdi, %rdx, CHAR_SIZE)
|
|
# ifdef USE_AS_STPCPY
|
|
leaq (VEC_SIZE * 3 + 0)(%rdi, %rdx, CHAR_SIZE), %rax
|
|
# endif
|
|
L(return_end):
|
|
ret
|
|
|
|
.p2align 4,, 6
|
|
L(ret_vec_x0_end):
|
|
bsf %VRCX, %VRCX
|
|
# ifdef USE_AS_STPCPY
|
|
leaq (%rdi, %rcx, CHAR_SIZE), %rax
|
|
# endif
|
|
inc %VRCX
|
|
VMOVU (-(VEC_SIZE))(%rsi, %rcx, CHAR_SIZE), %VMM(0)
|
|
VMOVU %VMM(0), (-(VEC_SIZE))(%rdi, %rcx, CHAR_SIZE)
|
|
ret
|
|
|
|
.p2align 4,, 8
|
|
L(ret_vec_x1):
|
|
bsf %VRCX, %VRCX
|
|
VMOVU (VEC_SIZE -(VEC_SIZE - CHAR_SIZE))(%rsi, %rcx, CHAR_SIZE), %VMM(0)
|
|
VMOVU %VMM(0), (VEC_SIZE -(VEC_SIZE - CHAR_SIZE))(%rdi, %rcx, CHAR_SIZE)
|
|
# ifdef USE_AS_STPCPY
|
|
leaq VEC_SIZE(%rdi, %rcx, CHAR_SIZE), %rax
|
|
# endif
|
|
ret
|
|
|
|
.p2align 4,, 4
|
|
L(ret_vec_x2):
|
|
bsf %VRCX, %VRCX
|
|
VMOVU ((VEC_SIZE * 2)-(VEC_SIZE - CHAR_SIZE))(%rsi, %rcx, CHAR_SIZE), %VMM(0)
|
|
VMOVU %VMM(0), ((VEC_SIZE * 2)-(VEC_SIZE - CHAR_SIZE))(%rdi, %rcx, CHAR_SIZE)
|
|
# ifdef USE_AS_STPCPY
|
|
leaq (VEC_SIZE * 2)(%rdi, %rcx, CHAR_SIZE), %rax
|
|
# endif
|
|
ret
|
|
|
|
/* ret_vec_x3 reuses return code after the loop. */
|
|
.p2align 4,, 6
|
|
L(ret_vec_x4):
|
|
bsf %VRCX, %VRCX
|
|
VMOVU ((VEC_SIZE * 4)-(VEC_SIZE - CHAR_SIZE))(%rsi, %rcx, CHAR_SIZE), %VMM(0)
|
|
VMOVU %VMM(0), ((VEC_SIZE * 4)-(VEC_SIZE - CHAR_SIZE))(%rdi, %rcx, CHAR_SIZE)
|
|
# ifdef USE_AS_STPCPY
|
|
leaq (VEC_SIZE * 4)(%rdi, %rcx, CHAR_SIZE), %rax
|
|
# endif
|
|
ret
|
|
|
|
|
|
.p2align 4,, 4
|
|
L(page_cross):
|
|
# ifndef USE_AS_STRCAT
|
|
vpxorq %VZERO_128, %VZERO_128, %VZERO_128
|
|
# endif
|
|
movq %rsi, %rcx
|
|
andq $(VEC_SIZE * -1), %rcx
|
|
|
|
VPCMPEQ (%rcx), %VZERO, %k0
|
|
KMOV %k0, %VRCX
|
|
# ifdef USE_AS_WCSCPY
|
|
andl $(VEC_SIZE - 1), %PAGE_ALIGN_REG
|
|
shrl $2, %PAGE_ALIGN_REG
|
|
# endif
|
|
shrx %VGPR(PAGE_ALIGN_REG_64), %VRCX, %VRCX
|
|
|
|
# if USE_MOVSB_IN_PAGE_CROSS
|
|
/* Optimizing more aggressively for space as this is very cold
|
|
code. This saves 2x cache lines. */
|
|
|
|
/* This adds once to the later result which will get correct
|
|
copy bounds. NB: this can never zero-out a non-zero RCX as
|
|
to be in the page cross case rsi cannot be aligned and we
|
|
already right-shift rcx by the misalignment. */
|
|
shl %VRCX
|
|
jz L(page_cross_continue)
|
|
# if !defined USE_AS_STPCPY && !defined USE_AS_STRCAT
|
|
movq %rdi, %rax
|
|
# endif
|
|
bsf %VRCX, %VRCX
|
|
REP_MOVS
|
|
|
|
# ifdef USE_AS_STPCPY
|
|
leaq -CHAR_SIZE(%rdi), %rax
|
|
# endif
|
|
ret
|
|
|
|
|
|
# else
|
|
/* Check if we found zero-char before end of page. */
|
|
test %VRCX, %VRCX
|
|
jz L(page_cross_continue)
|
|
|
|
/* Traditional copy case, essentially same as used in non-page-
|
|
cross case but since we can't reuse VMM(0) we need twice as
|
|
many loads from rsi. */
|
|
|
|
# ifndef USE_AS_STRCAT
|
|
xorl %edx, %edx
|
|
# endif
|
|
/* Dependency on rdi must already have been satisfied. */
|
|
bsf %VRCX, %VRDX
|
|
# ifdef USE_AS_STPCPY
|
|
leaq (%rdi, %rdx, CHAR_SIZE), %rax
|
|
# elif !defined USE_AS_STRCAT
|
|
movq %rdi, %rax
|
|
# endif
|
|
|
|
# if VEC_SIZE == 64
|
|
# ifdef USE_AS_WCSCPY
|
|
testb %cl, %cl
|
|
# else
|
|
test %ecx, %ecx
|
|
# endif
|
|
jz L(page_cross_copy_32_63)
|
|
# endif
|
|
|
|
# ifdef USE_AS_WCSCPY
|
|
testb $0xf, %cl
|
|
# else
|
|
testw %cx, %cx
|
|
# endif
|
|
jz L(page_cross_copy_16_31)
|
|
|
|
# ifdef USE_AS_WCSCPY
|
|
testb $0x3, %cl
|
|
# else
|
|
testb %cl, %cl
|
|
# endif
|
|
jz L(page_cross_copy_8_15)
|
|
|
|
# ifdef USE_AS_WCSCPY
|
|
movl (%rsi), %esi
|
|
movl %esi, (%rdi)
|
|
movl $0, (%END_REG)
|
|
ret
|
|
# else
|
|
|
|
testb $0x7, %cl
|
|
jz L(page_cross_copy_4_7)
|
|
|
|
test %edx, %edx
|
|
jz L(page_cross_set_null_term)
|
|
movzwl (%rsi), %ecx
|
|
movw %cx, (%rdi)
|
|
L(page_cross_set_null_term):
|
|
movb $0, (%END_REG)
|
|
ret
|
|
|
|
|
|
.p2align 4,, 4
|
|
L(page_cross_copy_4_7):
|
|
movl (%rsi), %ecx
|
|
movl -(4 - CHAR_SIZE)(%rsi, %rdx, CHAR_SIZE), %esi
|
|
movl %ecx, (%rdi)
|
|
movl %esi, -(4 - CHAR_SIZE)(%END_REG)
|
|
ret
|
|
# endif
|
|
|
|
# if VEC_SIZE == 64
|
|
.p2align 4,, 4
|
|
L(page_cross_copy_32_63):
|
|
VMOVU (%rsi), %VMM_256(0)
|
|
VMOVU -(32 - CHAR_SIZE)(%rsi, %rdx, CHAR_SIZE), %VMM_256(1)
|
|
VMOVU %VMM_256(0), (%rdi)
|
|
VMOVU %VMM_256(1), -(32 - CHAR_SIZE)(%END_REG)
|
|
ret
|
|
# endif
|
|
|
|
.p2align 4,, 4
|
|
L(page_cross_copy_16_31):
|
|
vmovdqu (%rsi), %xmm0
|
|
vmovdqu -(16 - CHAR_SIZE)(%rsi, %rdx, CHAR_SIZE), %xmm1
|
|
vmovdqu %xmm0, (%rdi)
|
|
vmovdqu %xmm1, -(16 - CHAR_SIZE)(%END_REG)
|
|
ret
|
|
|
|
.p2align 4,, 4
|
|
L(page_cross_copy_8_15):
|
|
movq (%rsi), %rcx
|
|
movq -(8 - CHAR_SIZE)(%rsi, %rdx, CHAR_SIZE), %rsi
|
|
movq %rcx, (%rdi)
|
|
movq %rsi, -(8 - CHAR_SIZE)(%END_REG)
|
|
ret
|
|
# endif
|
|
END(STRCPY)
|
|
#endif
|