2010-01-12 19:22:03 +00:00
|
|
|
/* Multiple versions of memmove
|
2012-10-10 14:48:45 +00:00
|
|
|
All versions must be listed in ifunc-impl-list.c.
|
2017-01-01 00:14:16 +00:00
|
|
|
Copyright (C) 2010-2017 Free Software Foundation, Inc.
|
2010-01-12 19:22:03 +00:00
|
|
|
Contributed by Intel Corporation.
|
|
|
|
This file is part of the GNU C Library.
|
|
|
|
|
|
|
|
The GNU C Library is free software; you can redistribute it and/or
|
|
|
|
modify it under the terms of the GNU Lesser General Public
|
|
|
|
License as published by the Free Software Foundation; either
|
|
|
|
version 2.1 of the License, or (at your option) any later version.
|
|
|
|
|
|
|
|
The GNU C Library is distributed in the hope that it will be useful,
|
|
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
Lesser General Public License for more details.
|
|
|
|
|
|
|
|
You should have received a copy of the GNU Lesser General Public
|
2012-02-09 23:18:22 +00:00
|
|
|
License along with the GNU C Library; if not, see
|
|
|
|
<http://www.gnu.org/licenses/>. */
|
2010-01-12 19:22:03 +00:00
|
|
|
|
|
|
|
#include <sysdep.h>
|
|
|
|
#include <init-arch.h>
|
|
|
|
|
|
|
|
/* Define multiple versions only for the definition in lib. */
|
2014-11-24 09:33:45 +00:00
|
|
|
#if IS_IN (libc)
|
2010-01-12 19:22:03 +00:00
|
|
|
.text
|
|
|
|
ENTRY(memmove)
|
|
|
|
.type memmove, @gnu_indirect_function
|
2015-08-13 10:39:22 +00:00
|
|
|
LOAD_GOT_AND_RTLD_GLOBAL_RO
|
|
|
|
LOAD_FUNC_GOT_EAX (__memmove_ia32)
|
|
|
|
HAS_CPU_FEATURE (SSE2)
|
i386: memcpy functions with SSE2 unaligned load/store
These new memcpy functions are the 32-bit version of x86_64 SSE2 unaligned
memcpy. Memcpy average performace benefit is 18% on Silvermont, other
platforms also improved about 35%, benchmarked on Silvermont, Haswell, Ivy
Bridge, Sandy Bridge and Westmere, performance results attached in
https://sourceware.org/ml/libc-alpha/2014-07/msg00157.html
* sysdeps/i386/i686/multiarch/bcopy-sse2-unaligned.S: New file.
* sysdeps/i386/i686/multiarch/memcpy-sse2-unaligned.S: Likewise.
* sysdeps/i386/i686/multiarch/memmove-sse2-unaligned.S: Likewise.
* sysdeps/i386/i686/multiarch/mempcpy-sse2-unaligned.S: Likewise.
* sysdeps/i386/i686/multiarch/bcopy.S: Select the sse2_unaligned
version if bit_Fast_Unaligned_Load is set.
* sysdeps/i386/i686/multiarch/memcpy.S: Likewise.
* sysdeps/i386/i686/multiarch/memcpy_chk.S: Likewise.
* sysdeps/i386/i686/multiarch/memmove.S: Likewise.
* sysdeps/i386/i686/multiarch/memmove_chk.S: Likewise.
* sysdeps/i386/i686/multiarch/mempcpy.S: Likewise.
* sysdeps/i386/i686/multiarch/mempcpy_chk.S: Likewise.
* sysdeps/i386/i686/multiarch/Makefile (sysdep_routines): Add
bcopy-sse2-unaligned, memcpy-sse2-unaligned,
memmove-sse2-unaligned and mempcpy-sse2-unaligned.
* sysdeps/i386/i686/multiarch/ifunc-impl-list.c (MAX_IFUNC): Set
to 4.
(__libc_ifunc_impl_list): Test __bcopy_sse2_unaligned,
__memmove_chk_sse2_unaligned, __memmove_sse2_unaligned,
__memcpy_chk_sse2_unaligned, __memcpy_sse2_unaligned,
__mempcpy_chk_sse2_unaligned, and __mempcpy_sse2_unaligned.
2014-12-29 11:39:46 +00:00
|
|
|
jz 2f
|
2015-08-13 10:39:22 +00:00
|
|
|
LOAD_FUNC_GOT_EAX (__memmove_sse2_unaligned)
|
|
|
|
HAS_ARCH_FEATURE (Fast_Unaligned_Load)
|
i386: memcpy functions with SSE2 unaligned load/store
These new memcpy functions are the 32-bit version of x86_64 SSE2 unaligned
memcpy. Memcpy average performace benefit is 18% on Silvermont, other
platforms also improved about 35%, benchmarked on Silvermont, Haswell, Ivy
Bridge, Sandy Bridge and Westmere, performance results attached in
https://sourceware.org/ml/libc-alpha/2014-07/msg00157.html
* sysdeps/i386/i686/multiarch/bcopy-sse2-unaligned.S: New file.
* sysdeps/i386/i686/multiarch/memcpy-sse2-unaligned.S: Likewise.
* sysdeps/i386/i686/multiarch/memmove-sse2-unaligned.S: Likewise.
* sysdeps/i386/i686/multiarch/mempcpy-sse2-unaligned.S: Likewise.
* sysdeps/i386/i686/multiarch/bcopy.S: Select the sse2_unaligned
version if bit_Fast_Unaligned_Load is set.
* sysdeps/i386/i686/multiarch/memcpy.S: Likewise.
* sysdeps/i386/i686/multiarch/memcpy_chk.S: Likewise.
* sysdeps/i386/i686/multiarch/memmove.S: Likewise.
* sysdeps/i386/i686/multiarch/memmove_chk.S: Likewise.
* sysdeps/i386/i686/multiarch/mempcpy.S: Likewise.
* sysdeps/i386/i686/multiarch/mempcpy_chk.S: Likewise.
* sysdeps/i386/i686/multiarch/Makefile (sysdep_routines): Add
bcopy-sse2-unaligned, memcpy-sse2-unaligned,
memmove-sse2-unaligned and mempcpy-sse2-unaligned.
* sysdeps/i386/i686/multiarch/ifunc-impl-list.c (MAX_IFUNC): Set
to 4.
(__libc_ifunc_impl_list): Test __bcopy_sse2_unaligned,
__memmove_chk_sse2_unaligned, __memmove_sse2_unaligned,
__memcpy_chk_sse2_unaligned, __memcpy_sse2_unaligned,
__mempcpy_chk_sse2_unaligned, and __mempcpy_sse2_unaligned.
2014-12-29 11:39:46 +00:00
|
|
|
jnz 2f
|
2015-08-13 10:39:22 +00:00
|
|
|
HAS_CPU_FEATURE (SSSE3)
|
2010-01-12 19:22:03 +00:00
|
|
|
jz 2f
|
2015-08-13 10:39:22 +00:00
|
|
|
LOAD_FUNC_GOT_EAX (__memmove_ssse3)
|
|
|
|
HAS_ARCH_FEATURE (Fast_Rep_String)
|
2010-01-12 19:22:03 +00:00
|
|
|
jz 2f
|
2015-08-13 10:39:22 +00:00
|
|
|
LOAD_FUNC_GOT_EAX (__memmove_ssse3_rep)
|
|
|
|
2: ret
|
2010-01-12 19:22:03 +00:00
|
|
|
END(memmove)
|
|
|
|
|
2015-08-13 10:39:22 +00:00
|
|
|
# ifdef SHARED
|
|
|
|
# undef ENTRY
|
|
|
|
# define ENTRY(name) \
|
2010-01-12 19:22:03 +00:00
|
|
|
.type __memmove_ia32, @function; \
|
|
|
|
.p2align 4; \
|
2012-10-10 14:48:45 +00:00
|
|
|
.globl __memmove_ia32; \
|
|
|
|
.hidden __memmove_ia32; \
|
2010-01-12 19:22:03 +00:00
|
|
|
__memmove_ia32: cfi_startproc; \
|
|
|
|
CALL_MCOUNT
|
|
|
|
# else
|
2015-08-13 10:39:22 +00:00
|
|
|
# undef ENTRY
|
|
|
|
# define ENTRY(name) \
|
2010-01-12 19:22:03 +00:00
|
|
|
.type __memmove_ia32, @function; \
|
|
|
|
.globl __memmove_ia32; \
|
|
|
|
.p2align 4; \
|
|
|
|
__memmove_ia32: cfi_startproc; \
|
|
|
|
CALL_MCOUNT
|
|
|
|
# endif
|
|
|
|
|
|
|
|
# undef END
|
|
|
|
# define END(name) \
|
|
|
|
cfi_endproc; .size __memmove_ia32, .-__memmove_ia32
|
|
|
|
|
|
|
|
# undef ENTRY_CHK
|
|
|
|
# define ENTRY_CHK(name) \
|
|
|
|
.type __memmove_chk_ia32, @function; \
|
|
|
|
.globl __memmove_chk_ia32; \
|
|
|
|
.p2align 4; \
|
|
|
|
__memmove_chk_ia32: cfi_startproc; \
|
|
|
|
CALL_MCOUNT
|
|
|
|
# undef END_CHK
|
|
|
|
# define END_CHK(name) \
|
|
|
|
cfi_endproc; .size __memmove_chk_ia32, .-__memmove_chk_ia32
|
|
|
|
|
|
|
|
# ifdef SHARED
|
|
|
|
# undef libc_hidden_builtin_def
|
|
|
|
/* IFUNC doesn't work with the hidden functions in shared library since
|
|
|
|
they will be called without setting up EBX needed for PLT which is
|
|
|
|
used by IFUNC. */
|
|
|
|
# define libc_hidden_builtin_def(name) \
|
|
|
|
.globl __GI_memmove; __GI_memmove = __memmove_ia32
|
|
|
|
# endif
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#include "../memmove.S"
|