mirror of
https://sourceware.org/git/glibc.git
synced 2024-11-08 14:20:07 +00:00
x86-64: Implement memmove family IFUNC selectors in C
Implement memmove family IFUNC selectors in C. All internal calls within libc.so can use IFUNC on x86-64 since unlike x86, x86-64 supports PC-relative addressing to access the GOT entry so that it can call via PLT without using an extra register. For libc.a, we can't use IFUNC for functions which are called before IFUNC has been initialized. Use IFUNC internally reduces the icache footprint since libc.so and other codes in the process use the same implementations. This patch uses IFUNC for memmove family functions within libc. * sysdeps/x86_64/multiarch/Makefile (sysdep_routines): Add memmove-sse2-unaligned-erms, memcpy_chk-nonshared, mempcpy_chk-nonshared and memmove_chk-nonshared. * sysdeps/x86_64/multiarch/ifunc-impl-list.c (__libc_ifunc_impl_list): Add tests for __memmove_chk_erms, __memcpy_chk_erms and __mempcpy_chk_erms. Update comments. * sysdeps/x86_64/multiarch/ifunc-memmove.h: New file. * sysdeps/x86_64/multiarch/memcpy.c: Likewise. * sysdeps/x86_64/multiarch/memcpy_chk-nonshared.S: Likewise. * sysdeps/x86_64/multiarch/memcpy_chk.c: Likewise. * sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S: Likewise. * sysdeps/x86_64/multiarch/memmove.c: Likewise. * sysdeps/x86_64/multiarch/memmove_chk-nonshared.S: Likewise. * sysdeps/x86_64/multiarch/memmove_chk.c: Likewise. * sysdeps/x86_64/multiarch/mempcpy.c: Likewise. * sysdeps/x86_64/multiarch/mempcpy_chk-nonshared.S: Likewise. * sysdeps/x86_64/multiarch/mempcpy_chk.c: Likewise. * sysdeps/x86_64/multiarch/memcpy.S: Removed. * sysdeps/x86_64/multiarch/memcpy_chk.S: Likewise. * sysdeps/x86_64/multiarch/memmove.S: Likewise. * sysdeps/x86_64/multiarch/memmove_chk.S: Likewise. * sysdeps/x86_64/multiarch/mempcpy.S: Likewise. * sysdeps/x86_64/multiarch/mempcpy_chk.S: Likewise. * sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S (__mempcpy_chk_erms): New function. (__memmove_chk_erms): Likewise. (__memcpy_chk_erms): New alias.
This commit is contained in:
H.J. Lu
parent
69052a3a95
commit
5c3e322d3b
32
ChangeLog
32
ChangeLog
@ -1,3 +1,35 @@
|
||||
2017-06-14 H.J. Lu <hongjiu.lu@intel.com>
|
||||
Erich Elsen <eriche@google.com>
|
||||
Adhemerval Zanella <adhemerval.zanella@linaro.org>
|
||||
|
||||
* sysdeps/x86_64/multiarch/Makefile (sysdep_routines): Add
|
||||
memmove-sse2-unaligned-erms, memcpy_chk-nonshared,
|
||||
mempcpy_chk-nonshared and memmove_chk-nonshared.
|
||||
* sysdeps/x86_64/multiarch/ifunc-impl-list.c
|
||||
(__libc_ifunc_impl_list): Add tests for __memmove_chk_erms,
|
||||
__memcpy_chk_erms and __mempcpy_chk_erms. Update comments.
|
||||
* sysdeps/x86_64/multiarch/ifunc-memmove.h: New file.
|
||||
* sysdeps/x86_64/multiarch/memcpy.c: Likewise.
|
||||
* sysdeps/x86_64/multiarch/memcpy_chk-nonshared.S: Likewise.
|
||||
* sysdeps/x86_64/multiarch/memcpy_chk.c: Likewise.
|
||||
* sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S: Likewise.
|
||||
* sysdeps/x86_64/multiarch/memmove.c: Likewise.
|
||||
* sysdeps/x86_64/multiarch/memmove_chk-nonshared.S: Likewise.
|
||||
* sysdeps/x86_64/multiarch/memmove_chk.c: Likewise.
|
||||
* sysdeps/x86_64/multiarch/mempcpy.c: Likewise.
|
||||
* sysdeps/x86_64/multiarch/mempcpy_chk-nonshared.S: Likewise.
|
||||
* sysdeps/x86_64/multiarch/mempcpy_chk.c: Likewise.
|
||||
* sysdeps/x86_64/multiarch/memcpy.S: Removed.
|
||||
* sysdeps/x86_64/multiarch/memcpy_chk.S: Likewise.
|
||||
* sysdeps/x86_64/multiarch/memmove.S: Likewise.
|
||||
* sysdeps/x86_64/multiarch/memmove_chk.S: Likewise.
|
||||
* sysdeps/x86_64/multiarch/mempcpy.S: Likewise.
|
||||
* sysdeps/x86_64/multiarch/mempcpy_chk.S: Likewise.
|
||||
* sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S
|
||||
(__mempcpy_chk_erms): New function.
|
||||
(__memmove_chk_erms): Likewise.
|
||||
(__memcpy_chk_erms): New alias.
|
||||
|
||||
2017-06-14 Florian Weimer <fweimer@redhat.com>
|
||||
|
||||
* sysdeps/i386/i686/multiarch/strcspn-c.c: Add IS_IN (libc) guard.
|
||||
|
||||
@ -26,6 +26,7 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c strcmp-ssse3 \
|
||||
strchr-sse2-no-bsf memcmp-ssse3 strstr-sse2-unaligned \
|
||||
strcspn-c strpbrk-c strspn-c varshift \
|
||||
memset-avx512-no-vzeroupper \
|
||||
memmove-sse2-unaligned-erms \
|
||||
memmove-avx-unaligned-erms \
|
||||
memmove-avx512-unaligned-erms \
|
||||
memset-avx2-unaligned-erms \
|
||||
@ -48,5 +49,7 @@ sysdep_routines += wmemcmp-sse4 wmemcmp-ssse3 wmemcmp-c \
|
||||
endif
|
||||
|
||||
ifeq ($(subdir),debug)
|
||||
sysdep_routines += wmemset_chk-nonshared
|
||||
sysdep_routines += memcpy_chk-nonshared mempcpy_chk-nonshared \
|
||||
memmove_chk-nonshared \
|
||||
wmemset_chk-nonshared
|
||||
endif
|
||||
|
||||
@ -57,7 +57,7 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
||||
__memcmp_ssse3)
|
||||
IFUNC_IMPL_ADD (array, i, memcmp, 1, __memcmp_sse2))
|
||||
|
||||
/* Support sysdeps/x86_64/multiarch/memmove_chk.S. */
|
||||
/* Support sysdeps/x86_64/multiarch/memmove_chk.c. */
|
||||
IFUNC_IMPL (i, name, __memmove_chk,
|
||||
IFUNC_IMPL_ADD (array, i, __memmove_chk,
|
||||
HAS_ARCH_FEATURE (AVX512F_Usable),
|
||||
@ -83,9 +83,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
||||
IFUNC_IMPL_ADD (array, i, __memmove_chk, 1,
|
||||
__memmove_chk_sse2_unaligned)
|
||||
IFUNC_IMPL_ADD (array, i, __memmove_chk, 1,
|
||||
__memmove_chk_sse2_unaligned_erms))
|
||||
__memmove_chk_sse2_unaligned_erms)
|
||||
IFUNC_IMPL_ADD (array, i, __memmove_chk, 1,
|
||||
__memmove_chk_erms))
|
||||
|
||||
/* Support sysdeps/x86_64/multiarch/memmove.S. */
|
||||
/* Support sysdeps/x86_64/multiarch/memmove.c. */
|
||||
IFUNC_IMPL (i, name, memmove,
|
||||
IFUNC_IMPL_ADD (array, i, memmove,
|
||||
HAS_ARCH_FEATURE (AVX_Usable),
|
||||
@ -410,7 +412,7 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
||||
__wmemset_avx512_unaligned))
|
||||
|
||||
#ifdef SHARED
|
||||
/* Support sysdeps/x86_64/multiarch/memcpy_chk.S. */
|
||||
/* Support sysdeps/x86_64/multiarch/memcpy_chk.c. */
|
||||
IFUNC_IMPL (i, name, __memcpy_chk,
|
||||
IFUNC_IMPL_ADD (array, i, __memcpy_chk,
|
||||
HAS_ARCH_FEATURE (AVX512F_Usable),
|
||||
@ -436,9 +438,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
||||
IFUNC_IMPL_ADD (array, i, __memcpy_chk, 1,
|
||||
__memcpy_chk_sse2_unaligned)
|
||||
IFUNC_IMPL_ADD (array, i, __memcpy_chk, 1,
|
||||
__memcpy_chk_sse2_unaligned_erms))
|
||||
__memcpy_chk_sse2_unaligned_erms)
|
||||
IFUNC_IMPL_ADD (array, i, __memcpy_chk, 1,
|
||||
__memcpy_chk_erms))
|
||||
|
||||
/* Support sysdeps/x86_64/multiarch/memcpy.S. */
|
||||
/* Support sysdeps/x86_64/multiarch/memcpy.c. */
|
||||
IFUNC_IMPL (i, name, memcpy,
|
||||
IFUNC_IMPL_ADD (array, i, memcpy,
|
||||
HAS_ARCH_FEATURE (AVX_Usable),
|
||||
@ -464,7 +468,7 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
||||
__memcpy_sse2_unaligned_erms)
|
||||
IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_erms))
|
||||
|
||||
/* Support sysdeps/x86_64/multiarch/mempcpy_chk.S. */
|
||||
/* Support sysdeps/x86_64/multiarch/mempcpy_chk.c. */
|
||||
IFUNC_IMPL (i, name, __mempcpy_chk,
|
||||
IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
|
||||
HAS_ARCH_FEATURE (AVX512F_Usable),
|
||||
@ -490,9 +494,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
||||
IFUNC_IMPL_ADD (array, i, __mempcpy_chk, 1,
|
||||
__mempcpy_chk_sse2_unaligned)
|
||||
IFUNC_IMPL_ADD (array, i, __mempcpy_chk, 1,
|
||||
__mempcpy_chk_sse2_unaligned_erms))
|
||||
__mempcpy_chk_sse2_unaligned_erms)
|
||||
IFUNC_IMPL_ADD (array, i, __mempcpy_chk, 1,
|
||||
__mempcpy_chk_erms))
|
||||
|
||||
/* Support sysdeps/x86_64/multiarch/mempcpy.S. */
|
||||
/* Support sysdeps/x86_64/multiarch/mempcpy.c. */
|
||||
IFUNC_IMPL (i, name, mempcpy,
|
||||
IFUNC_IMPL_ADD (array, i, mempcpy,
|
||||
HAS_ARCH_FEATURE (AVX512F_Usable),
|
||||
|
||||
80
sysdeps/x86_64/multiarch/ifunc-memmove.h
Normal file
80
sysdeps/x86_64/multiarch/ifunc-memmove.h
Normal file
@ -0,0 +1,80 @@
|
||||
/* Common definition for memcpy, mempcpy and memmove implementation.
|
||||
All versions must be listed in ifunc-impl-list.c.
|
||||
Copyright (C) 2017 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<http://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <init-arch.h>
|
||||
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE (erms) attribute_hidden;
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned)
|
||||
attribute_hidden;
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned_erms)
|
||||
attribute_hidden;
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden;
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3_back) attribute_hidden;
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned) attribute_hidden;
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx_unaligned_erms)
|
||||
attribute_hidden;
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned)
|
||||
attribute_hidden;
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_unaligned_erms)
|
||||
attribute_hidden;
|
||||
extern __typeof (REDIRECT_NAME) OPTIMIZE (avx512_no_vzeroupper)
|
||||
attribute_hidden;
|
||||
|
||||
static inline void *
|
||||
IFUNC_SELECTOR (void)
|
||||
{
|
||||
const struct cpu_features* cpu_features = __get_cpu_features ();
|
||||
|
||||
if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_ERMS))
|
||||
return OPTIMIZE (erms);
|
||||
|
||||
if (CPU_FEATURES_ARCH_P (cpu_features, AVX512F_Usable)
|
||||
&& !CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_AVX512))
|
||||
{
|
||||
if (CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER))
|
||||
return OPTIMIZE (avx512_no_vzeroupper);
|
||||
|
||||
if (CPU_FEATURES_CPU_P (cpu_features, ERMS))
|
||||
return OPTIMIZE (avx512_unaligned_erms);
|
||||
|
||||
return OPTIMIZE (avx512_unaligned);
|
||||
}
|
||||
|
||||
if (CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load))
|
||||
{
|
||||
if (CPU_FEATURES_CPU_P (cpu_features, ERMS))
|
||||
return OPTIMIZE (avx_unaligned_erms);
|
||||
|
||||
return OPTIMIZE (avx_unaligned);
|
||||
}
|
||||
|
||||
if (!CPU_FEATURES_CPU_P (cpu_features, SSSE3)
|
||||
|| CPU_FEATURES_ARCH_P (cpu_features, Fast_Unaligned_Copy))
|
||||
{
|
||||
if (CPU_FEATURES_CPU_P (cpu_features, ERMS))
|
||||
return OPTIMIZE (sse2_unaligned_erms);
|
||||
|
||||
return OPTIMIZE (sse2_unaligned);
|
||||
}
|
||||
|
||||
if (CPU_FEATURES_ARCH_P (cpu_features, Fast_Copy_Backward))
|
||||
return OPTIMIZE (ssse3_back);
|
||||
|
||||
return OPTIMIZE (ssse3);
|
||||
}
|
||||
@ -1,75 +0,0 @@
|
||||
/* Multiple versions of memcpy
|
||||
All versions must be listed in ifunc-impl-list.c.
|
||||
Copyright (C) 2010-2017 Free Software Foundation, Inc.
|
||||
Contributed by Intel Corporation.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<http://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <sysdep.h>
|
||||
#include <init-arch.h>
|
||||
|
||||
/* Define multiple versions only for the definition in lib and for
|
||||
DSO. In static binaries we need memcpy before the initialization
|
||||
happened. */
|
||||
#if defined SHARED && IS_IN (libc)
|
||||
.text
|
||||
ENTRY(__new_memcpy)
|
||||
.type __new_memcpy, @gnu_indirect_function
|
||||
LOAD_RTLD_GLOBAL_RO_RDX
|
||||
lea __memcpy_erms(%rip), %RAX_LP
|
||||
HAS_ARCH_FEATURE (Prefer_ERMS)
|
||||
jnz 2f
|
||||
HAS_ARCH_FEATURE (Prefer_No_AVX512)
|
||||
jnz 1f
|
||||
HAS_ARCH_FEATURE (AVX512F_Usable)
|
||||
jz 1f
|
||||
lea __memcpy_avx512_no_vzeroupper(%rip), %RAX_LP
|
||||
HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER)
|
||||
jnz 2f
|
||||
lea __memcpy_avx512_unaligned_erms(%rip), %RAX_LP
|
||||
HAS_CPU_FEATURE (ERMS)
|
||||
jnz 2f
|
||||
lea __memcpy_avx512_unaligned(%rip), %RAX_LP
|
||||
ret
|
||||
1: lea __memcpy_avx_unaligned(%rip), %RAX_LP
|
||||
HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
|
||||
jz L(Fast_Unaligned_Load)
|
||||
HAS_CPU_FEATURE (ERMS)
|
||||
jz 2f
|
||||
lea __memcpy_avx_unaligned_erms(%rip), %RAX_LP
|
||||
ret
|
||||
L(Fast_Unaligned_Load):
|
||||
lea __memcpy_sse2_unaligned(%rip), %RAX_LP
|
||||
HAS_ARCH_FEATURE (Fast_Unaligned_Copy)
|
||||
jz L(SSSE3)
|
||||
HAS_CPU_FEATURE (ERMS)
|
||||
jz 2f
|
||||
lea __memcpy_sse2_unaligned_erms(%rip), %RAX_LP
|
||||
ret
|
||||
L(SSSE3):
|
||||
HAS_CPU_FEATURE (SSSE3)
|
||||
jz 2f
|
||||
lea __memcpy_ssse3_back(%rip), %RAX_LP
|
||||
HAS_ARCH_FEATURE (Fast_Copy_Backward)
|
||||
jnz 2f
|
||||
lea __memcpy_ssse3(%rip), %RAX_LP
|
||||
2: ret
|
||||
END(__new_memcpy)
|
||||
|
||||
# undef memcpy
|
||||
# include <shlib-compat.h>
|
||||
versioned_symbol (libc, __new_memcpy, memcpy, GLIBC_2_14);
|
||||
#endif
|
||||
39
sysdeps/x86_64/multiarch/memcpy.c
Normal file
39
sysdeps/x86_64/multiarch/memcpy.c
Normal file
@ -0,0 +1,39 @@
|
||||
/* Multiple versions of memcpy.
|
||||
All versions must be listed in ifunc-impl-list.c.
|
||||
Copyright (C) 2017 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<http://www.gnu.org/licenses/>. */
|
||||
|
||||
/* Define multiple versions only for the definition in lib and for
|
||||
DSO. In static binaries we need memcpy before the initialization
|
||||
happened. */
|
||||
#if defined SHARED && IS_IN (libc)
|
||||
# define memcpy __redirect_memcpy
|
||||
# include <string.h>
|
||||
# undef memcpy
|
||||
|
||||
# define SYMBOL_NAME memcpy
|
||||
# include "ifunc-memmove.h"
|
||||
|
||||
libc_ifunc_redirected (__redirect_memcpy, __new_memcpy,
|
||||
IFUNC_SELECTOR ());
|
||||
|
||||
__hidden_ver1 (__new_memcpy, __GI_memcpy, __redirect_memcpy)
|
||||
__attribute__ ((visibility ("hidden")));
|
||||
|
||||
# include <shlib-compat.h>
|
||||
versioned_symbol (libc, __new_memcpy, memcpy, GLIBC_2_14);
|
||||
#endif
|
||||
21
sysdeps/x86_64/multiarch/memcpy_chk-nonshared.S
Normal file
21
sysdeps/x86_64/multiarch/memcpy_chk-nonshared.S
Normal file
@ -0,0 +1,21 @@
|
||||
/* Non-shared version of memcpy_chk for x86-64.
|
||||
Copyright (C) 2017 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<http://www.gnu.org/licenses/>. */
|
||||
|
||||
#if IS_IN (libc) && !defined SHARED
|
||||
# include <sysdeps/x86_64/memcpy_chk.S>
|
||||
#endif
|
||||
@ -1,72 +0,0 @@
|
||||
/* Multiple versions of __memcpy_chk
|
||||
All versions must be listed in ifunc-impl-list.c.
|
||||
Copyright (C) 2010-2017 Free Software Foundation, Inc.
|
||||
Contributed by Intel Corporation.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<http://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <sysdep.h>
|
||||
#include <init-arch.h>
|
||||
|
||||
/* Define multiple versions only for the definition in lib and for
|
||||
DSO. There are no multiarch memcpy functions for static binaries.
|
||||
*/
|
||||
#if IS_IN (libc)
|
||||
# ifdef SHARED
|
||||
.text
|
||||
ENTRY(__memcpy_chk)
|
||||
.type __memcpy_chk, @gnu_indirect_function
|
||||
LOAD_RTLD_GLOBAL_RO_RDX
|
||||
HAS_ARCH_FEATURE (Prefer_No_AVX512)
|
||||
jnz 1f
|
||||
HAS_ARCH_FEATURE (AVX512F_Usable)
|
||||
jz 1f
|
||||
lea __memcpy_chk_avx512_no_vzeroupper(%rip), %RAX_LP
|
||||
HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER)
|
||||
jnz 2f
|
||||
lea __memcpy_chk_avx512_unaligned_erms(%rip), %RAX_LP
|
||||
HAS_CPU_FEATURE (ERMS)
|
||||
jnz 2f
|
||||
lea __memcpy_chk_avx512_unaligned(%rip), %RAX_LP
|
||||
ret
|
||||
1: lea __memcpy_chk_avx_unaligned(%rip), %RAX_LP
|
||||
HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
|
||||
jz L(Fast_Unaligned_Load)
|
||||
HAS_CPU_FEATURE (ERMS)
|
||||
jz 2f
|
||||
lea __memcpy_chk_avx_unaligned_erms(%rip), %RAX_LP
|
||||
ret
|
||||
L(Fast_Unaligned_Load):
|
||||
lea __memcpy_chk_sse2_unaligned(%rip), %RAX_LP
|
||||
HAS_ARCH_FEATURE (Fast_Unaligned_Copy)
|
||||
jz L(SSSE3)
|
||||
HAS_CPU_FEATURE (ERMS)
|
||||
jz 2f
|
||||
lea __memcpy_chk_sse2_unaligned_erms(%rip), %RAX_LP
|
||||
ret
|
||||
L(SSSE3):
|
||||
HAS_CPU_FEATURE (SSSE3)
|
||||
jz 2f
|
||||
lea __memcpy_chk_ssse3_back(%rip), %RAX_LP
|
||||
HAS_ARCH_FEATURE (Fast_Copy_Backward)
|
||||
jnz 2f
|
||||
lea __memcpy_chk_ssse3(%rip), %RAX_LP
|
||||
2: ret
|
||||
END(__memcpy_chk)
|
||||
# else
|
||||
# include "../memcpy_chk.S"
|
||||
# endif
|
||||
#endif
|
||||
31
sysdeps/x86_64/multiarch/memcpy_chk.c
Normal file
31
sysdeps/x86_64/multiarch/memcpy_chk.c
Normal file
@ -0,0 +1,31 @@
|
||||
/* Multiple versions of __memcpy_chk
|
||||
All versions must be listed in ifunc-impl-list.c.
|
||||
Copyright (C) 2017 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<http://www.gnu.org/licenses/>. */
|
||||
|
||||
/* Define multiple versions only for the definition in libc.so. */
|
||||
#if IS_IN (libc) && defined SHARED
|
||||
# define __memcpy_chk __redirect_memcpy_chk
|
||||
# include <string.h>
|
||||
# undef __memcpy_chk
|
||||
|
||||
# define SYMBOL_NAME memcpy_chk
|
||||
# include "ifunc-memmove.h"
|
||||
|
||||
libc_ifunc_redirected (__redirect_memcpy_chk, __memcpy_chk,
|
||||
IFUNC_SELECTOR ());
|
||||
#endif
|
||||
35
sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S
Normal file
35
sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S
Normal file
@ -0,0 +1,35 @@
|
||||
/* memmove with SSE2.
|
||||
Copyright (C) 2017 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<http://www.gnu.org/licenses/>. */
|
||||
|
||||
#if IS_IN (libc)
|
||||
# define MEMMOVE_SYMBOL(p,s) p##_sse2_##s
|
||||
#endif
|
||||
|
||||
#if !defined SHARED || !IS_IN (libc)
|
||||
weak_alias (__mempcpy, mempcpy)
|
||||
#endif
|
||||
|
||||
#include <sysdeps/x86_64/memmove.S>
|
||||
|
||||
#if defined SHARED && IS_IN (libc)
|
||||
# include <shlib-compat.h>
|
||||
# if SHLIB_COMPAT (libc, GLIBC_2_2_5, GLIBC_2_14)
|
||||
/* Use __memmove_sse2_unaligned to support overlapping addresses. */
|
||||
compat_symbol (libc, __memmove_sse2_unaligned, memcpy, GLIBC_2_2_5);
|
||||
# endif
|
||||
#endif
|
||||
@ -152,6 +152,11 @@ END (MEMMOVE_SYMBOL (__memmove, unaligned))
|
||||
|
||||
# if VEC_SIZE == 16
|
||||
# if defined SHARED
|
||||
ENTRY (__mempcpy_chk_erms)
|
||||
cmpq %rdx, %rcx
|
||||
jb HIDDEN_JUMPTARGET (__chk_fail)
|
||||
END (__mempcpy_chk_erms)
|
||||
|
||||
/* Only used to measure performance of REP MOVSB. */
|
||||
ENTRY (__mempcpy_erms)
|
||||
movq %rdi, %rax
|
||||
@ -160,6 +165,11 @@ ENTRY (__mempcpy_erms)
|
||||
END (__mempcpy_erms)
|
||||
# endif
|
||||
|
||||
ENTRY (__memmove_chk_erms)
|
||||
cmpq %rdx, %rcx
|
||||
jb HIDDEN_JUMPTARGET (__chk_fail)
|
||||
END (__memmove_chk_erms)
|
||||
|
||||
ENTRY (__memmove_erms)
|
||||
movq %rdi, %rax
|
||||
L(start_movsb):
|
||||
@ -185,6 +195,7 @@ L(movsb_backward):
|
||||
END (__memmove_erms)
|
||||
# if defined SHARED
|
||||
strong_alias (__memmove_erms, __memcpy_erms)
|
||||
strong_alias (__memmove_chk_erms, __memcpy_chk_erms)
|
||||
# endif
|
||||
# endif
|
||||
|
||||
|
||||
@ -1,101 +0,0 @@
|
||||
/* Multiple versions of memmove
|
||||
All versions must be listed in ifunc-impl-list.c.
|
||||
Copyright (C) 2016-2017 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<http://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <sysdep.h>
|
||||
#include <init-arch.h>
|
||||
|
||||
/* Define multiple versions only for the definition in lib and for
|
||||
DSO. */
|
||||
#if IS_IN (libc)
|
||||
.text
|
||||
ENTRY(__libc_memmove)
|
||||
.type __libc_memmove, @gnu_indirect_function
|
||||
LOAD_RTLD_GLOBAL_RO_RDX
|
||||
lea __memmove_erms(%rip), %RAX_LP
|
||||
HAS_ARCH_FEATURE (Prefer_ERMS)
|
||||
jnz 2f
|
||||
HAS_ARCH_FEATURE (Prefer_No_AVX512)
|
||||
jnz 1f
|
||||
HAS_ARCH_FEATURE (AVX512F_Usable)
|
||||
jz 1f
|
||||
lea __memmove_avx512_no_vzeroupper(%rip), %RAX_LP
|
||||
HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER)
|
||||
jnz 2f
|
||||
lea __memmove_avx512_unaligned_erms(%rip), %RAX_LP
|
||||
HAS_CPU_FEATURE (ERMS)
|
||||
jnz 2f
|
||||
lea __memmove_avx512_unaligned(%rip), %RAX_LP
|
||||
ret
|
||||
1: lea __memmove_avx_unaligned(%rip), %RAX_LP
|
||||
HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
|
||||
jz L(Fast_Unaligned_Load)
|
||||
HAS_CPU_FEATURE (ERMS)
|
||||
jz 2f
|
||||
lea __memmove_avx_unaligned_erms(%rip), %RAX_LP
|
||||
ret
|
||||
L(Fast_Unaligned_Load):
|
||||
lea __memmove_sse2_unaligned(%rip), %RAX_LP
|
||||
HAS_ARCH_FEATURE (Fast_Unaligned_Copy)
|
||||
jz L(SSSE3)
|
||||
HAS_CPU_FEATURE (ERMS)
|
||||
jz 2f
|
||||
lea __memmove_sse2_unaligned_erms(%rip), %RAX_LP
|
||||
ret
|
||||
L(SSSE3):
|
||||
HAS_CPU_FEATURE (SSSE3)
|
||||
jz 2f
|
||||
lea __memmove_ssse3_back(%rip), %RAX_LP
|
||||
HAS_ARCH_FEATURE (Fast_Copy_Backward)
|
||||
jnz 2f
|
||||
lea __memmove_ssse3(%rip), %RAX_LP
|
||||
2: ret
|
||||
END(__libc_memmove)
|
||||
#endif
|
||||
|
||||
#if IS_IN (libc)
|
||||
# define MEMMOVE_SYMBOL(p,s) p##_sse2_##s
|
||||
|
||||
# ifdef SHARED
|
||||
libc_hidden_ver (__memmove_sse2_unaligned, memmove)
|
||||
libc_hidden_ver (__memcpy_sse2_unaligned, memcpy)
|
||||
libc_hidden_ver (__mempcpy_sse2_unaligned, mempcpy)
|
||||
libc_hidden_ver (__mempcpy_sse2_unaligned, __mempcpy)
|
||||
|
||||
# undef libc_hidden_builtin_def
|
||||
/* It doesn't make sense to send libc-internal memmove calls through a PLT.
|
||||
The speedup we get from using SSE2 instructions is likely eaten away
|
||||
by the indirect call in the PLT. */
|
||||
# define libc_hidden_builtin_def
|
||||
# endif
|
||||
strong_alias (__libc_memmove, memmove)
|
||||
#endif
|
||||
|
||||
#if !defined SHARED || !IS_IN (libc)
|
||||
weak_alias (__mempcpy, mempcpy)
|
||||
#endif
|
||||
|
||||
#include "../memmove.S"
|
||||
|
||||
#if defined SHARED && IS_IN (libc)
|
||||
# include <shlib-compat.h>
|
||||
# if SHLIB_COMPAT (libc, GLIBC_2_2_5, GLIBC_2_14)
|
||||
/* Use __memmove_sse2_unaligned to support overlapping addresses. */
|
||||
compat_symbol (libc, __memmove_sse2_unaligned, memcpy, GLIBC_2_2_5);
|
||||
# endif
|
||||
#endif
|
||||
38
sysdeps/x86_64/multiarch/memmove.c
Normal file
38
sysdeps/x86_64/multiarch/memmove.c
Normal file
@ -0,0 +1,38 @@
|
||||
/* Multiple versions of memmmove.
|
||||
All versions must be listed in ifunc-impl-list.c.
|
||||
Copyright (C) 2016-2017 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<http://www.gnu.org/licenses/>. */
|
||||
|
||||
/* Define multiple versions only for the definition in lib and for
|
||||
DSO. */
|
||||
#if IS_IN (libc)
|
||||
# define memmove __redirect_memmove
|
||||
# include <string.h>
|
||||
# undef memmove
|
||||
|
||||
# define SYMBOL_NAME memmove
|
||||
# include "ifunc-memmove.h"
|
||||
|
||||
libc_ifunc_redirected (__redirect_memmove, __libc_memmove,
|
||||
IFUNC_SELECTOR ());
|
||||
|
||||
strong_alias (__libc_memmove, memmove);
|
||||
# ifdef SHARED
|
||||
__hidden_ver1 (__libc_memmove, __GI_memmove, __redirect_memmove)
|
||||
__attribute__ ((visibility ("hidden")));
|
||||
# endif
|
||||
#endif
|
||||
21
sysdeps/x86_64/multiarch/memmove_chk-nonshared.S
Normal file
21
sysdeps/x86_64/multiarch/memmove_chk-nonshared.S
Normal file
@ -0,0 +1,21 @@
|
||||
/* Non-shared version of memmove_chk for x86-64.
|
||||
Copyright (C) 2017 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<http://www.gnu.org/licenses/>. */
|
||||
|
||||
#if IS_IN (libc) && !defined SHARED
|
||||
# include <sysdeps/x86_64/memmove_chk.S>
|
||||
#endif
|
||||
@ -1,71 +0,0 @@
|
||||
/* Multiple versions of __memmove_chk
|
||||
All versions must be listed in ifunc-impl-list.c.
|
||||
Copyright (C) 2016-2017 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<http://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <sysdep.h>
|
||||
#include <init-arch.h>
|
||||
|
||||
/* Define multiple versions only for the definition in lib and for
|
||||
DSO. There are no multiarch memmove functions for static binaries.
|
||||
*/
|
||||
#if IS_IN (libc)
|
||||
# ifdef SHARED
|
||||
.text
|
||||
ENTRY(__memmove_chk)
|
||||
.type __memmove_chk, @gnu_indirect_function
|
||||
LOAD_RTLD_GLOBAL_RO_RDX
|
||||
HAS_ARCH_FEATURE (Prefer_No_AVX512)
|
||||
jnz 1f
|
||||
HAS_ARCH_FEATURE (AVX512F_Usable)
|
||||
jz 1f
|
||||
lea __memmove_chk_avx512_no_vzeroupper(%rip), %RAX_LP
|
||||
HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER)
|
||||
jnz 2f
|
||||
lea __memmove_chk_avx512_unaligned_erms(%rip), %RAX_LP
|
||||
HAS_CPU_FEATURE (ERMS)
|
||||
jnz 2f
|
||||
lea __memmove_chk_avx512_unaligned(%rip), %RAX_LP
|
||||
ret
|
||||
1: lea __memmove_chk_avx_unaligned(%rip), %RAX_LP
|
||||
HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
|
||||
jz L(Fast_Unaligned_Load)
|
||||
HAS_CPU_FEATURE (ERMS)
|
||||
jz 2f
|
||||
lea __memmove_chk_avx_unaligned_erms(%rip), %RAX_LP
|
||||
ret
|
||||
L(Fast_Unaligned_Load):
|
||||
lea __memmove_chk_sse2_unaligned(%rip), %RAX_LP
|
||||
HAS_ARCH_FEATURE (Fast_Unaligned_Copy)
|
||||
jz L(SSSE3)
|
||||
HAS_CPU_FEATURE (ERMS)
|
||||
jz 2f
|
||||
lea __memmove_chk_sse2_unaligned_erms(%rip), %RAX_LP
|
||||
ret
|
||||
L(SSSE3):
|
||||
HAS_CPU_FEATURE (SSSE3)
|
||||
jz 2f
|
||||
lea __memmove_chk_ssse3_back(%rip), %RAX_LP
|
||||
HAS_ARCH_FEATURE (Fast_Copy_Backward)
|
||||
jnz 2f
|
||||
lea __memmove_chk_ssse3(%rip), %RAX_LP
|
||||
2: ret
|
||||
END(__memmove_chk)
|
||||
# else
|
||||
# include "../memmove_chk.S"
|
||||
# endif
|
||||
#endif
|
||||
31
sysdeps/x86_64/multiarch/memmove_chk.c
Normal file
31
sysdeps/x86_64/multiarch/memmove_chk.c
Normal file
@ -0,0 +1,31 @@
|
||||
/* Multiple versions of __memmove_chk
|
||||
All versions must be listed in ifunc-impl-list.c.
|
||||
Copyright (C) 2017 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<http://www.gnu.org/licenses/>. */
|
||||
|
||||
/* Define multiple versions only for the definition in libc.so. */
|
||||
#if IS_IN (libc) && defined SHARED
|
||||
# define __memmove_chk __redirect_memmove_chk
|
||||
# include <string.h>
|
||||
# undef __memmove_chk
|
||||
|
||||
# define SYMBOL_NAME memmove_chk
|
||||
# include "ifunc-memmove.h"
|
||||
|
||||
libc_ifunc_redirected (__redirect_memmove_chk, __memmove_chk,
|
||||
IFUNC_SELECTOR ());
|
||||
#endif
|
||||
@ -1,73 +0,0 @@
|
||||
/* Multiple versions of mempcpy
|
||||
All versions must be listed in ifunc-impl-list.c.
|
||||
Copyright (C) 2010-2017 Free Software Foundation, Inc.
|
||||
Contributed by Intel Corporation.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<http://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <sysdep.h>
|
||||
#include <init-arch.h>
|
||||
|
||||
/* Define multiple versions only for the definition in lib and for
|
||||
DSO. In static binaries we need mempcpy before the initialization
|
||||
happened. */
|
||||
#if defined SHARED && IS_IN (libc)
|
||||
.text
|
||||
ENTRY(__mempcpy)
|
||||
.type __mempcpy, @gnu_indirect_function
|
||||
LOAD_RTLD_GLOBAL_RO_RDX
|
||||
lea __mempcpy_erms(%rip), %RAX_LP
|
||||
HAS_ARCH_FEATURE (Prefer_ERMS)
|
||||
jnz 2f
|
||||
HAS_ARCH_FEATURE (Prefer_No_AVX512)
|
||||
jnz 1f
|
||||
HAS_ARCH_FEATURE (AVX512F_Usable)
|
||||
jz 1f
|
||||
lea __mempcpy_avx512_no_vzeroupper(%rip), %RAX_LP
|
||||
HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER)
|
||||
jnz 2f
|
||||
lea __mempcpy_avx512_unaligned_erms(%rip), %RAX_LP
|
||||
HAS_CPU_FEATURE (ERMS)
|
||||
jnz 2f
|
||||
lea __mempcpy_avx512_unaligned(%rip), %RAX_LP
|
||||
ret
|
||||
1: lea __mempcpy_avx_unaligned(%rip), %RAX_LP
|
||||
HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
|
||||
jz L(Fast_Unaligned_Load)
|
||||
HAS_CPU_FEATURE (ERMS)
|
||||
jz 2f
|
||||
lea __mempcpy_avx_unaligned_erms(%rip), %RAX_LP
|
||||
ret
|
||||
L(Fast_Unaligned_Load):
|
||||
lea __mempcpy_sse2_unaligned(%rip), %RAX_LP
|
||||
HAS_ARCH_FEATURE (Fast_Unaligned_Copy)
|
||||
jz L(SSSE3)
|
||||
HAS_CPU_FEATURE (ERMS)
|
||||
jz 2f
|
||||
lea __mempcpy_sse2_unaligned_erms(%rip), %RAX_LP
|
||||
ret
|
||||
L(SSSE3):
|
||||
HAS_CPU_FEATURE (SSSE3)
|
||||
jz 2f
|
||||
lea __mempcpy_ssse3_back(%rip), %RAX_LP
|
||||
HAS_ARCH_FEATURE (Fast_Copy_Backward)
|
||||
jnz 2f
|
||||
lea __mempcpy_ssse3(%rip), %RAX_LP
|
||||
2: ret
|
||||
END(__mempcpy)
|
||||
|
||||
weak_alias (__mempcpy, mempcpy)
|
||||
#endif
|
||||
40
sysdeps/x86_64/multiarch/mempcpy.c
Normal file
40
sysdeps/x86_64/multiarch/mempcpy.c
Normal file
@ -0,0 +1,40 @@
|
||||
/* Multiple versions of mempcpy.
|
||||
All versions must be listed in ifunc-impl-list.c.
|
||||
Copyright (C) 2017 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<http://www.gnu.org/licenses/>. */
|
||||
|
||||
/* Define multiple versions only for the definition in lib and for
|
||||
DSO. In static binaries we need mempcpy before the initialization
|
||||
happened. */
|
||||
#if defined SHARED && IS_IN (libc)
|
||||
# define mempcpy __redirect_mempcpy
|
||||
# define __mempcpy __redirect___mempcpy
|
||||
# include <string.h>
|
||||
# undef mempcpy
|
||||
# undef __mempcpy
|
||||
|
||||
# define SYMBOL_NAME mempcpy
|
||||
# include "ifunc-memmove.h"
|
||||
|
||||
libc_ifunc_redirected (__redirect_mempcpy, __mempcpy, IFUNC_SELECTOR ());
|
||||
|
||||
weak_alias (__mempcpy, mempcpy)
|
||||
__hidden_ver1 (__mempcpy, __GI___mempcpy, __redirect___mempcpy)
|
||||
__attribute__ ((visibility ("hidden")));
|
||||
__hidden_ver1 (mempcpy, __GI_mempcpy, __redirect_mempcpy)
|
||||
__attribute__ ((visibility ("hidden")));
|
||||
#endif
|
||||
21
sysdeps/x86_64/multiarch/mempcpy_chk-nonshared.S
Normal file
21
sysdeps/x86_64/multiarch/mempcpy_chk-nonshared.S
Normal file
@ -0,0 +1,21 @@
|
||||
/* Non-shared version of mempcpy_chk for x86-64.
|
||||
Copyright (C) 2017 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<http://www.gnu.org/licenses/>. */
|
||||
|
||||
#if IS_IN (libc) && !defined SHARED
|
||||
# include <sysdeps/x86_64/mempcpy_chk.S>
|
||||
#endif
|
||||
@ -1,72 +0,0 @@
|
||||
/* Multiple versions of __mempcpy_chk
|
||||
All versions must be listed in ifunc-impl-list.c.
|
||||
Copyright (C) 2010-2017 Free Software Foundation, Inc.
|
||||
Contributed by Intel Corporation.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<http://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <sysdep.h>
|
||||
#include <init-arch.h>
|
||||
|
||||
/* Define multiple versions only for the definition in lib and for
|
||||
DSO. There are no multiarch mempcpy functions for static binaries.
|
||||
*/
|
||||
#if IS_IN (libc)
|
||||
# ifdef SHARED
|
||||
.text
|
||||
ENTRY(__mempcpy_chk)
|
||||
.type __mempcpy_chk, @gnu_indirect_function
|
||||
LOAD_RTLD_GLOBAL_RO_RDX
|
||||
HAS_ARCH_FEATURE (Prefer_No_AVX512)
|
||||
jnz 1f
|
||||
HAS_ARCH_FEATURE (AVX512F_Usable)
|
||||
jz 1f
|
||||
lea __mempcpy_chk_avx512_no_vzeroupper(%rip), %RAX_LP
|
||||
HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER)
|
||||
jnz 2f
|
||||
lea __mempcpy_chk_avx512_unaligned_erms(%rip), %RAX_LP
|
||||
HAS_CPU_FEATURE (ERMS)
|
||||
jnz 2f
|
||||
lea __mempcpy_chk_avx512_unaligned(%rip), %RAX_LP
|
||||
ret
|
||||
1: lea __mempcpy_chk_avx_unaligned(%rip), %RAX_LP
|
||||
HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
|
||||
jz L(Fast_Unaligned_Load)
|
||||
HAS_CPU_FEATURE (ERMS)
|
||||
jz 2f
|
||||
lea __mempcpy_chk_avx_unaligned_erms(%rip), %RAX_LP
|
||||
ret
|
||||
L(Fast_Unaligned_Load):
|
||||
lea __mempcpy_chk_sse2_unaligned(%rip), %RAX_LP
|
||||
HAS_ARCH_FEATURE (Fast_Unaligned_Copy)
|
||||
jz L(SSSE3)
|
||||
HAS_CPU_FEATURE (ERMS)
|
||||
jz 2f
|
||||
lea __mempcpy_chk_sse2_unaligned_erms(%rip), %RAX_LP
|
||||
ret
|
||||
L(SSSE3):
|
||||
HAS_CPU_FEATURE (SSSE3)
|
||||
jz 2f
|
||||
lea __mempcpy_chk_ssse3_back(%rip), %RAX_LP
|
||||
HAS_ARCH_FEATURE (Fast_Copy_Backward)
|
||||
jnz 2f
|
||||
lea __mempcpy_chk_ssse3(%rip), %RAX_LP
|
||||
2: ret
|
||||
END(__mempcpy_chk)
|
||||
# else
|
||||
# include "../mempcpy_chk.S"
|
||||
# endif
|
||||
#endif
|
||||
31
sysdeps/x86_64/multiarch/mempcpy_chk.c
Normal file
31
sysdeps/x86_64/multiarch/mempcpy_chk.c
Normal file
@ -0,0 +1,31 @@
|
||||
/* Multiple versions of __mempcpy_chk
|
||||
All versions must be listed in ifunc-impl-list.c.
|
||||
Copyright (C) 2017 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<http://www.gnu.org/licenses/>. */
|
||||
|
||||
/* Define multiple versions only for the definition in libc.so. */
|
||||
#if IS_IN (libc) && defined SHARED
|
||||
# define __mempcpy_chk __redirect_mempcpy_chk
|
||||
# include <string.h>
|
||||
# undef __mempcpy_chk
|
||||
|
||||
# define SYMBOL_NAME mempcpy_chk
|
||||
# include "ifunc-memmove.h"
|
||||
|
||||
libc_ifunc_redirected (__redirect_mempcpy_chk, __mempcpy_chk,
|
||||
IFUNC_SELECTOR ());
|
||||
#endif
|
||||
Loading…
Reference in New Issue
Block a user