AuroraMiddleware/glibc
1
0
Fork 0
mirror of https://sourceware.org/git/glibc.git synced 2025-01-07 10:00:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

AArch64: Cleanup emag memset

Browse Source 
Cleanup emag memset - merge the memset_base64.S file, remove
the unused ZVA code (since it is disabled on emag).

Reviewed-by: Adhemerval Zanella  <adhemerval.zanella@linaro.org>
(cherry picked from commit 9627ab99b5)
This commit is contained in:
Wilco Dijkstra 2023-10-26 16:34:47 +01:00
parent bfca39cce7
commit 7e999181c2
4 changed files with 90 additions and 197 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
sysdeps/aarch64/multiarch/ifunc-impl-list.c
View File

@ -62,7 +62,7 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
/* Enable this on non-falkor processors too so that other cores
can do a comparative analysis with __memset_generic. */
IFUNC_IMPL_ADD (array, i, memset, (zva_size == 64), __memset_falkor)
IFUNC_IMPL_ADD (array, i, memset, (zva_size == 64), __memset_emag)
IFUNC_IMPL_ADD (array, i, memset, 1, __memset_emag)
IFUNC_IMPL_ADD (array, i, memset, 1, __memset_kunpeng)
#if HAVE_AARCH64_SVE_ASM
IFUNC_IMPL_ADD (array, i, memset, sve && zva_size == 256, __memset_a64fx)

2
sysdeps/aarch64/multiarch/memset.c
View File

@ -56,7 +56,7 @@ select_memset_ifunc (void)
if ((IS_FALKOR (midr) || IS_PHECDA (midr)) && zva_size == 64)
return __memset_falkor;
if (IS_EMAG (midr) && zva_size == 64)
if (IS_EMAG (midr))
return __memset_emag;
return __memset_generic;

185
sysdeps/aarch64/multiarch/memset_base64.S
View File

@ -1,185 +0,0 @@
/* Copyright (C) 2018-2021 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library. If not, see
<https://www.gnu.org/licenses/>. */
#include <sysdep.h>
#include "memset-reg.h"
#ifndef MEMSET
# define MEMSET __memset_base64
#endif
/* To disable DC ZVA, set this threshold to 0. */
#ifndef DC_ZVA_THRESHOLD
# define DC_ZVA_THRESHOLD 512
#endif
/* Assumptions:
*
* ARMv8-a, AArch64, unaligned accesses
*
*/
ENTRY (MEMSET)
PTR_ARG (0)
SIZE_ARG (2)
bfi valw, valw, 8, 8
bfi valw, valw, 16, 16
bfi val, val, 32, 32
add dstend, dstin, count
cmp count, 96
b.hi L(set_long)
cmp count, 16
b.hs L(set_medium)
/* Set 0..15 bytes. */
tbz count, 3, 1f
str val, [dstin]
str val, [dstend, -8]
ret
.p2align 3
1: tbz count, 2, 2f
str valw, [dstin]
str valw, [dstend, -4]
ret
2: cbz count, 3f
strb valw, [dstin]
tbz count, 1, 3f
strh valw, [dstend, -2]
3: ret
.p2align 3
/* Set 16..96 bytes. */
L(set_medium):
stp val, val, [dstin]
tbnz count, 6, L(set96)
stp val, val, [dstend, -16]
tbz count, 5, 1f
stp val, val, [dstin, 16]
stp val, val, [dstend, -32]
1: ret
.p2align 4
/* Set 64..96 bytes. Write 64 bytes from the start and
32 bytes from the end. */
L(set96):
stp val, val, [dstin, 16]
stp val, val, [dstin, 32]
stp val, val, [dstin, 48]
stp val, val, [dstend, -32]
stp val, val, [dstend, -16]
ret
.p2align 4
L(set_long):
stp val, val, [dstin]
bic dst, dstin, 15
#if DC_ZVA_THRESHOLD
cmp count, DC_ZVA_THRESHOLD
ccmp val, 0, 0, cs
b.eq L(zva_64)
#endif
/* Small-size or non-zero memset does not use DC ZVA. */
sub count, dstend, dst
/*
* Adjust count and bias for loop. By substracting extra 1 from count,
* it is easy to use tbz instruction to check whether loop tailing
* count is less than 33 bytes, so as to bypass 2 unneccesary stps.
*/
sub count, count, 64+16+1
#if DC_ZVA_THRESHOLD
/* Align loop on 16-byte boundary, this might be friendly to i-cache. */
nop
#endif
1: stp val, val, [dst, 16]
stp val, val, [dst, 32]
stp val, val, [dst, 48]
stp val, val, [dst, 64]!
subs count, count, 64
b.hs 1b
tbz count, 5, 1f /* Remaining count is less than 33 bytes? */
stp val, val, [dst, 16]
stp val, val, [dst, 32]
1: stp val, val, [dstend, -32]
stp val, val, [dstend, -16]
ret
#if DC_ZVA_THRESHOLD
.p2align 3
L(zva_64):
stp val, val, [dst, 16]
stp val, val, [dst, 32]
stp val, val, [dst, 48]
bic dst, dst, 63
/*
* Previous memory writes might cross cache line boundary, and cause
* cache line partially dirty. Zeroing this kind of cache line using
* DC ZVA will incur extra cost, for it requires loading untouched
* part of the line from memory before zeoring.
*
* So, write the first 64 byte aligned block using stp to force
* fully dirty cache line.
*/
stp val, val, [dst, 64]
stp val, val, [dst, 80]
stp val, val, [dst, 96]
stp val, val, [dst, 112]
sub count, dstend, dst
/*
* Adjust count and bias for loop. By substracting extra 1 from count,
* it is easy to use tbz instruction to check whether loop tailing
* count is less than 33 bytes, so as to bypass 2 unneccesary stps.
*/
sub count, count, 128+64+64+1
add dst, dst, 128
nop
/* DC ZVA sets 64 bytes each time. */
1: dc zva, dst
add dst, dst, 64
subs count, count, 64
b.hs 1b
/*
* Write the last 64 byte aligned block using stp to force fully
* dirty cache line.
*/
stp val, val, [dst, 0]
stp val, val, [dst, 16]
stp val, val, [dst, 32]
stp val, val, [dst, 48]
tbz count, 5, 1f /* Remaining count is less than 33 bytes? */
stp val, val, [dst, 64]
stp val, val, [dst, 80]
1: stp val, val, [dstend, -32]
stp val, val, [dstend, -16]
ret
#endif
END (MEMSET)

98
sysdeps/aarch64/multiarch/memset_emag.S
View File

@ -18,17 +18,95 @@
<https://www.gnu.org/licenses/>. */
#include <sysdep.h>
#include "memset-reg.h"
#define MEMSET __memset_emag
/*
* Using DC ZVA to zero memory does not produce better performance if
* memory size is not very large, especially when there are multiple
* processes/threads contending memory/cache. Here we set threshold to
* zero to disable using DC ZVA, which is good for multi-process/thread
* workloads.
/* Assumptions:
*
* ARMv8-a, AArch64, unaligned accesses
*
*/
#define DC_ZVA_THRESHOLD 0
ENTRY (__memset_emag)
#include "./memset_base64.S"
PTR_ARG (0)
SIZE_ARG (2)
bfi valw, valw, 8, 8
bfi valw, valw, 16, 16
bfi val, val, 32, 32
add dstend, dstin, count
cmp count, 96
b.hi L(set_long)
cmp count, 16
b.hs L(set_medium)
/* Set 0..15 bytes. */
tbz count, 3, 1f
str val, [dstin]
str val, [dstend, -8]
ret
.p2align 3
1: tbz count, 2, 2f
str valw, [dstin]
str valw, [dstend, -4]
ret
2: cbz count, 3f
strb valw, [dstin]
tbz count, 1, 3f
strh valw, [dstend, -2]
3: ret
.p2align 3
/* Set 16..96 bytes. */
L(set_medium):
stp val, val, [dstin]
tbnz count, 6, L(set96)
stp val, val, [dstend, -16]
tbz count, 5, 1f
stp val, val, [dstin, 16]
stp val, val, [dstend, -32]
1: ret
.p2align 4
/* Set 64..96 bytes. Write 64 bytes from the start and
32 bytes from the end. */
L(set96):
stp val, val, [dstin, 16]
stp val, val, [dstin, 32]
stp val, val, [dstin, 48]
stp val, val, [dstend, -32]
stp val, val, [dstend, -16]
ret
.p2align 4
L(set_long):
stp val, val, [dstin]
bic dst, dstin, 15
/* Small-size or non-zero memset does not use DC ZVA. */
sub count, dstend, dst
/*
* Adjust count and bias for loop. By subtracting extra 1 from count,
* it is easy to use tbz instruction to check whether loop tailing
* count is less than 33 bytes, so as to bypass 2 unnecessary stps.
*/
sub count, count, 64+16+1
1: stp val, val, [dst, 16]
stp val, val, [dst, 32]
stp val, val, [dst, 48]
stp val, val, [dst, 64]!
subs count, count, 64
b.hs 1b
tbz count, 5, 1f /* Remaining count is less than 33 bytes? */
stp val, val, [dst, 16]
stp val, val, [dst, 32]
1: stp val, val, [dstend, -32]
stp val, val, [dstend, -16]
ret
END (__memset_emag)