x86: Optimize memchr-evex.S and implement with VMM headers

Optimizations are:

1. Use the fact that tzcnt(0) -> VEC_SIZE for memchr to save a branch
   in short string case.
2. Restructure code so that small strings are given the hot path.
	- This is a net-zero on the benchmark suite but in general makes
      sense as smaller sizes are far more common.
3. Use more code-size efficient instructions.
	- tzcnt ...     -> bsf ...
	- vpcmpb $0 ... -> vpcmpeq ...
4. Align labels less aggressively, especially if it doesn't save fetch
   blocks / causes the basic-block to span extra cache-lines.

The optimizations (especially for point 2) make the memchr and
rawmemchr code essentially incompatible so split rawmemchr-evex
to a new file.

Code Size Changes:
memchr-evex.S       : -107 bytes
rawmemchr-evex.S    :  -53 bytes

Net perf changes:

Reported as geometric mean of all improvements / regressions from N=10
runs of the benchtests. Value as New Time / Old Time so < 1.0 is
improvement and 1.0 is regression.

memchr-evex.S       : 0.928
rawmemchr-evex.S    : 0.986 (Less targets cross cache lines)

Full results attached in email.

Full check passes on x86-64.

sysdeps/x86_64/multiarch/rawmemchr-evex-rtm.S

@@ -1,3 +1,6 @@
-#define MEMCHR __rawmemchr_evex_rtm
+#define RAWMEMCHR	__rawmemchr_evex_rtm
-#define USE_AS_RAWMEMCHR 1
+
-#include "memchr-evex-rtm.S"
+#define USE_IN_RTM	1
+#define SECTION(p)	p##.evex.rtm
+
+#include "rawmemchr-evex.S"

sysdeps/x86_64/multiarch/rawmemchr-evex.S

@@ -1,7 +1,308 @@
-#ifndef RAWMEMCHR
+/* rawmemchr optimized with 256-bit EVEX instructions.
-# define RAWMEMCHR	__rawmemchr_evex
+   Copyright (C) 2022 Free Software Foundation, Inc.
-#endif
+   This file is part of the GNU C Library.
-#define USE_AS_RAWMEMCHR	1
-#define MEMCHR	RAWMEMCHR
 
-#include "memchr-evex.S"
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#include <isa-level.h>
+#include <sysdep.h>
+
+#if ISA_SHOULD_BUILD (4)
+
+# ifndef VEC_SIZE
+#  include "x86-evex256-vecs.h"
+# endif
+
+# ifndef RAWMEMCHR
+#  define RAWMEMCHR	__rawmemchr_evex
+# endif
+
+
+# define PC_SHIFT_GPR	rdi
+# define REG_WIDTH	VEC_SIZE
+# define VPTESTN	vptestnmb
+# define VPBROADCAST	vpbroadcastb
+# define VPMINU	vpminub
+# define VPCMP	vpcmpb
+# define VPCMPEQ	vpcmpeqb
+# define CHAR_SIZE	1
+
+# include "reg-macros.h"
+
+/* If not in an RTM and VEC_SIZE != 64 (the VEC_SIZE = 64
+   doesn't have VEX encoding), use VEX encoding in loop so we
+   can use vpcmpeqb + vptern which is more efficient than the
+   EVEX alternative.  */
+# if defined USE_IN_RTM || VEC_SIZE == 64
+#  undef COND_VZEROUPPER
+#  undef VZEROUPPER_RETURN
+#  undef VZEROUPPER
+
+
+#  define COND_VZEROUPPER
+#  define VZEROUPPER_RETURN	ret
+#  define VZEROUPPER
+
+#  define USE_TERN_IN_LOOP	0
+# else
+#  define USE_TERN_IN_LOOP	1
+#  undef VZEROUPPER
+#  define VZEROUPPER	vzeroupper
+# endif
+
+# define CHAR_PER_VEC	VEC_SIZE
+
+# if CHAR_PER_VEC == 64
+
+#  define TAIL_RETURN_LBL	first_vec_x2
+#  define TAIL_RETURN_OFFSET	(CHAR_PER_VEC * 2)
+
+#  define FALLTHROUGH_RETURN_LBL	first_vec_x3
+#  define FALLTHROUGH_RETURN_OFFSET	(CHAR_PER_VEC * 3)
+
+# else	/* !(CHAR_PER_VEC == 64) */
+
+#  define TAIL_RETURN_LBL	first_vec_x3
+#  define TAIL_RETURN_OFFSET	(CHAR_PER_VEC * 3)
+
+#  define FALLTHROUGH_RETURN_LBL	first_vec_x2
+#  define FALLTHROUGH_RETURN_OFFSET	(CHAR_PER_VEC * 2)
+# endif	/* !(CHAR_PER_VEC == 64) */
+
+
+# define VMATCH	VMM(0)
+# define VMATCH_LO	VMM_lo(0)
+
+# define PAGE_SIZE	4096
+
+	.section SECTION(.text), "ax", @progbits
+ENTRY_P2ALIGN (RAWMEMCHR, 6)
+	VPBROADCAST %esi, %VMATCH
+	/* Check if we may cross page boundary with one vector load.  */
+	movl	%edi, %eax
+	andl	$(PAGE_SIZE - 1), %eax
+	cmpl	$(PAGE_SIZE - VEC_SIZE), %eax
+	ja	L(page_cross)
+
+	VPCMPEQ	(%rdi), %VMATCH, %k0
+	KMOV	%k0, %VRAX
+
+	test	%VRAX, %VRAX
+	jz	L(aligned_more)
+L(first_vec_x0):
+	bsf	%VRAX, %VRAX
+	addq	%rdi, %rax
+	ret
+
+	.p2align 4,, 4
+L(first_vec_x4):
+	bsf	%VRAX, %VRAX
+	leaq	(VEC_SIZE * 4)(%rdi, %rax), %rax
+	ret
+
+	/* For VEC_SIZE == 32 we can fit this in aligning bytes so might
+	   as well place it more locally.  For VEC_SIZE == 64 we reuse
+	   return code at the end of loop's return.  */
+# if VEC_SIZE == 32
+	.p2align 4,, 4
+L(FALLTHROUGH_RETURN_LBL):
+	bsf	%VRAX, %VRAX
+	leaq	(FALLTHROUGH_RETURN_OFFSET)(%rdi, %rax), %rax
+	ret
+# endif
+
+	.p2align 4,, 6
+L(page_cross):
+	/* eax has lower page-offset bits of rdi so xor will zero them
+	   out.  */
+	xorq	%rdi, %rax
+	VPCMPEQ	(PAGE_SIZE - VEC_SIZE)(%rax), %VMATCH, %k0
+	KMOV	%k0, %VRAX
+
+	/* Shift out out-of-bounds matches.  */
+	shrx	%VRDI, %VRAX, %VRAX
+	test	%VRAX, %VRAX
+	jnz	L(first_vec_x0)
+
+	.p2align 4,, 10
+L(aligned_more):
+L(page_cross_continue):
+	/* Align pointer.  */
+	andq	$(VEC_SIZE * -1), %rdi
+
+	VPCMPEQ	VEC_SIZE(%rdi), %VMATCH, %k0
+	KMOV	%k0, %VRAX
+	test	%VRAX, %VRAX
+	jnz	L(first_vec_x1)
+
+	VPCMPEQ	(VEC_SIZE * 2)(%rdi), %VMATCH, %k0
+	KMOV	%k0, %VRAX
+	test	%VRAX, %VRAX
+	jnz	L(first_vec_x2)
+
+	VPCMPEQ	(VEC_SIZE * 3)(%rdi), %VMATCH, %k0
+	KMOV	%k0, %VRAX
+	test	%VRAX, %VRAX
+	jnz	L(first_vec_x3)
+
+	VPCMPEQ	(VEC_SIZE * 4)(%rdi), %VMATCH, %k0
+	KMOV	%k0, %VRAX
+	test	%VRAX, %VRAX
+	jnz	L(first_vec_x4)
+
+	subq	$-(VEC_SIZE * 1), %rdi
+# if VEC_SIZE == 64
+	/* Saves code size.  No evex512 processor has partial register
+	   stalls.  If that change this can be replaced with `andq
+	   $-(VEC_SIZE * 4), %rdi`.  */
+	xorb	%dil, %dil
+# else
+	andq	$-(VEC_SIZE * 4), %rdi
+# endif
+
+# if USE_TERN_IN_LOOP
+	/* copy VMATCH to low ymm so we can use vpcmpeq which is not
+	   encodable with EVEX registers.  NB: this is VEC_SIZE == 32
+	   only as there is no way to encode vpcmpeq with zmm0-15.  */
+	vmovdqa64 %VMATCH, %VMATCH_LO
+# endif
+
+	.p2align 4
+L(loop_4x_vec):
+	/* Two versions of the loop.  One that does not require
+	   vzeroupper by not using ymm0-15 and another does that
+	   require vzeroupper because it uses ymm0-15.  The reason why
+	   ymm0-15 is used at all is because there is no EVEX encoding
+	   vpcmpeq and with vpcmpeq this loop can be performed more
+	   efficiently.  The non-vzeroupper version is safe for RTM
+	   while the vzeroupper version should be prefered if RTM are
+	   not supported.   Which loop version we use is determined by
+	   USE_TERN_IN_LOOP.  */
+
+# if USE_TERN_IN_LOOP
+	/* Since vptern can only take 3x vectors fastest to do 1 vec
+	   seperately with EVEX vpcmp.  */
+	VPCMPEQ	(VEC_SIZE * 4)(%rdi), %VMATCH, %k1
+	/* Compare 3x with vpcmpeq and or them all together with vptern.
+	 */
+
+	VPCMPEQ	(VEC_SIZE * 5)(%rdi), %VMATCH_LO, %VMM_lo(2)
+	subq	$(VEC_SIZE * -4), %rdi
+	VPCMPEQ	(VEC_SIZE * 2)(%rdi), %VMATCH_LO, %VMM_lo(3)
+	VPCMPEQ	(VEC_SIZE * 3)(%rdi), %VMATCH_LO, %VMM_lo(4)
+
+	/* 254 is mask for oring VEC_lo(2), VEC_lo(3), VEC_lo(4) into
+	   VEC_lo(4).  */
+	vpternlogd $254, %VMM_lo(2), %VMM_lo(3), %VMM_lo(4)
+	vpmovmskb %VMM_lo(4), %VRCX
+
+	KMOV	%k1, %eax
+
+	/* NB:  rax has match from first VEC and rcx has matches from
+	   VEC 2-4.  If rax is non-zero we will return that match.  If
+	   rax is zero adding won't disturb the bits in rcx.  */
+	add	%rax, %rcx
+# else
+	/* Loop version that uses EVEX encoding.  */
+	VPCMP	$4, (VEC_SIZE * 4)(%rdi), %VMATCH, %k1
+	vpxorq	(VEC_SIZE * 5)(%rdi), %VMATCH, %VMM(2)
+	vpxorq	(VEC_SIZE * 6)(%rdi), %VMATCH, %VMM(3)
+	VPCMPEQ	(VEC_SIZE * 7)(%rdi), %VMATCH, %k3
+	VPMINU	%VMM(2), %VMM(3), %VMM(3){%k1}{z}
+	VPTESTN	%VMM(3), %VMM(3), %k2
+	subq	$(VEC_SIZE * -4), %rdi
+	KORTEST %k2, %k3
+# endif
+	jz	L(loop_4x_vec)
+
+# if USE_TERN_IN_LOOP
+	test	%VRAX, %VRAX
+# else
+	KMOV	%k1, %VRAX
+	inc	%VRAX
+# endif
+	jnz	L(last_vec_x0)
+
+
+# if USE_TERN_IN_LOOP
+	vpmovmskb %VMM_lo(2), %VRAX
+# else
+	VPTESTN	%VMM(2), %VMM(2), %k1
+	KMOV	%k1, %VRAX
+# endif
+	test	%VRAX, %VRAX
+	jnz	L(last_vec_x1)
+
+
+# if USE_TERN_IN_LOOP
+	vpmovmskb %VMM_lo(3), %VRAX
+# else
+	KMOV	%k2, %VRAX
+# endif
+
+	/* No longer need any of the lo vecs (ymm0-15) so vzeroupper
+	   (only if used VEX encoded loop).  */
+	COND_VZEROUPPER
+
+	/* Seperate logic for VEC_SIZE == 64 and VEC_SIZE == 32 for
+	   returning last 2x VEC. For VEC_SIZE == 64 we test each VEC
+	   individually, for VEC_SIZE == 32 we combine them in a single
+	   64-bit GPR.  */
+# if CHAR_PER_VEC == 64
+#  if USE_TERN_IN_LOOP
+#   error "Unsupported"
+#  endif
+
+
+	/* If CHAR_PER_VEC == 64 we can't combine the last two VEC.  */
+	test	%VRAX, %VRAX
+	jnz	L(first_vec_x2)
+	KMOV	%k3, %VRAX
+L(FALLTHROUGH_RETURN_LBL):
+# else
+	/* CHAR_PER_VEC <= 32 so we can combine the results from the
+	   last 2x VEC.  */
+#  if !USE_TERN_IN_LOOP
+	KMOV	%k3, %VRCX
+#  endif
+	salq	$CHAR_PER_VEC, %rcx
+	addq	%rcx, %rax
+# endif
+	bsf	%rax, %rax
+	leaq	(FALLTHROUGH_RETURN_OFFSET)(%rdi, %rax), %rax
+	ret
+
+	.p2align 4,, 8
+L(TAIL_RETURN_LBL):
+	bsf	%rax, %rax
+	leaq	(TAIL_RETURN_OFFSET)(%rdi, %rax), %rax
+	ret
+
+	.p2align 4,, 8
+L(last_vec_x1):
+	COND_VZEROUPPER
+L(first_vec_x1):
+	bsf	%VRAX, %VRAX
+	leaq	(VEC_SIZE * 1)(%rdi, %rax), %rax
+	ret
+
+	.p2align 4,, 8
+L(last_vec_x0):
+	COND_VZEROUPPER
+	bsf	%VRAX, %VRAX
+	addq	%rdi, %rax
+	ret
+END (RAWMEMCHR)
+#endif