qiurui
/
glibc
mirror of git://sourceware.org/git/glibc.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

x86: Optimize strlen-evex.S 

No bug. This commit optimizes strlen-evex.S. The
optimizations are mostly small things but they add up to roughly
10-30% performance improvement for strlen. The results for strnlen are
bit more ambiguous. test-strlen, test-strnlen, test-wcslen, and
test-wcsnlen are all passing.

Signed-off-by: Noah Goldstein <goldstein.w.n@gmail.com>
(cherry picked from commit 4ba6558684)
This commit is contained in:
Noah Goldstein 2021-04-19 19:36:06 -04:00 committed by H.J. Lu
parent 937f2c783a
commit 489006c3c5
1 changed files with 352 additions and 299 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

651
sysdeps/x86_64/multiarch/strlen-evex.S
View File

@ -29,11 +29,13 @@
# ifdef USE_AS_WCSLEN # ifdef USE_AS_WCSLEN
# define VPCMP vpcmpd # define VPCMP vpcmpd
# define VPMINU vpminud # define VPMINU vpminud
# define SHIFT_REG r9d # define SHIFT_REG ecx
# define CHAR_SIZE 4
# else # else
# define VPCMP vpcmpb # define VPCMP vpcmpb
# define VPMINU vpminub # define VPMINU vpminub
# define SHIFT_REG ecx # define SHIFT_REG edx
# define CHAR_SIZE 1
# endif # endif
# define XMMZERO xmm16 # define XMMZERO xmm16
@ -46,132 +48,165 @@
# define YMM6 ymm22 # define YMM6 ymm22
# define VEC_SIZE 32 # define VEC_SIZE 32
# define PAGE_SIZE 4096
# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE)
.section .text.evex,"ax",@progbits .section .text.evex,"ax",@progbits
ENTRY (STRLEN) ENTRY (STRLEN)
# ifdef USE_AS_STRNLEN # ifdef USE_AS_STRNLEN
/* Check for zero length. */ /* Check zero length. */
test %RSI_LP, %RSI_LP test %RSI_LP, %RSI_LP
jz L(zero) jz L(zero)
# ifdef USE_AS_WCSLEN # ifdef __ILP32__
shl $2, %RSI_LP
# elif defined __ILP32__
/* Clear the upper 32 bits. */ /* Clear the upper 32 bits. */
movl %esi, %esi movl %esi, %esi
# endif # endif
mov %RSI_LP, %R8_LP mov %RSI_LP, %R8_LP
# endif # endif
movl %edi, %ecx movl %edi, %eax
movq %rdi, %rdx
vpxorq %XMMZERO, %XMMZERO, %XMMZERO vpxorq %XMMZERO, %XMMZERO, %XMMZERO
/* Clear high bits from edi. Only keeping bits relevant to page
cross check. */
andl $(PAGE_SIZE - 1), %eax
/* Check if we may cross page boundary with one vector load. */ /* Check if we may cross page boundary with one vector load. */
andl $(2 * VEC_SIZE - 1), %ecx cmpl $(PAGE_SIZE - VEC_SIZE), %eax
cmpl $VEC_SIZE, %ecx ja L(cross_page_boundary)
ja L(cros_page_boundary)
/* Check the first VEC_SIZE bytes. Each bit in K0 represents a /* Check the first VEC_SIZE bytes. Each bit in K0 represents a
null byte. */ null byte. */
VPCMP $0, (%rdi), %YMMZERO, %k0 VPCMP $0, (%rdi), %YMMZERO, %k0
kmovd %k0, %eax kmovd %k0, %eax
testl %eax, %eax
# ifdef USE_AS_STRNLEN # ifdef USE_AS_STRNLEN
jnz L(first_vec_x0_check) /* If length < CHAR_PER_VEC handle special. */
/* Adjust length and check the end of data. */ cmpq $CHAR_PER_VEC, %rsi
subq $VEC_SIZE, %rsi jbe L(first_vec_x0)
jbe L(max)
# else
jnz L(first_vec_x0)
# endif # endif
/* Align data for aligned loads in the loop. */
addq $VEC_SIZE, %rdi
andl $(VEC_SIZE - 1), %ecx
andq $-VEC_SIZE, %rdi
# ifdef USE_AS_STRNLEN
/* Adjust length. */
addq %rcx, %rsi
subq $(VEC_SIZE * 4), %rsi
jbe L(last_4x_vec_or_less)
# endif
jmp L(more_4x_vec)
.p2align 4
L(cros_page_boundary):
andl $(VEC_SIZE - 1), %ecx
andq $-VEC_SIZE, %rdi
# ifdef USE_AS_WCSLEN
/* NB: Divide shift count by 4 since each bit in K0 represent 4
bytes. */
movl %ecx, %SHIFT_REG
sarl $2, %SHIFT_REG
# endif
VPCMP $0, (%rdi), %YMMZERO, %k0
kmovd %k0, %eax
/* Remove the leading bytes. */
sarxl %SHIFT_REG, %eax, %eax
testl %eax, %eax testl %eax, %eax
jz L(aligned_more) jz L(aligned_more)
tzcntl %eax, %eax tzcntl %eax, %eax
# ifdef USE_AS_WCSLEN ret
/* NB: Multiply wchar_t count by 4 to get the number of bytes. */
sall $2, %eax
# endif
# ifdef USE_AS_STRNLEN # ifdef USE_AS_STRNLEN
/* Check the end of data. */ L(zero):
cmpq %rax, %rsi xorl %eax, %eax
jbe L(max) ret
.p2align 4
L(first_vec_x0):
/* Set bit for max len so that tzcnt will return min of max len
and position of first match. */
btsq %rsi, %rax
tzcntl %eax, %eax
ret
# endif # endif
addq %rdi, %rax
addq %rcx, %rax .p2align 4
subq %rdx, %rax L(first_vec_x1):
# ifdef USE_AS_WCSLEN tzcntl %eax, %eax
shrq $2, %rax /* Safe to use 32 bit instructions as these are only called for
size = [1, 159]. */
# ifdef USE_AS_STRNLEN
/* Use ecx which was computed earlier to compute correct value.
*/
leal -(CHAR_PER_VEC * 4 + 1)(%rcx, %rax), %eax
# else
subl %edx, %edi
# ifdef USE_AS_WCSLEN
/* NB: Divide bytes by 4 to get the wchar_t count. */
sarl $2, %edi
# endif
leal CHAR_PER_VEC(%rdi, %rax), %eax
# endif # endif
ret ret
.p2align 4 .p2align 4
L(first_vec_x2):
tzcntl %eax, %eax
/* Safe to use 32 bit instructions as these are only called for
size = [1, 159]. */
# ifdef USE_AS_STRNLEN
/* Use ecx which was computed earlier to compute correct value.
*/
leal -(CHAR_PER_VEC * 3 + 1)(%rcx, %rax), %eax
# else
subl %edx, %edi
# ifdef USE_AS_WCSLEN
/* NB: Divide bytes by 4 to get the wchar_t count. */
sarl $2, %edi
# endif
leal (CHAR_PER_VEC * 2)(%rdi, %rax), %eax
# endif
ret
.p2align 4
L(first_vec_x3):
tzcntl %eax, %eax
/* Safe to use 32 bit instructions as these are only called for
size = [1, 159]. */
# ifdef USE_AS_STRNLEN
/* Use ecx which was computed earlier to compute correct value.
*/
leal -(CHAR_PER_VEC * 2 + 1)(%rcx, %rax), %eax
# else
subl %edx, %edi
# ifdef USE_AS_WCSLEN
/* NB: Divide bytes by 4 to get the wchar_t count. */
sarl $2, %edi
# endif
leal (CHAR_PER_VEC * 3)(%rdi, %rax), %eax
# endif
ret
.p2align 4
L(first_vec_x4):
tzcntl %eax, %eax
/* Safe to use 32 bit instructions as these are only called for
size = [1, 159]. */
# ifdef USE_AS_STRNLEN
/* Use ecx which was computed earlier to compute correct value.
*/
leal -(CHAR_PER_VEC + 1)(%rcx, %rax), %eax
# else
subl %edx, %edi
# ifdef USE_AS_WCSLEN
/* NB: Divide bytes by 4 to get the wchar_t count. */
sarl $2, %edi
# endif
leal (CHAR_PER_VEC * 4)(%rdi, %rax), %eax
# endif
ret
.p2align 5
L(aligned_more): L(aligned_more):
# ifdef USE_AS_STRNLEN movq %rdi, %rdx
/* "rcx" is less than VEC_SIZE. Calculate "rdx + rcx - VEC_SIZE" /* Align data to VEC_SIZE. */
with "rdx - (VEC_SIZE - rcx)" instead of "(rdx + rcx) - VEC_SIZE" andq $-(VEC_SIZE), %rdi
to void possible addition overflow. */ L(cross_page_continue):
negq %rcx
addq $VEC_SIZE, %rcx
/* Check the end of data. */
subq %rcx, %rsi
jbe L(max)
# endif
addq $VEC_SIZE, %rdi
# ifdef USE_AS_STRNLEN
subq $(VEC_SIZE * 4), %rsi
jbe L(last_4x_vec_or_less)
# endif
L(more_4x_vec):
/* Check the first 4 * VEC_SIZE. Only one VEC_SIZE at a time /* Check the first 4 * VEC_SIZE. Only one VEC_SIZE at a time
since data is only aligned to VEC_SIZE. */ since data is only aligned to VEC_SIZE. */
VPCMP $0, (%rdi), %YMMZERO, %k0 # ifdef USE_AS_STRNLEN
kmovd %k0, %eax /* + CHAR_SIZE because it simplies the logic in
testl %eax, %eax last_4x_vec_or_less. */
jnz L(first_vec_x0) leaq (VEC_SIZE * 5 + CHAR_SIZE)(%rdi), %rcx
subq %rdx, %rcx
# ifdef USE_AS_WCSLEN
/* NB: Divide bytes by 4 to get the wchar_t count. */
sarl $2, %ecx
# endif
# endif
/* Load first VEC regardless. */
VPCMP $0, VEC_SIZE(%rdi), %YMMZERO, %k0 VPCMP $0, VEC_SIZE(%rdi), %YMMZERO, %k0
# ifdef USE_AS_STRNLEN
/* Adjust length. If near end handle specially. */
subq %rcx, %rsi
jb L(last_4x_vec_or_less)
# endif
kmovd %k0, %eax kmovd %k0, %eax
testl %eax, %eax testl %eax, %eax
jnz L(first_vec_x1) jnz L(first_vec_x1)
VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMZERO, %k0 VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMZERO, %k0
kmovd %k0, %eax kmovd %k0, %eax
testl %eax, %eax test %eax, %eax
jnz L(first_vec_x2) jnz L(first_vec_x2)
VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMZERO, %k0 VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMZERO, %k0
@ -179,258 +214,276 @@ L(more_4x_vec):
testl %eax, %eax testl %eax, %eax
jnz L(first_vec_x3) jnz L(first_vec_x3)
addq $(VEC_SIZE * 4), %rdi VPCMP $0, (VEC_SIZE * 4)(%rdi), %YMMZERO, %k0
kmovd %k0, %eax
testl %eax, %eax
jnz L(first_vec_x4)
addq $VEC_SIZE, %rdi
# ifdef USE_AS_STRNLEN # ifdef USE_AS_STRNLEN
subq $(VEC_SIZE * 4), %rsi /* Check if at last VEC_SIZE * 4 length. */
jbe L(last_4x_vec_or_less) cmpq $(CHAR_PER_VEC * 4 - 1), %rsi
# endif jbe L(last_4x_vec_or_less_load)
movl %edi, %ecx
/* Align data to 4 * VEC_SIZE. */ andl $(VEC_SIZE * 4 - 1), %ecx
movq %rdi, %rcx # ifdef USE_AS_WCSLEN
andl $(4 * VEC_SIZE - 1), %ecx /* NB: Divide bytes by 4 to get the wchar_t count. */
andq $-(4 * VEC_SIZE), %rdi sarl $2, %ecx
# endif
# ifdef USE_AS_STRNLEN /* Readjust length. */
/* Adjust length. */
addq %rcx, %rsi addq %rcx, %rsi
# endif # endif
/* Align data to VEC_SIZE * 4. */
andq $-(VEC_SIZE * 4), %rdi
/* Compare 4 * VEC at a time forward. */
.p2align 4 .p2align 4
L(loop_4x_vec): L(loop_4x_vec):
/* Compare 4 * VEC at a time forward. */ /* Load first VEC regardless. */
VMOVA (%rdi), %YMM1 VMOVA (VEC_SIZE * 4)(%rdi), %YMM1
VMOVA VEC_SIZE(%rdi), %YMM2 # ifdef USE_AS_STRNLEN
VMOVA (VEC_SIZE * 2)(%rdi), %YMM3 /* Break if at end of length. */
VMOVA (VEC_SIZE * 3)(%rdi), %YMM4 subq $(CHAR_PER_VEC * 4), %rsi
jb L(last_4x_vec_or_less_cmpeq)
# endif
/* Save some code size by microfusing VPMINU with the load. Since
the matches in ymm2/ymm4 can only be returned if there where no
matches in ymm1/ymm3 respectively there is no issue with overlap.
*/
VPMINU (VEC_SIZE * 5)(%rdi), %YMM1, %YMM2
VMOVA (VEC_SIZE * 6)(%rdi), %YMM3
VPMINU (VEC_SIZE * 7)(%rdi), %YMM3, %YMM4
VPMINU %YMM1, %YMM2, %YMM5 VPCMP $0, %YMM2, %YMMZERO, %k0
VPMINU %YMM3, %YMM4, %YMM6 VPCMP $0, %YMM4, %YMMZERO, %k1
subq $-(VEC_SIZE * 4), %rdi
kortestd %k0, %k1
jz L(loop_4x_vec)
VPMINU %YMM5, %YMM6, %YMM5 /* Check if end was in first half. */
VPCMP $0, %YMM5, %YMMZERO, %k0 kmovd %k0, %eax
ktestd %k0, %k0 subq %rdx, %rdi
jnz L(4x_vec_end) # ifdef USE_AS_WCSLEN
shrq $2, %rdi
# endif
testl %eax, %eax
jz L(second_vec_return)
addq $(VEC_SIZE * 4), %rdi VPCMP $0, %YMM1, %YMMZERO, %k2
kmovd %k2, %edx
# ifndef USE_AS_STRNLEN /* Combine VEC1 matches (edx) with VEC2 matches (eax). */
jmp L(loop_4x_vec) # ifdef USE_AS_WCSLEN
sall $CHAR_PER_VEC, %eax
orl %edx, %eax
tzcntl %eax, %eax
# else # else
subq $(VEC_SIZE * 4), %rsi salq $CHAR_PER_VEC, %rax
ja L(loop_4x_vec) orq %rdx, %rax
tzcntq %rax, %rax
# endif
addq %rdi, %rax
ret
# ifdef USE_AS_STRNLEN
L(last_4x_vec_or_less_load):
/* Depending on entry adjust rdi / prepare first VEC in YMM1. */
VMOVA (VEC_SIZE * 4)(%rdi), %YMM1
L(last_4x_vec_or_less_cmpeq):
VPCMP $0, %YMM1, %YMMZERO, %k0
addq $(VEC_SIZE * 3), %rdi
L(last_4x_vec_or_less): L(last_4x_vec_or_less):
/* Less than 4 * VEC and aligned to VEC_SIZE. */
addl $(VEC_SIZE * 2), %esi
jle L(last_2x_vec)
VPCMP $0, (%rdi), %YMMZERO, %k0
kmovd %k0, %eax kmovd %k0, %eax
testl %eax, %eax /* If remaining length > VEC_SIZE * 2. This works if esi is off by
jnz L(first_vec_x0) VEC_SIZE * 4. */
testl $(CHAR_PER_VEC * 2), %esi
jnz L(last_4x_vec)
VPCMP $0, VEC_SIZE(%rdi), %YMMZERO, %k0 /* length may have been negative or positive by an offset of
kmovd %k0, %eax CHAR_PER_VEC * 4 depending on where this was called from. This
fixes that. */
andl $(CHAR_PER_VEC * 4 - 1), %esi
testl %eax, %eax testl %eax, %eax
jnz L(first_vec_x1) jnz L(last_vec_x1_check)
/* Check the end of data. */
subl $CHAR_PER_VEC, %esi
jb L(max)
VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMZERO, %k0
kmovd %k0, %eax
tzcntl %eax, %eax
/* Check the end of data. */
cmpl %eax, %esi
jb L(max)
subq %rdx, %rdi
# ifdef USE_AS_WCSLEN
/* NB: Divide bytes by 4 to get the wchar_t count. */
sarq $2, %rdi
# endif
leaq (CHAR_PER_VEC * 2)(%rdi, %rax), %rax
ret
L(max):
movq %r8, %rax
ret
# endif
/* Placed here in strnlen so that the jcc L(last_4x_vec_or_less)
in the 4x VEC loop can use 2 byte encoding. */
.p2align 4
L(second_vec_return):
VPCMP $0, %YMM3, %YMMZERO, %k0
/* Combine YMM3 matches (k0) with YMM4 matches (k1). */
# ifdef USE_AS_WCSLEN
kunpckbw %k0, %k1, %k0
kmovd %k0, %eax
tzcntl %eax, %eax
# else
kunpckdq %k0, %k1, %k0
kmovq %k0, %rax
tzcntq %rax, %rax
# endif
leaq (CHAR_PER_VEC * 2)(%rdi, %rax), %rax
ret
# ifdef USE_AS_STRNLEN
L(last_vec_x1_check):
tzcntl %eax, %eax
/* Check the end of data. */
cmpl %eax, %esi
jb L(max)
subq %rdx, %rdi
# ifdef USE_AS_WCSLEN
/* NB: Divide bytes by 4 to get the wchar_t count. */
sarq $2, %rdi
# endif
leaq (CHAR_PER_VEC)(%rdi, %rax), %rax
ret
.p2align 4
L(last_4x_vec):
/* Test first 2x VEC normally. */
testl %eax, %eax
jnz L(last_vec_x1)
VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMZERO, %k0 VPCMP $0, (VEC_SIZE * 2)(%rdi), %YMMZERO, %k0
kmovd %k0, %eax kmovd %k0, %eax
testl %eax, %eax testl %eax, %eax
jnz L(first_vec_x2_check) jnz L(last_vec_x2)
subl $VEC_SIZE, %esi
jle L(max)
/* Normalize length. */
andl $(CHAR_PER_VEC * 4 - 1), %esi
VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMZERO, %k0 VPCMP $0, (VEC_SIZE * 3)(%rdi), %YMMZERO, %k0
kmovd %k0, %eax kmovd %k0, %eax
testl %eax, %eax testl %eax, %eax
jnz L(first_vec_x3_check) jnz L(last_vec_x3)
movq %r8, %rax
/* Check the end of data. */
subl $(CHAR_PER_VEC * 3), %esi
jb L(max)
VPCMP $0, (VEC_SIZE * 4)(%rdi), %YMMZERO, %k0
kmovd %k0, %eax
tzcntl %eax, %eax
/* Check the end of data. */
cmpl %eax, %esi
jb L(max_end)
subq %rdx, %rdi
# ifdef USE_AS_WCSLEN # ifdef USE_AS_WCSLEN
shrq $2, %rax /* NB: Divide bytes by 4 to get the wchar_t count. */
sarq $2, %rdi
# endif # endif
leaq (CHAR_PER_VEC * 4)(%rdi, %rax), %rax
ret ret
.p2align 4 .p2align 4
L(last_2x_vec): L(last_vec_x1):
addl $(VEC_SIZE * 2), %esi tzcntl %eax, %eax
subq %rdx, %rdi
# ifdef USE_AS_WCSLEN
/* NB: Divide bytes by 4 to get the wchar_t count. */
sarq $2, %rdi
# endif
leaq (CHAR_PER_VEC)(%rdi, %rax), %rax
ret
.p2align 4
L(last_vec_x2):
tzcntl %eax, %eax
subq %rdx, %rdi
# ifdef USE_AS_WCSLEN
/* NB: Divide bytes by 4 to get the wchar_t count. */
sarq $2, %rdi
# endif
leaq (CHAR_PER_VEC * 2)(%rdi, %rax), %rax
ret
.p2align 4
L(last_vec_x3):
tzcntl %eax, %eax
subl $(CHAR_PER_VEC * 2), %esi
/* Check the end of data. */
cmpl %eax, %esi
jb L(max_end)
subq %rdx, %rdi
# ifdef USE_AS_WCSLEN
/* NB: Divide bytes by 4 to get the wchar_t count. */
sarq $2, %rdi
# endif
leaq (CHAR_PER_VEC * 3)(%rdi, %rax), %rax
ret
L(max_end):
movq %r8, %rax
ret
# endif
/* Cold case for crossing page with first load. */
.p2align 4
L(cross_page_boundary):
movq %rdi, %rdx
/* Align data to VEC_SIZE. */
andq $-VEC_SIZE, %rdi
VPCMP $0, (%rdi), %YMMZERO, %k0 VPCMP $0, (%rdi), %YMMZERO, %k0
kmovd %k0, %eax kmovd %k0, %eax
/* Remove the leading bytes. */
# ifdef USE_AS_WCSLEN
/* NB: Divide shift count by 4 since each bit in K0 represent 4
bytes. */
movl %edx, %ecx
shrl $2, %ecx
andl $(CHAR_PER_VEC - 1), %ecx
# endif
/* SHIFT_REG is ecx for USE_AS_WCSLEN and edx otherwise. */
sarxl %SHIFT_REG, %eax, %eax
testl %eax, %eax testl %eax, %eax
jnz L(first_vec_x0_check) # ifndef USE_AS_STRNLEN
subl $VEC_SIZE, %esi jz L(cross_page_continue)
jle L(max)
VPCMP $0, VEC_SIZE(%rdi), %YMMZERO, %k0
kmovd %k0, %eax
testl %eax, %eax
jnz L(first_vec_x1_check)
movq %r8, %rax
# ifdef USE_AS_WCSLEN
shrq $2, %rax
# endif
ret
.p2align 4
L(first_vec_x0_check):
tzcntl %eax, %eax tzcntl %eax, %eax
# ifdef USE_AS_WCSLEN ret
/* NB: Multiply wchar_t count by 4 to get the number of bytes. */ # else
sall $2, %eax jnz L(cross_page_less_vec)
# ifndef USE_AS_WCSLEN
movl %edx, %ecx
andl $(CHAR_PER_VEC - 1), %ecx
# endif # endif
movl $CHAR_PER_VEC, %eax
subl %ecx, %eax
/* Check the end of data. */ /* Check the end of data. */
cmpq %rax, %rsi cmpq %rax, %rsi
jbe L(max) ja L(cross_page_continue)
addq %rdi, %rax movl %esi, %eax
subq %rdx, %rax
# ifdef USE_AS_WCSLEN
shrq $2, %rax
# endif
ret ret
L(cross_page_less_vec):
.p2align 4
L(first_vec_x1_check):
tzcntl %eax, %eax tzcntl %eax, %eax
# ifdef USE_AS_WCSLEN /* Select min of length and position of first null. */
/* NB: Multiply wchar_t count by 4 to get the number of bytes. */
sall $2, %eax
# endif
/* Check the end of data. */
cmpq %rax, %rsi cmpq %rax, %rsi
jbe L(max) cmovb %esi, %eax
addq $VEC_SIZE, %rax
addq %rdi, %rax
subq %rdx, %rax
# ifdef USE_AS_WCSLEN
shrq $2, %rax
# endif
ret
.p2align 4
L(first_vec_x2_check):
tzcntl %eax, %eax
# ifdef USE_AS_WCSLEN
/* NB: Multiply wchar_t count by 4 to get the number of bytes. */
sall $2, %eax
# endif
/* Check the end of data. */
cmpq %rax, %rsi
jbe L(max)
addq $(VEC_SIZE * 2), %rax
addq %rdi, %rax
subq %rdx, %rax
# ifdef USE_AS_WCSLEN
shrq $2, %rax
# endif
ret
.p2align 4
L(first_vec_x3_check):
tzcntl %eax, %eax
# ifdef USE_AS_WCSLEN
/* NB: Multiply wchar_t count by 4 to get the number of bytes. */
sall $2, %eax
# endif
/* Check the end of data. */
cmpq %rax, %rsi
jbe L(max)
addq $(VEC_SIZE * 3), %rax
addq %rdi, %rax
subq %rdx, %rax
# ifdef USE_AS_WCSLEN
shrq $2, %rax
# endif
ret
.p2align 4
L(max):
movq %r8, %rax
# ifdef USE_AS_WCSLEN
shrq $2, %rax
# endif
ret
.p2align 4
L(zero):
xorl %eax, %eax
ret ret
# endif # endif
.p2align 4
L(first_vec_x0):
tzcntl %eax, %eax
# ifdef USE_AS_WCSLEN
/* NB: Multiply wchar_t count by 4 to get the number of bytes. */
sall $2, %eax
# endif
addq %rdi, %rax
subq %rdx, %rax
# ifdef USE_AS_WCSLEN
shrq $2, %rax
# endif
ret
.p2align 4
L(first_vec_x1):
tzcntl %eax, %eax
# ifdef USE_AS_WCSLEN
/* NB: Multiply wchar_t count by 4 to get the number of bytes. */
sall $2, %eax
# endif
addq $VEC_SIZE, %rax
addq %rdi, %rax
subq %rdx, %rax
# ifdef USE_AS_WCSLEN
shrq $2, %rax
# endif
ret
.p2align 4
L(first_vec_x2):
tzcntl %eax, %eax
# ifdef USE_AS_WCSLEN
/* NB: Multiply wchar_t count by 4 to get the number of bytes. */
sall $2, %eax
# endif
addq $(VEC_SIZE * 2), %rax
addq %rdi, %rax
subq %rdx, %rax
# ifdef USE_AS_WCSLEN
shrq $2, %rax
# endif
ret
.p2align 4
L(4x_vec_end):
VPCMP $0, %YMM1, %YMMZERO, %k0
kmovd %k0, %eax
testl %eax, %eax
jnz L(first_vec_x0)
VPCMP $0, %YMM2, %YMMZERO, %k1
kmovd %k1, %eax
testl %eax, %eax
jnz L(first_vec_x1)
VPCMP $0, %YMM3, %YMMZERO, %k2
kmovd %k2, %eax
testl %eax, %eax
jnz L(first_vec_x2)
VPCMP $0, %YMM4, %YMMZERO, %k3
kmovd %k3, %eax
L(first_vec_x3):
tzcntl %eax, %eax
# ifdef USE_AS_WCSLEN
/* NB: Multiply wchar_t count by 4 to get the number of bytes. */
sall $2, %eax
# endif
addq $(VEC_SIZE * 3), %rax
addq %rdi, %rax
subq %rdx, %rax
# ifdef USE_AS_WCSLEN
shrq $2, %rax
# endif
ret
END (STRLEN) END (STRLEN)
#endif #endif