| /* |
| * strchr - find a character in a string |
| * |
| * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| * See https://llvm.org/LICENSE.txt for license information. |
| * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| */ |
| |
| /* Assumptions: |
| * |
| * ARMv8-a, AArch64 |
| * Neon Available. |
| */ |
| |
| #include "../asmdefs.h" |
| |
| /* Arguments and results. */ |
| #define srcin x0 |
| #define chrin w1 |
| |
| #define result x0 |
| |
| #define src x2 |
| #define tmp1 x3 |
| #define wtmp2 w4 |
| #define tmp3 x5 |
| |
| #define vrepchr v0 |
| #define qdata q1 |
| #define vdata v1 |
| #define vhas_nul v2 |
| #define vhas_chr v3 |
| #define vrepmask_0 v4 |
| #define vrepmask_c v5 |
| #define vend v6 |
| |
| #define L(l) .L ## l |
| |
| /* Core algorithm. |
| |
| For each 16-byte chunk we calculate a 64-bit syndrome value, with |
| four bits per byte (LSB is always in bits 0 and 1, for both big |
| and little-endian systems). For each tuple, bit 0 is set if |
| the relevant byte matched the requested character; bit 1 is set |
| if the relevant byte matched the NUL end of string (we trigger |
| off bit0 for the special case of looking for NUL) and bits 2 and 3 |
| are not used. |
| Since the bits in the syndrome reflect exactly the order in which |
| things occur in the original string a count_trailing_zeros() |
| operation will identify exactly which byte is causing the termination, |
| and why. */ |
| |
| /* Locals and temporaries. */ |
| |
| ENTRY(__strchr_aarch64_mte) |
| /* Magic constant 0x10011001 to allow us to identify which lane |
| matches the requested byte. Magic constant 0x20022002 used |
| similarly for NUL termination. */ |
| mov wtmp2, #0x1001 |
| movk wtmp2, #0x1001, lsl #16 |
| dup vrepchr.16b, chrin |
| bic src, srcin, #15 /* Work with aligned 16-byte chunks. */ |
| dup vrepmask_c.4s, wtmp2 |
| ands tmp1, srcin, #15 |
| add vrepmask_0.4s, vrepmask_c.4s, vrepmask_c.4s /* equiv: lsl #1 */ |
| b.eq L(loop) |
| |
| /* Input string is not 16-byte aligned. Rather than forcing |
| the padding bytes to a safe value, we calculate the syndrome |
| for all the bytes, but then mask off those bits of the |
| syndrome that are related to the padding. */ |
| ldr qdata, [src], #16 |
| cmeq vhas_nul.16b, vdata.16b, #0 |
| cmeq vhas_chr.16b, vdata.16b, vrepchr.16b |
| and vhas_nul.16b, vhas_nul.16b, vrepmask_0.16b |
| and vhas_chr.16b, vhas_chr.16b, vrepmask_c.16b |
| lsl tmp1, tmp1, #2 |
| orr vend.16b, vhas_nul.16b, vhas_chr.16b |
| mov tmp3, #~0 |
| addp vend.16b, vend.16b, vend.16b /* 128->64 */ |
| lsl tmp1, tmp3, tmp1 |
| |
| mov tmp3, vend.d[0] |
| ands tmp1, tmp3, tmp1 /* Mask padding bits. */ |
| b.ne L(tail) |
| |
| L(loop): |
| ldr qdata, [src], #32 |
| cmeq vhas_nul.16b, vdata.16b, #0 |
| cmeq vhas_chr.16b, vdata.16b, vrepchr.16b |
| /* Use a fast check for the termination condition. */ |
| orr vend.16b, vhas_nul.16b, vhas_chr.16b |
| addp vend.16b, vend.16b, vend.16b /* 128->64 */ |
| mov tmp1, vend.d[0] |
| cbnz tmp1, L(end) |
| |
| ldr qdata, [src, #-16] |
| cmeq vhas_nul.16b, vdata.16b, #0 |
| cmeq vhas_chr.16b, vdata.16b, vrepchr.16b |
| /* Use a fast check for the termination condition. */ |
| orr vend.16b, vhas_nul.16b, vhas_chr.16b |
| addp vend.16b, vend.16b, vend.16b /* 128->64 */ |
| mov tmp1, vend.d[0] |
| cbz tmp1, L(loop) |
| |
| /* Adjust src for next two subtractions. */ |
| add src, src, #16 |
| L(end): |
| /* Termination condition found. Now need to establish exactly why |
| we terminated. */ |
| and vhas_nul.16b, vhas_nul.16b, vrepmask_0.16b |
| and vhas_chr.16b, vhas_chr.16b, vrepmask_c.16b |
| sub src, src, #16 |
| orr vend.16b, vhas_nul.16b, vhas_chr.16b |
| addp vend.16b, vend.16b, vend.16b /* 128->64 */ |
| |
| mov tmp1, vend.d[0] |
| L(tail): |
| /* Count the trailing zeros, by bit reversing... */ |
| rbit tmp1, tmp1 |
| /* Re-bias source. */ |
| sub src, src, #16 |
| clz tmp1, tmp1 /* And counting the leading zeros. */ |
| /* Tmp1 is even if the target character was found first. Otherwise |
| we've found the end of string and we weren't looking for NUL. */ |
| tst tmp1, #1 |
| add result, src, tmp1, lsr #2 |
| csel result, result, xzr, eq |
| ret |
| |
| END(__strchr_aarch64_mte) |