| /* |
| * strlen - calculate the length of a string |
| * |
| * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| * See https://llvm.org/LICENSE.txt for license information. |
| * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| */ |
| |
| /* Assumptions: |
| * |
| * ARMv8-a, AArch64. |
| */ |
| |
| #include "../asmdefs.h" |
| |
| /* Arguments and results. */ |
| #define srcin x0 |
| #define len x0 |
| |
| /* Locals and temporaries. */ |
| #define src x1 |
| #define data1 x2 |
| #define data2 x3 |
| #define has_nul1 x4 |
| #define has_nul2 x5 |
| #define tmp1 x4 |
| #define tmp2 x5 |
| #define tmp3 x6 |
| #define tmp4 x7 |
| #define zeroones x8 |
| #define offset x9 |
| |
| /* NUL detection works on the principle that (X - 1) & (~X) & 0x80 |
| (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and |
| can be done in parallel across the entire word. A faster check |
| (X - 1) & 0x80 is zero for non-NUL ASCII characters, but gives |
| false hits for characters 129..255. */ |
| |
| #define REP8_01 0x0101010101010101 |
| #define REP8_7f 0x7f7f7f7f7f7f7f7f |
| |
| /* This implementation is compatible with Memory Tagging. All loads |
| are 16 bytes in size and 16 bytes aligned. This also avoids the |
| need for page boundary checks. This implementation is correct |
| even without Memory Tagging, but other implementations could be |
| more beneficial if Memory Tagging is not enabled. |
| |
| First load is aligned down and can contain bytes that are located |
| before the string. This is handled by modifying the "zeroones" |
| mask. The bytes that need to be ignored are set to zero. |
| If the string is aligned in such a way that 8 or more bytes from |
| the first load should be ignored, there is a special case |
| (skip_first_8_bytes) which only compares the second 8 bytes. |
| |
| If there is a NUL byte in the first load, we calculate the length |
| from the 2 8-byte words using conditional select to reduce branch |
| mispredictions. |
| |
| If the string is longer than 16 bytes, we check 32 bytes per |
| iteration using the fast NUL check (main_loop). If we encounter |
| non-ASCII characters, we fallback to a second loop |
| (nonascii_loop) using the full NUL check. */ |
| |
| ENTRY(__strlen_aarch64_mte) |
| bic src, srcin, 15 /* Align down to 16 bytes. */ |
| mov zeroones, REP8_01 |
| /* (offset & 63) holds number of bits to ignore in a register.*/ |
| lsl offset, srcin, 3 |
| ldp data1, data2, [src], -16 |
| lsl tmp1, zeroones, offset /* Shift (offset & 63). */ |
| #ifdef __AARCH64EB__ |
| /* For big-endian, carry propagation (if the final byte in the |
| string is 0x01) means we cannot use has_nul1/2 directly. |
| e.g. 0x0100 - 0x0101 = 0xffff, so 0x01 will be mistaken for NUL. |
| Since we expect strings to be small and early-exit, |
| byte-swap the data now so has_null1/2 will be correct. */ |
| rev data1, data1 |
| rev data2, data2 |
| #endif |
| tbnz srcin, 3, L(skip_first_8_bytes) |
| sub tmp1, data1, tmp1 |
| orr tmp2, data1, REP8_7f |
| sub tmp3, data2, zeroones |
| orr tmp4, data2, REP8_7f |
| bics has_nul1, tmp1, tmp2 |
| bic has_nul2, tmp3, tmp4 |
| /* If comparison happens, C flag is always set. */ |
| ccmp has_nul2, 0, 0, eq |
| beq L(main_loop) |
| |
| /* Enter with C = has_nul1 == 0. */ |
| csel has_nul1, has_nul1, has_nul2, cc |
| and tmp2, srcin, 7 /* Bytes to ignore. */ |
| rev has_nul1, has_nul1 |
| neg tmp2, tmp2 |
| clz tmp1, has_nul1 /* Count bits before NUL. */ |
| /* Add 8 if NUL byte is not in first register. */ |
| add tmp3, tmp2, 8 |
| csel len, tmp2, tmp3, cc |
| add len, len, tmp1, lsr 3 |
| ret |
| |
| L(skip_first_8_bytes): |
| sub tmp1, data2, tmp1 |
| orr tmp2, data2, REP8_7f |
| bics has_nul1, tmp1, tmp2 |
| beq L(main_loop) |
| |
| rev has_nul1, has_nul1 |
| lsl tmp1, has_nul1, offset /* Ignore bytes before string. */ |
| clz tmp1, tmp1 /* Count bits before NUL. */ |
| lsr len, tmp1, 3 |
| ret |
| |
| /* The inner loop processes 32 bytes per iteration and uses the fast |
| NUL check. If we encounter non-ASCII characters, use a second |
| loop with the accurate NUL check. */ |
| .p2align 4 |
| L(main_loop): |
| ldp data1, data2, [src, 32]! |
| sub tmp1, data1, zeroones |
| sub tmp3, data2, zeroones |
| orr tmp2, tmp1, tmp3 |
| tst tmp2, zeroones, lsl 7 |
| bne 1f |
| ldp data1, data2, [src, 16] |
| sub tmp1, data1, zeroones |
| sub tmp3, data2, zeroones |
| orr tmp2, tmp1, tmp3 |
| tst tmp2, zeroones, lsl 7 |
| beq L(main_loop) |
| add src, src, 16 |
| 1: |
| /* The fast check failed, so do the slower, accurate NUL check. */ |
| orr tmp2, data1, REP8_7f |
| orr tmp4, data2, REP8_7f |
| bics has_nul1, tmp1, tmp2 |
| bic has_nul2, tmp3, tmp4 |
| ccmp has_nul2, 0, 0, eq |
| beq L(nonascii_loop) |
| |
| /* Enter with C = has_nul1 == 0. */ |
| L(tail): |
| #ifdef __AARCH64EB__ |
| /* For big-endian, carry propagation (if the final byte in the |
| string is 0x01) means we cannot use has_nul1/2 directly. The |
| easiest way to get the correct byte is to byte-swap the data |
| and calculate the syndrome a second time. */ |
| csel data1, data1, data2, cc |
| rev data1, data1 |
| sub tmp1, data1, zeroones |
| orr tmp2, data1, REP8_7f |
| bic has_nul1, tmp1, tmp2 |
| #else |
| csel has_nul1, has_nul1, has_nul2, cc |
| #endif |
| sub len, src, srcin |
| rev has_nul1, has_nul1 |
| add tmp2, len, 8 |
| clz tmp1, has_nul1 |
| csel len, len, tmp2, cc |
| add len, len, tmp1, lsr 3 |
| ret |
| |
| L(nonascii_loop): |
| ldp data1, data2, [src, 16]! |
| sub tmp1, data1, zeroones |
| orr tmp2, data1, REP8_7f |
| sub tmp3, data2, zeroones |
| orr tmp4, data2, REP8_7f |
| bics has_nul1, tmp1, tmp2 |
| bic has_nul2, tmp3, tmp4 |
| ccmp has_nul2, 0, 0, eq |
| bne L(tail) |
| ldp data1, data2, [src, 16]! |
| sub tmp1, data1, zeroones |
| orr tmp2, data1, REP8_7f |
| sub tmp3, data2, zeroones |
| orr tmp4, data2, REP8_7f |
| bics has_nul1, tmp1, tmp2 |
| bic has_nul2, tmp3, tmp4 |
| ccmp has_nul2, 0, 0, eq |
| beq L(nonascii_loop) |
| b L(tail) |
| |
| END(__strlen_aarch64_mte) |
