1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Copyright (c) 2012-2022, Arm Limited. 4 * 5 * Adapted from the original at: 6 * https://github.com/ARM-software/optimized-routines/blob/189dfefe37d54c5b/string/aarch64/strcmp.S 7 */ 8 9#include <linux/linkage.h> 10#include <asm/assembler.h> 11 12/* Assumptions: 13 * 14 * ARMv8-a, AArch64. 15 * MTE compatible. 16 */ 17 18#define L(label) .L ## label 19 20#define REP8_01 0x0101010101010101 21#define REP8_7f 0x7f7f7f7f7f7f7f7f 22 23#define src1 x0 24#define src2 x1 25#define result x0 26 27#define data1 x2 28#define data1w w2 29#define data2 x3 30#define data2w w3 31#define has_nul x4 32#define diff x5 33#define off1 x5 34#define syndrome x6 35#define tmp x6 36#define data3 x7 37#define zeroones x8 38#define shift x9 39#define off2 x10 40 41/* NUL detection works on the principle that (X - 1) & (~X) & 0x80 42 (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and 43 can be done in parallel across the entire word. 44 Since carry propagation makes 0x1 bytes before a NUL byte appear 45 NUL too in big-endian, byte-reverse the data before the NUL check. */ 46 47 48SYM_FUNC_START(__pi_strcmp) 49 sub off2, src2, src1 50 mov zeroones, REP8_01 51 and tmp, src1, 7 52 tst off2, 7 53 b.ne L(misaligned8) 54 cbnz tmp, L(mutual_align) 55 56 .p2align 4 57 58L(loop_aligned): 59 ldr data2, [src1, off2] 60 ldr data1, [src1], 8 61L(start_realigned): 62 sub has_nul, data1, zeroones 63 orr tmp, data1, REP8_7f 64 bics has_nul, has_nul, tmp /* Non-zero if NUL terminator. */ 65 ccmp data1, data2, 0, eq 66 b.eq L(loop_aligned) 67 eor diff, data1, data2 68 orr syndrome, diff, has_nul 69L(end): 70 rev syndrome, syndrome 71 rev data1, data1 72 rev data2, data2 73 clz shift, syndrome 74 /* The most-significant-non-zero bit of the syndrome marks either the 75 first bit that is different, or the top bit of the first zero byte. 76 Shifting left now will bring the critical information into the 77 top bits. */ 78 lsl data1, data1, shift 79 lsl data2, data2, shift 80 /* But we need to zero-extend (char is unsigned) the value and then 81 perform a signed 32-bit subtraction. */ 82 lsr data1, data1, 56 83 sub result, data1, data2, lsr 56 84 ret 85 86 .p2align 4 87 88L(mutual_align): 89 /* Sources are mutually aligned, but are not currently at an 90 alignment boundary. Round down the addresses and then mask off 91 the bytes that precede the start point. */ 92 bic src1, src1, 7 93 ldr data2, [src1, off2] 94 ldr data1, [src1], 8 95 neg shift, src2, lsl 3 /* Bits to alignment -64. */ 96 mov tmp, -1 97 lsr tmp, tmp, shift 98 orr data1, data1, tmp 99 orr data2, data2, tmp 100 b L(start_realigned) 101 102L(misaligned8): 103 /* Align SRC1 to 8 bytes and then compare 8 bytes at a time, always 104 checking to make sure that we don't access beyond the end of SRC2. */ 105 cbz tmp, L(src1_aligned) 106L(do_misaligned): 107 ldrb data1w, [src1], 1 108 ldrb data2w, [src2], 1 109 cmp data1w, 0 110 ccmp data1w, data2w, 0, ne /* NZCV = 0b0000. */ 111 b.ne L(done) 112 tst src1, 7 113 b.ne L(do_misaligned) 114 115L(src1_aligned): 116 neg shift, src2, lsl 3 117 bic src2, src2, 7 118 ldr data3, [src2], 8 119 lsr tmp, zeroones, shift 120 orr data3, data3, tmp 121 sub has_nul, data3, zeroones 122 orr tmp, data3, REP8_7f 123 bics has_nul, has_nul, tmp 124 b.ne L(tail) 125 126 sub off1, src2, src1 127 128 .p2align 4 129 130L(loop_unaligned): 131 ldr data3, [src1, off1] 132 ldr data2, [src1, off2] 133 sub has_nul, data3, zeroones 134 orr tmp, data3, REP8_7f 135 ldr data1, [src1], 8 136 bics has_nul, has_nul, tmp 137 ccmp data1, data2, 0, eq 138 b.eq L(loop_unaligned) 139 140 lsl tmp, has_nul, shift 141 eor diff, data1, data2 142 orr syndrome, diff, tmp 143 cbnz syndrome, L(end) 144L(tail): 145 ldr data1, [src1] 146 neg shift, shift 147 lsr data2, data3, shift 148 lsr has_nul, has_nul, shift 149 eor diff, data1, data2 150 orr syndrome, diff, has_nul 151 b L(end) 152 153L(done): 154 sub result, data1, data2 155 ret 156SYM_FUNC_END(__pi_strcmp) 157SYM_FUNC_ALIAS_WEAK(strcmp, __pi_strcmp) 158EXPORT_SYMBOL_NOKASAN(strcmp) 159