1/*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (C) 2018 Turing Robotic Industries Inc. 5 * Copyright (C) 2022 Dmitry Chagin <dchagin@FreeBSD.org> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29#include <machine/asm.h> 30#include <machine/param.h> 31#include <machine/vmparam.h> 32 33#include <sys/errno.h> 34 35#include "assym.inc" 36 37.macro check_user_access user_arg, limit, bad_addr_func 38 ldr x7, =(\limit) 39 cmp x\user_arg, x7 40 b.cs \bad_addr_func 41.endm 42 43futex_fault: 44 SET_FAULT_HANDLER(xzr, x1) 45 EXIT_USER_ACCESS_CHECK(w0, x1) 46futex_fault_nopcb: 47 mov x0, #EFAULT 48 ret 49 50#define LINUX_FUTEX_MAX_LOOPS 128 51 52/* 53 * int oparg, uint32_t *uaddr, int *oldval 54 * 55 * Return 0 on success, errno on failure, 56 * EAGAIN is returned if LL/SC operation fails. 57 * 58 * XXX. VM_MAXUSER_ADDRESS is not applicable here, should be replaced 59 * by something like LINUX_SHAREDPAGE. 60 */ 61 62/* (int *)uaddr2 = oparg */ 63ENTRY(futex_xchgl) 64 check_user_access 1, (VM_MAXUSER_ADDRESS-3), futex_fault_nopcb 65 adr x9, futex_fault /* Load the fault handler */ 66 SET_FAULT_HANDLER(x9, x4) /* And set it */ 67 ENTER_USER_ACCESS(w9, x4) 68 mov w5, #LINUX_FUTEX_MAX_LOOPS 69 prfm pstl1strm, [x1] 70 mov w6, w0 /* Save oparg */ 711: ldxr w4, [x1] /* Load oldval from uaddr */ 72 stlxr w0, w6, [x1] /* Store oparg to uaddr */ 73 cbz w0, 3f /* Exit on success */ 74 sub w5, w5, w0 /* Dec loop counter, w0 is 1 */ 75 cbnz w5, 1b /* Loop */ 76 mov x0, #EAGAIN /* Store of newval failed */ 773: dmb ish 78 EXIT_USER_ACCESS(w9) 79 SET_FAULT_HANDLER(xzr, x9) /* Reset the fault handler */ 80 str w4, [x2] /* Store oldval */ 81 ret 82END(futex_xchgl) 83 84/* (int *)uaddr2 += oparg */ 85ENTRY(futex_addl) 86 check_user_access 1, (VM_MAXUSER_ADDRESS-3), futex_fault_nopcb 87 adr x9, futex_fault 88 SET_FAULT_HANDLER(x9, x4) 89 ENTER_USER_ACCESS(w9, x4) 90 mov w5, #LINUX_FUTEX_MAX_LOOPS 91 prfm pstl1strm, [x1] 92 mov w6, w0 931: ldxr w4, [x1] 94 add w3, w4, w6 /* oldval + oparg */ 95 stlxr w0, w3, [x1] 96 cbz w0, 3f 97 sub w5, w5, w0 98 cbnz w5, 1b 99 mov x0, #EAGAIN 1003: dmb ish 101 EXIT_USER_ACCESS(w9) 102 SET_FAULT_HANDLER(xzr, x9) 103 str w4, [x2] 104 ret 105END(futex_addl) 106 107/* (int *)uaddr2 |= oparg */ 108ENTRY(futex_orl) 109 check_user_access 1, (VM_MAXUSER_ADDRESS-3), futex_fault_nopcb 110 adr x9, futex_fault 111 SET_FAULT_HANDLER(x9, x4) 112 ENTER_USER_ACCESS(w9, x4) 113 mov w5, #LINUX_FUTEX_MAX_LOOPS 114 prfm pstl1strm, [x1] 115 mov w6, w0 1161: ldxr w4, [x1] 117 orr w3, w4, w6 /* oldavl |= oparg */ 118 stlxr w0, w3, [x1] 119 cbz w0, 3f 120 sub w5, w5, w0 121 cbnz w5, 1b 122 mov x0, #EAGAIN 1233: dmb ish 124 EXIT_USER_ACCESS(w9) 125 SET_FAULT_HANDLER(xzr, x9) 126 str w4, [x2] 127 ret 128END(futex_orl) 129 130/* (int *)uaddr2 &= oparg */ 131ENTRY(futex_andl) 132 check_user_access 1, (VM_MAXUSER_ADDRESS-3), futex_fault_nopcb 133 adr x9, futex_fault 134 SET_FAULT_HANDLER(x9, x4) 135 ENTER_USER_ACCESS(w9, x4) 136 mov w5, #LINUX_FUTEX_MAX_LOOPS 137 prfm pstl1strm, [x1] 138 mov w6, w0 1391: ldxr w4, [x1] 140 and w3, w4, w6 /* oldval &= oparg */ 141 stlxr w0, w3, [x1] 142 cbz w0, 3f 143 sub w5, w5, w0 144 cbnz w5, 1b 145 mov x0, #EAGAIN 1463: dmb ish 147 EXIT_USER_ACCESS(w9) 148 SET_FAULT_HANDLER(xzr, x9) 149 str w4, [x2] 150 ret 151END(futex_andl) 152 153/* (int *)uaddr2 ^= oparg */ 154ENTRY(futex_xorl) 155 check_user_access 1, (VM_MAXUSER_ADDRESS-3), futex_fault_nopcb 156 adr x9, futex_fault 157 SET_FAULT_HANDLER(x9, x4) 158 ENTER_USER_ACCESS(w9, x4) 159 mov w5, #LINUX_FUTEX_MAX_LOOPS 160 prfm pstl1strm, [x1] 161 mov w6, w0 1621: ldxr w4, [x1] 163 eor w3, w4, w6 /* oldval ^= oparg */ 164 stlxr w0, w3, [x1] 165 cbz w0, 3f 166 sub w5, w5, w0 167 cbnz w5, 1b 168 mov x0, #EAGAIN 1693: dmb ish 170 EXIT_USER_ACCESS(w9) 171 SET_FAULT_HANDLER(xzr, x9) 172 str w4, [x2] 173 ret 174END(futex_xorl) 175