1/*- 2 * Copyright (c) 2014 Andrew Turner 3 * Copyright (c) 2014-2015 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * Portions of this software were developed by Andrew Turner 7 * under sponsorship from the FreeBSD Foundation 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 */ 31 32#include <sys/elf_common.h> 33 34#include <machine/asm.h> 35#include <machine/setjmp.h> 36#include <machine/param.h> 37#include <machine/vmparam.h> 38 39#include "assym.inc" 40 41.macro check_user_access user_arg, limit, bad_addr_func 42 /* 43 * TBI is enabled from 15.0. Clear the top byte of the userspace 44 * address before checking whether it's within the given limit. 45 * The later load/store instructions will fault if TBI is disabled 46 * for the current process. 47 */ 48 and x6, x\user_arg, #(~TBI_ADDR_MASK) 49 ldr x7, =(\limit) 50 cmp x6, x7 51 b.cs \bad_addr_func 52.endm 53 54/* 55 * One of the fu* or su* functions failed, return -1. 56 */ 57ENTRY(fsu_fault) 58 SET_FAULT_HANDLER(xzr, x1) /* Reset the handler function */ 59 EXIT_USER_ACCESS_CHECK(w0, x1) 60fsu_fault_nopcb: 61 mov x0, #-1 62 ret 63END(fsu_fault) 64 65/* 66 * int swapueword8_llsc(volatile uint8_t *, uint8_t *) 67 */ 68ENTRY(swapueword8_llsc) 69 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 70 adr x6, fsu_fault /* Load the fault handler */ 71 SET_FAULT_HANDLER(x6, x4) /* And set it */ 72 ENTER_USER_ACCESS(w6, x4) 73 74 ldrb w7, [x1] 75 76 ldxrb w2, [x0] 77 stxrb w3, w7, [x0] 78 cbnz w3, 1f 79 80 strb w2, [x1] /* Stash old value in *val */ 81 821: EXIT_USER_ACCESS(w6) 83 SET_FAULT_HANDLER(xzr, x6) 84 mov w0, w3 85 ret 86END(swapueword8_llsc) 87 88/* 89 * int swapueword8_lse(volatile uint8_t *, uint8_t *) 90 */ 91ENTRY(swapueword8_lse) 92 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 93 adr x6, fsu_fault /* Load the fault handler */ 94 SET_FAULT_HANDLER(x6, x4) /* And set it */ 95 ENTER_USER_ACCESS(w6, x4) 96 97 ldrb w7, [x1] 98 99 .arch_extension lse 100 swpb w7, w2, [x0] 101 .arch_extension nolse 102 103 strb w2, [x1] /* Stash old value in *val */ 104 105 EXIT_USER_ACCESS(w6) 106 SET_FAULT_HANDLER(xzr, x6) 107 mov w0, #0 108 ret 109END(swapueword8_lse) 110 111/* 112 * int swapueword32_llsc(volatile uint32_t *, uint32_t *) 113 */ 114ENTRY(swapueword32_llsc) 115 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 116 adr x6, fsu_fault /* Load the fault handler */ 117 SET_FAULT_HANDLER(x6, x4) /* And set it */ 118 ENTER_USER_ACCESS(w6, x4) 119 120 ldr w7, [x1] 121 122 ldxr w2, [x0] /* Stash the old value in w2 */ 123 stxr w3, w7, [x0] /* Store new value */ 124 cbnz w3, 1f 125 126 str w2, [x1] /* Stash old value in *val */ 127 1281: EXIT_USER_ACCESS(w6) 129 SET_FAULT_HANDLER(xzr, x6) 130 mov w0, w3 131 ret 132END(swapueword32_llsc) 133 134/* 135 * int swapueword32_lse(volatile uint32_t *, uint32_t *) 136 */ 137ENTRY(swapueword32_lse) 138 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 139 adr x6, fsu_fault /* Load the fault handler */ 140 SET_FAULT_HANDLER(x6, x4) /* And set it */ 141 ENTER_USER_ACCESS(w6, x4) 142 143 ldr w7, [x1] 144 145 .arch_extension lse 146 swp w7, w2, [x0] 147 .arch_extension nolse 148 149 str w2, [x1] /* Stash old value in *val */ 150 151 EXIT_USER_ACCESS(w6) 152 SET_FAULT_HANDLER(xzr, x6) 153 mov w0, #0 154 ret 155END(swapueword32_llsc) 156 157/* 158 * int casueword32_llsc(volatile uint32_t *, uint32_t, uint32_t *, uint32_t) 159 */ 160ENTRY(casueword32_llsc) 161 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 162 adr x6, fsu_fault /* Load the fault handler */ 163 mov w5, #1 164 SET_FAULT_HANDLER(x6, x4) /* And set it */ 165 ENTER_USER_ACCESS(w6, x4) 166 ldxr w4, [x0] /* Load-exclusive the data */ 167 cmp w4, w1 /* Compare */ 168 b.ne 1f /* Not equal, exit */ 169 stxr w5, w3, [x0] /* Store the new data */ 1701: EXIT_USER_ACCESS(w6) 171 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ 172 str w4, [x2] /* Store the read data */ 173 mov w0, w5 /* Result same as store status */ 174 ret /* Return */ 175END(casueword32_llsc) 176 177/* 178 * int casueword32_lse(volatile uint32_t *, uint32_t, uint32_t *, uint32_t) 179 */ 180ENTRY(casueword32_lse) 181 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 182 adr x6, fsu_fault /* Load the fault handler */ 183 SET_FAULT_HANDLER(x6, x4) /* And set it */ 184 ENTER_USER_ACCESS(w6, x4) 185 mov w7, w1 /* Back up the compare value */ 186 .arch_extension lse 187 cas w1, w3, [x0] /* Compare and Swap */ 188 .arch_extension nolse 189 cmp w1, w7 /* Check if successful */ 190 cset w0, ne /* Return 0 on success, 1 on failure */ 191 EXIT_USER_ACCESS(w6) 192 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ 193 str w1, [x2] /* Store the read data */ 194 ret /* Return */ 195END(casueword32_lse) 196 197/* 198 * int casueword_llsc(volatile u_long *, u_long, u_long *, u_long) 199 */ 200ENTRY(casueword_llsc) 201 check_user_access 0, (VM_MAXUSER_ADDRESS-7), fsu_fault_nopcb 202 adr x6, fsu_fault /* Load the fault handler */ 203 mov w5, #1 204 SET_FAULT_HANDLER(x6, x4) /* And set it */ 205 ENTER_USER_ACCESS(w6, x4) 206 ldxr x4, [x0] /* Load-exclusive the data */ 207 cmp x4, x1 /* Compare */ 208 b.ne 1f /* Not equal, exit */ 209 stxr w5, x3, [x0] /* Store the new data */ 2101: EXIT_USER_ACCESS(w6) 211 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ 212 str x4, [x2] /* Store the read data */ 213 mov w0, w5 /* Result same as store status */ 214 ret /* Return */ 215END(casueword_llsc) 216 217/* 218 * int casueword_lse(volatile u_long *, u_long, u_long *, u_long) 219 */ 220ENTRY(casueword_lse) 221 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 222 adr x6, fsu_fault /* Load the fault handler */ 223 SET_FAULT_HANDLER(x6, x4) /* And set it */ 224 ENTER_USER_ACCESS(w6, x4) 225 mov x7, x1 /* Back up the compare value */ 226 .arch_extension lse 227 cas x1, x3, [x0] /* Compare and Swap */ 228 .arch_extension nolse 229 cmp x1, x7 /* Check if successful */ 230 cset w0, ne /* Return 0 on success, 1 on failure */ 231 EXIT_USER_ACCESS(w6) 232 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ 233 str x1, [x2] /* Store the read data */ 234 ret /* Return */ 235END(casueword_lse) 236 237.macro fsudata insn, ret_reg, user_arg 238 adr x7, fsu_fault /* Load the fault handler */ 239 SET_FAULT_HANDLER(x7, x6) /* And set it */ 240 \insn \ret_reg, [x\user_arg] /* Try accessing the data */ 241 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ 242.endm 243 244/* 245 * int fubyte(volatile const void *) 246 */ 247ENTRY(fubyte) 248 check_user_access 0, (VM_MAXUSER_ADDRESS), fsu_fault_nopcb 249 fsudata ldtrb, w0, 0 250 ret /* Return */ 251END(fubyte) 252 253/* 254 * int fuword(volatile const void *) 255 */ 256ENTRY(fuword16) 257 check_user_access 0, (VM_MAXUSER_ADDRESS-1), fsu_fault_nopcb 258 fsudata ldtrh, w0, 0 259 ret /* Return */ 260END(fuword16) 261 262/* 263 * int32_t fueword32(volatile const void *, int32_t *) 264 */ 265ENTRY(fueword32) 266 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 267 fsudata ldtr, w0, 0 268 str w0, [x1] /* Save the data in kernel space */ 269 mov w0, #0 /* Success */ 270 ret /* Return */ 271END(fueword32) 272 273/* 274 * long fueword(volatile const void *, int64_t *) 275 * int64_t fueword64(volatile const void *, int64_t *) 276 */ 277EENTRY(fueword64) 278ENTRY(fueword) 279 check_user_access 0, (VM_MAXUSER_ADDRESS-7), fsu_fault_nopcb 280 fsudata ldtr, x0, 0 281 str x0, [x1] /* Save the data in kernel space */ 282 mov x0, #0 /* Success */ 283 ret /* Return */ 284END(fueword) 285EEND(fueword64) 286 287/* 288 * int subyte(volatile void *, int) 289 */ 290ENTRY(subyte) 291 check_user_access 0, (VM_MAXUSER_ADDRESS), fsu_fault_nopcb 292 fsudata sttrb, w1, 0 293 mov x0, #0 /* Success */ 294 ret /* Return */ 295END(subyte) 296 297/* 298 * int suword16(volatile void *, int) 299 */ 300ENTRY(suword16) 301 check_user_access 0, (VM_MAXUSER_ADDRESS-1), fsu_fault_nopcb 302 fsudata sttrh, w1, 0 303 mov x0, #0 /* Success */ 304 ret /* Return */ 305END(suword16) 306 307/* 308 * int suword32(volatile void *, int) 309 */ 310ENTRY(suword32) 311 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 312 fsudata sttr, w1, 0 313 mov x0, #0 /* Success */ 314 ret /* Return */ 315END(suword32) 316 317/* 318 * int suword(volatile void *, long) 319 */ 320EENTRY(suword64) 321ENTRY(suword) 322 check_user_access 0, (VM_MAXUSER_ADDRESS-7), fsu_fault_nopcb 323 fsudata sttr, x1, 0 324 mov x0, #0 /* Success */ 325 ret /* Return */ 326END(suword) 327EEND(suword64) 328 329ENTRY(setjmp) 330 /* Store the stack pointer */ 331 mov x8, sp 332 str x8, [x0], #8 333 334 /* Store the general purpose registers and lr */ 335 stp x19, x20, [x0], #16 336 stp x21, x22, [x0], #16 337 stp x23, x24, [x0], #16 338 stp x25, x26, [x0], #16 339 stp x27, x28, [x0], #16 340 stp x29, lr, [x0], #16 341 342 /* Return value */ 343 mov x0, #0 344 ret 345END(setjmp) 346 347ENTRY(longjmp) 348 /* Restore the stack pointer */ 349 ldr x8, [x0], #8 350 mov sp, x8 351 352 /* Restore the general purpose registers and lr */ 353 ldp x19, x20, [x0], #16 354 ldp x21, x22, [x0], #16 355 ldp x23, x24, [x0], #16 356 ldp x25, x26, [x0], #16 357 ldp x27, x28, [x0], #16 358 ldp x29, lr, [x0], #16 359 360 /* Load the return value */ 361 mov x0, x1 362 ret 363END(longjmp) 364 365/* 366 * pagezero, simple implementation 367 */ 368ENTRY(pagezero_simple) 369 add x1, x0, #PAGE_SIZE 370 3711: 372 stp xzr, xzr, [x0], #0x10 373 stp xzr, xzr, [x0], #0x10 374 stp xzr, xzr, [x0], #0x10 375 stp xzr, xzr, [x0], #0x10 376 cmp x0, x1 377 b.ne 1b 378 ret 379 380END(pagezero_simple) 381 382/* 383 * pagezero, cache assisted 384 */ 385ENTRY(pagezero_cache) 386 add x1, x0, #PAGE_SIZE 387 388 adrp x2, dczva_line_size 389 ldr x2, [x2, :lo12:dczva_line_size] 390 3911: 392 dc zva, x0 393 add x0, x0, x2 394 cmp x0, x1 395 b.ne 1b 396 ret 397 398END(pagezero_cache) 399 400GNU_PROPERTY_AARCH64_FEATURE_1_NOTE(GNU_PROPERTY_AARCH64_FEATURE_1_VAL) 401