1/*- 2 * Copyright (c) 2015 The FreeBSD Foundation 3 * 4 * This software was developed by Andrew Turner under 5 * sponsorship from the FreeBSD Foundation. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 */ 29 30#include <sys/elf_common.h> 31#include <sys/errno.h> 32 33#include <machine/asm.h> 34#include <machine/param.h> 35#include <machine/vmparam.h> 36 37#include "assym.inc" 38 39.macro check_user_access user_arg, size_arg, bad_access_func 40 /* 41 * TBI is enabled from 15.0. Clear the top byte of the userspace 42 * address before checking whether it's within the given limit. 43 * The later load/store instructions will fault if TBI is disabled 44 * for the current process. 45 */ 46 and x6, x\user_arg, #(~TBI_ADDR_MASK) 47 adds x6, x6, x\size_arg 48 b.cs \bad_access_func 49 ldr x7, =VM_MAXUSER_ADDRESS 50 cmp x6, x7 51 b.hi \bad_access_func 52.endm 53 54/* 55 * Fault handler for the copy{in,out} functions below. 56 */ 57ENTRY(copyio_fault) 58 SET_FAULT_HANDLER(xzr, x1) /* Clear the handler */ 59 EXIT_USER_ACCESS_CHECK(w0, x1) 60copyio_fault_nopcb: 61 mov x0, #EFAULT 62 ret 63END(copyio_fault) 64 65/* 66 * Copies from a kernel to user address 67 * 68 * int copyout_std(const void *kaddr, void *udaddr, size_t len) 69 */ 70ENTRY(copyout_std) 71 cbz x2, 1f 72 check_user_access 1, 2, copyio_fault_nopcb 73 74 b copycommon 75 761: mov x0, xzr /* return 0 */ 77 ret 78 79END(copyout_std) 80 81/* 82 * Copies from a kernel to user address 83 * 84 * int copyout_mops(const void *kaddr, void *udaddr, size_t len) 85 */ 86ENTRY(copyout_mops) 87 cbz x2, 1f 88 check_user_access 1, 2, copyio_fault_nopcb 89 90 adr x6, copyio_fault /* Get the handler address */ 91 SET_FAULT_HANDLER(x6, x7) /* Set the handler */ 92 93 .inst 0x19001441 /* cpyfpwt [x1]!, [x0]!, x2! */ 94 .inst 0x19401441 /* cpyfmwt [x1]!, [x0]!, x2! */ 95 .inst 0x19801441 /* cpyfewt [x1]!, [x0]!, x2! */ 96 97 SET_FAULT_HANDLER(xzr, x7) /* Clear the handler */ 98 991: mov x0, xzr /* return 0 */ 100 ret 101 102END(copyout_mops) 103 104/* 105 * Copies from a user to kernel address 106 * 107 * int copyin_std(const void *uaddr, void *kdaddr, size_t len) 108 */ 109ENTRY(copyin_std) 110 cbz x2, 1f 111 check_user_access 0, 2, copyio_fault_nopcb 112 113 b copycommon 114 1151: mov x0, xzr /* return 0 */ 116 ret 117 118END(copyin_std) 119 120/* 121 * Copies from a user to kernel address 122 * 123 * int copyin_mops(const void *uaddr, void *kdaddr, size_t len) 124 */ 125ENTRY(copyin_mops) 126 cbz x2, 1f 127 check_user_access 0, 2, copyio_fault_nopcb 128 129 adr x6, copyio_fault /* Get the handler address */ 130 SET_FAULT_HANDLER(x6, x7) /* Set the handler */ 131 132 .inst 0x19002441 /* cpyfprt [x1]!, [x0]!, x2! */ 133 .inst 0x19402441 /* cpyfmrt [x1]!, [x0]!, x2! */ 134 .inst 0x19802441 /* cpyfert [x1]!, [x0]!, x2! */ 135 136 SET_FAULT_HANDLER(xzr, x7) /* Clear the handler */ 137 1381: mov x0, xzr /* return 0 */ 139 ret 140 141END(copyin_mops) 142 143/* 144 * Copies a string from a user to kernel address 145 * 146 * int copyinstr(const void *udaddr, void *kaddr, size_t len, size_t *done) 147 */ 148ENTRY(copyinstr) 149 mov x5, xzr /* count = 0 */ 150 mov w4, #1 /* If zero return faulure */ 151 cbz x2, 3f /* If len == 0 then skip loop */ 152 153 adr x6, copyio_fault /* Get the handler address */ 154 SET_FAULT_HANDLER(x6, x7) /* Set the handler */ 155 156 /* 157 * As in check_user_access mask off the TBI bits for the cmp 158 * instruction. The load will fail trap if TBI is disabled, but we 159 * need to check the address didn't wrap. 160 */ 161 and x6, x0, #(~TBI_ADDR_MASK) 162 ldr x7, =VM_MAXUSER_ADDRESS 1631: cmp x6, x7 164 b.cs copyio_fault 165 ldtrb w4, [x0] /* Load from uaddr */ 166 add x0, x0, #1 /* Next char */ 167 strb w4, [x1], #1 /* Store in kaddr */ 168 add x5, x5, #1 /* count++ */ 169 add x6, x6, #1 /* Increment masked address */ 170 cbz w4, 2f /* Break when NUL-terminated */ 171 sub x2, x2, #1 /* len-- */ 172 cbnz x2, 1b 173 1742: SET_FAULT_HANDLER(xzr, x7) /* Clear the handler */ 175 176 1773: cbz x3, 4f /* Check if done != NULL */ 178 str x5, [x3] /* done = count */ 179 1804: mov w1, #ENAMETOOLONG /* Load ENAMETOOLONG to return if failed */ 181 cmp w4, #0 /* Check if we saved the NUL-terminator */ 182 csel w0, wzr, w1, eq /* If so return success, else failure */ 183 ret 184END(copyinstr) 185 186/* 187 * Local helper 188 * 189 * x0 - src pointer 190 * x1 - dst pointer 191 * x2 - size 192 * lr - the return address, so jump here instead of calling 193 * 194 * This function is optimized to minimize concurrent memory accesses. In 195 * present form it is suited for cores with a single memory prefetching 196 * unit. 197 * ARM64TODO: 198 * Consider using separate functions for each ARM64 core. Adding memory 199 * access interleaving might increase a total throughput on A57 or A72. 200 */ 201 .text 202 .align 4 203 .local copycommon 204 .type copycommon,@function 205 206copycommon: 207 adr x6, copyio_fault /* Get the handler address */ 208 SET_FAULT_HANDLER(x6, x7) /* Set the handler */ 209 ENTER_USER_ACCESS(w6, x7) 210 211 /* Check alignment */ 212 orr x3, x0, x1 213 ands x3, x3, 0x07 214 b.eq aligned 215 216 /* Unaligned is byte by byte copy */ 217byte_by_byte: 218 ldrb w3, [x0], #0x01 219 strb w3, [x1], #0x01 220 subs x2, x2, #0x01 221 b.ne byte_by_byte 222 b ending 223 224aligned: 225 cmp x2, #0x10 226 b.lt lead_out 227 cmp x2, #0x40 228 b.lt by_dwords_start 229 230 /* Block copy */ 231 lsr x15, x2, #0x06 232by_blocks: 233 ldp x3, x4, [x0], #0x10 234 ldp x5, x6, [x0], #0x10 235 ldp x7, x8, [x0], #0x10 236 ldp x9, x10, [x0], #0x10 237 stp x3, x4, [x1], #0x10 238 stp x5, x6, [x1], #0x10 239 stp x7, x8, [x1], #0x10 240 stp x9, x10, [x1], #0x10 241 242 subs x15, x15, #0x01 243 b.ne by_blocks 244 245 and x2, x2, #0x3f 246 247by_dwords_start: 248 lsr x15, x2, #0x04 249 cbz x15, lead_out 250by_dwords: 251 ldp x3, x4, [x0], #0x10 252 stp x3, x4, [x1], #0x10 253 subs x15, x15, #0x01 254 b.ne by_dwords 255 256 /* Less than 16 bytes to copy */ 257lead_out: 258 tbz x2, #0x03, last_word 259 ldr x3, [x0], #0x08 260 str x3, [x1], #0x08 261 262last_word: 263 tbz x2, #0x02, last_hword 264 ldr w3, [x0], #0x04 265 str w3, [x1], #0x04 266 267last_hword: 268 tbz x2, #0x01, last_byte 269 ldrh w3, [x0], #0x02 270 strh w3, [x1], #0x02 271 272last_byte: 273 tbz x2, #0x00, ending 274 ldrb w3, [x0] 275 strb w3, [x1] 276 277ending: 278 EXIT_USER_ACCESS_CHECK(w6, x7) 279 SET_FAULT_HANDLER(xzr, x7) /* Clear the handler */ 280 281 mov x0, xzr /* return 0 */ 282 ret 283 .size copycommon, . - copycommon 284 285GNU_PROPERTY_AARCH64_FEATURE_1_NOTE(GNU_PROPERTY_AARCH64_FEATURE_1_VAL) 286