1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2018 The FreeBSD Foundation
5 *
6 * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
7 * under sponsorship from the FreeBSD Foundation.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #include <sys/param.h>
32 #include <sys/lock.h>
33 #include <sys/mutex.h>
34 #include <sys/pcpu.h>
35 #include <sys/proc.h>
36 #include <sys/sched.h>
37 #include <sys/sysctl.h>
38 #include <sys/systm.h>
39 #include <vm/vm.h>
40 #include <vm/vm_param.h>
41 #include <vm/vm_extern.h>
42 #include <vm/pmap.h>
43 #include <vm/vm_map.h>
44 #include <vm/vm_page.h>
45
46 int copyin_fast(const void *udaddr, void *kaddr, size_t len, u_int);
47 static int (*copyin_fast_tramp)(const void *, void *, size_t, u_int);
48 int copyout_fast(const void *kaddr, void *udaddr, size_t len, u_int);
49 static int (*copyout_fast_tramp)(const void *, void *, size_t, u_int);
50 int fubyte_fast(volatile const void *base, u_int kcr3);
51 static int (*fubyte_fast_tramp)(volatile const void *, u_int);
52 int fuword16_fast(volatile const void *base, u_int kcr3);
53 static int (*fuword16_fast_tramp)(volatile const void *, u_int);
54 int fueword_fast(volatile const void *base, long *val, u_int kcr3);
55 static int (*fueword_fast_tramp)(volatile const void *, long *, u_int);
56 int subyte_fast(volatile void *base, int val, u_int kcr3);
57 static int (*subyte_fast_tramp)(volatile void *, int, u_int);
58 int suword16_fast(volatile void *base, int val, u_int kcr3);
59 static int (*suword16_fast_tramp)(volatile void *, int, u_int);
60 int suword_fast(volatile void *base, long val, u_int kcr3);
61 static int (*suword_fast_tramp)(volatile void *, long, u_int);
62
63 static int fast_copyout = 1;
64 SYSCTL_INT(_machdep, OID_AUTO, fast_copyout, CTLFLAG_RWTUN,
65 &fast_copyout, 0,
66 "");
67
68 void
copyout_init_tramp(void)69 copyout_init_tramp(void)
70 {
71
72 copyin_fast_tramp = (int (*)(const void *, void *, size_t, u_int))(
73 (uintptr_t)copyin_fast + setidt_disp);
74 copyout_fast_tramp = (int (*)(const void *, void *, size_t, u_int))(
75 (uintptr_t)copyout_fast + setidt_disp);
76 fubyte_fast_tramp = (int (*)(volatile const void *, u_int))(
77 (uintptr_t)fubyte_fast + setidt_disp);
78 fuword16_fast_tramp = (int (*)(volatile const void *, u_int))(
79 (uintptr_t)fuword16_fast + setidt_disp);
80 fueword_fast_tramp = (int (*)(volatile const void *, long *, u_int))(
81 (uintptr_t)fueword_fast + setidt_disp);
82 subyte_fast_tramp = (int (*)(volatile void *, int, u_int))(
83 (uintptr_t)subyte_fast + setidt_disp);
84 suword16_fast_tramp = (int (*)(volatile void *, int, u_int))(
85 (uintptr_t)suword16_fast + setidt_disp);
86 suword_fast_tramp = (int (*)(volatile void *, long, u_int))(
87 (uintptr_t)suword_fast + setidt_disp);
88 }
89
90 int
cp_slow0(vm_offset_t uva,size_t len,bool write,void (* f)(void *,void *),void * arg)91 cp_slow0(vm_offset_t uva, size_t len, bool write,
92 void (*f)(void *, void *), void *arg)
93 {
94 struct pcpu *pc;
95 vm_page_t m[2];
96 char *kaddr;
97 int error, i, plen;
98 bool sleepable;
99
100 plen = howmany(uva - trunc_page(uva) + len, PAGE_SIZE);
101 MPASS(plen <= nitems(m));
102 error = 0;
103 i = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, uva, len,
104 (write ? VM_PROT_WRITE : VM_PROT_READ) | VM_PROT_QUICK_NOFAULT,
105 m, nitems(m));
106 if (i != plen)
107 return (EFAULT);
108 sched_pin();
109 pc = get_pcpu();
110 if (!THREAD_CAN_SLEEP() || curthread->td_vslock_sz > 0 ||
111 (curthread->td_pflags & TDP_NOFAULTING) != 0) {
112 sleepable = false;
113 mtx_lock(&pc->pc_copyout_mlock);
114 kaddr = pc->pc_copyout_maddr;
115 } else {
116 sleepable = true;
117 sx_xlock(&pc->pc_copyout_slock);
118 kaddr = pc->pc_copyout_saddr;
119 }
120 pmap_cp_slow0_map((vm_offset_t)kaddr, plen, m);
121 kaddr += uva - trunc_page(uva);
122 f(kaddr, arg);
123 sched_unpin();
124 if (sleepable)
125 sx_xunlock(&pc->pc_copyout_slock);
126 else
127 mtx_unlock(&pc->pc_copyout_mlock);
128 vm_page_unhold_pages(m, plen);
129 return (error);
130 }
131
132 struct copyinstr_arg0 {
133 char *kc;
134 size_t len;
135 size_t alen;
136 bool end;
137 };
138
139 static void
copyinstr_slow0(void * kva,void * arg)140 copyinstr_slow0(void *kva, void *arg)
141 {
142 struct copyinstr_arg0 *ca;
143 char *src;
144 char c;
145
146 ca = arg;
147 src = kva;
148 MPASS(ca->alen == 0 && ca->len > 0 && !ca->end);
149 while (ca->alen < ca->len && !ca->end) {
150 c = *(src + ca->alen);
151 *ca->kc = c;
152 ca->alen++;
153 ca->kc++;
154 if (c == '\0')
155 ca->end = true;
156 }
157 }
158
159 int
copyinstr(const void * udaddr,void * kaddr,size_t maxlen,size_t * lencopied)160 copyinstr(const void *udaddr, void *kaddr, size_t maxlen, size_t *lencopied)
161 {
162 struct copyinstr_arg0 ca;
163 vm_offset_t uc;
164 size_t plen;
165 int error;
166
167 error = 0;
168 ca.end = false;
169 for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = kaddr;
170 plen < maxlen && !ca.end; uc += ca.alen, plen += ca.alen) {
171 ca.len = round_page(uc) - uc;
172 if (ca.len == 0)
173 ca.len = PAGE_SIZE;
174 if (plen + ca.len > maxlen)
175 ca.len = maxlen - plen;
176 ca.alen = 0;
177 if (cp_slow0(uc, ca.len, false, copyinstr_slow0, &ca) != 0) {
178 error = EFAULT;
179 break;
180 }
181 }
182 if (!ca.end && plen == maxlen && error == 0)
183 error = ENAMETOOLONG;
184 if (lencopied != NULL)
185 *lencopied = plen;
186 return (error);
187 }
188
189 struct copyin_arg0 {
190 char *kc;
191 size_t len;
192 };
193
194 static void
copyin_slow0(void * kva,void * arg)195 copyin_slow0(void *kva, void *arg)
196 {
197 struct copyin_arg0 *ca;
198
199 ca = arg;
200 bcopy(kva, ca->kc, ca->len);
201 }
202
203 int
copyin(const void * udaddr,void * kaddr,size_t len)204 copyin(const void *udaddr, void *kaddr, size_t len)
205 {
206 struct copyin_arg0 ca;
207 vm_offset_t uc;
208 size_t plen;
209
210 if ((uintptr_t)udaddr + len < (uintptr_t)udaddr ||
211 (uintptr_t)udaddr + len > VM_MAXUSER_ADDRESS)
212 return (EFAULT);
213 if (len == 0 || (fast_copyout && len <= TRAMP_COPYOUT_SZ &&
214 copyin_fast_tramp(udaddr, kaddr, len, pmap_get_kcr3()) == 0))
215 return (0);
216 for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = kaddr;
217 plen < len; uc += ca.len, ca.kc += ca.len, plen += ca.len) {
218 ca.len = round_page(uc) - uc;
219 if (ca.len == 0)
220 ca.len = PAGE_SIZE;
221 if (plen + ca.len > len)
222 ca.len = len - plen;
223 if (cp_slow0(uc, ca.len, false, copyin_slow0, &ca) != 0)
224 return (EFAULT);
225 }
226 return (0);
227 }
228
229 static void
copyout_slow0(void * kva,void * arg)230 copyout_slow0(void *kva, void *arg)
231 {
232 struct copyin_arg0 *ca;
233
234 ca = arg;
235 bcopy(ca->kc, kva, ca->len);
236 }
237
238 int
copyout(const void * kaddr,void * udaddr,size_t len)239 copyout(const void *kaddr, void *udaddr, size_t len)
240 {
241 struct copyin_arg0 ca;
242 vm_offset_t uc;
243 size_t plen;
244
245 if ((uintptr_t)udaddr + len < (uintptr_t)udaddr ||
246 (uintptr_t)udaddr + len > VM_MAXUSER_ADDRESS)
247 return (EFAULT);
248 if (len == 0 || (fast_copyout && len <= TRAMP_COPYOUT_SZ &&
249 copyout_fast_tramp(kaddr, udaddr, len, pmap_get_kcr3()) == 0))
250 return (0);
251 for (plen = 0, uc = (vm_offset_t)udaddr, ca.kc = __DECONST(void *, kaddr);
252 plen < len; uc += ca.len, ca.kc += ca.len, plen += ca.len) {
253 ca.len = round_page(uc) - uc;
254 if (ca.len == 0)
255 ca.len = PAGE_SIZE;
256 if (plen + ca.len > len)
257 ca.len = len - plen;
258 if (cp_slow0(uc, ca.len, true, copyout_slow0, &ca) != 0)
259 return (EFAULT);
260 }
261 return (0);
262 }
263
264 /*
265 * Fetch (load) a 32-bit word, a 16-bit word, or an 8-bit byte from user
266 * memory.
267 */
268
269 static void
fubyte_slow0(void * kva,void * arg)270 fubyte_slow0(void *kva, void *arg)
271 {
272
273 *(int *)arg = *(u_char *)kva;
274 }
275
276 int
fubyte(volatile const void * base)277 fubyte(volatile const void *base)
278 {
279 int res;
280
281 if ((uintptr_t)base + sizeof(uint8_t) < (uintptr_t)base ||
282 (uintptr_t)base + sizeof(uint8_t) > VM_MAXUSER_ADDRESS)
283 return (-1);
284 if (fast_copyout) {
285 res = fubyte_fast_tramp(base, pmap_get_kcr3());
286 if (res != -1)
287 return (res);
288 }
289 if (cp_slow0((vm_offset_t)base, sizeof(char), false, fubyte_slow0,
290 &res) != 0)
291 return (-1);
292 return (res);
293 }
294
295 static void
fuword16_slow0(void * kva,void * arg)296 fuword16_slow0(void *kva, void *arg)
297 {
298
299 *(int *)arg = *(uint16_t *)kva;
300 }
301
302 int
fuword16(volatile const void * base)303 fuword16(volatile const void *base)
304 {
305 int res;
306
307 if ((uintptr_t)base + sizeof(uint16_t) < (uintptr_t)base ||
308 (uintptr_t)base + sizeof(uint16_t) > VM_MAXUSER_ADDRESS)
309 return (-1);
310 if (fast_copyout) {
311 res = fuword16_fast_tramp(base, pmap_get_kcr3());
312 if (res != -1)
313 return (res);
314 }
315 if (cp_slow0((vm_offset_t)base, sizeof(uint16_t), false,
316 fuword16_slow0, &res) != 0)
317 return (-1);
318 return (res);
319 }
320
321 static void
fueword_slow0(void * kva,void * arg)322 fueword_slow0(void *kva, void *arg)
323 {
324
325 *(uint32_t *)arg = *(uint32_t *)kva;
326 }
327
328 int
fueword(volatile const void * base,long * val)329 fueword(volatile const void *base, long *val)
330 {
331 uint32_t res;
332
333 if ((uintptr_t)base + sizeof(*val) < (uintptr_t)base ||
334 (uintptr_t)base + sizeof(*val) > VM_MAXUSER_ADDRESS)
335 return (-1);
336 if (fast_copyout) {
337 if (fueword_fast_tramp(base, val, pmap_get_kcr3()) == 0)
338 return (0);
339 }
340 if (cp_slow0((vm_offset_t)base, sizeof(long), false, fueword_slow0,
341 &res) != 0)
342 return (-1);
343 *val = res;
344 return (0);
345 }
346
347 int
fueword32(volatile const void * base,int32_t * val)348 fueword32(volatile const void *base, int32_t *val)
349 {
350
351 return (fueword(base, (long *)val));
352 }
353
354 /*
355 * Store a 32-bit word, a 16-bit word, or an 8-bit byte to user memory.
356 */
357
358 static void
subyte_slow0(void * kva,void * arg)359 subyte_slow0(void *kva, void *arg)
360 {
361
362 *(u_char *)kva = *(int *)arg;
363 }
364
365 int
subyte(volatile void * base,int byte)366 subyte(volatile void *base, int byte)
367 {
368
369 if ((uintptr_t)base + sizeof(uint8_t) < (uintptr_t)base ||
370 (uintptr_t)base + sizeof(uint8_t) > VM_MAXUSER_ADDRESS)
371 return (-1);
372 if (fast_copyout && subyte_fast_tramp(base, byte, pmap_get_kcr3()) == 0)
373 return (0);
374 return (cp_slow0((vm_offset_t)base, sizeof(u_char), true, subyte_slow0,
375 &byte) != 0 ? -1 : 0);
376 }
377
378 static void
suword16_slow0(void * kva,void * arg)379 suword16_slow0(void *kva, void *arg)
380 {
381
382 *(int *)kva = *(uint16_t *)arg;
383 }
384
385 int
suword16(volatile void * base,int word)386 suword16(volatile void *base, int word)
387 {
388
389 if ((uintptr_t)base + sizeof(uint16_t) < (uintptr_t)base ||
390 (uintptr_t)base + sizeof(uint16_t) > VM_MAXUSER_ADDRESS)
391 return (-1);
392 if (fast_copyout && suword16_fast_tramp(base, word, pmap_get_kcr3())
393 == 0)
394 return (0);
395 return (cp_slow0((vm_offset_t)base, sizeof(int16_t), true,
396 suword16_slow0, &word) != 0 ? -1 : 0);
397 }
398
399 static void
suword_slow0(void * kva,void * arg)400 suword_slow0(void *kva, void *arg)
401 {
402
403 *(int *)kva = *(uint32_t *)arg;
404 }
405
406 int
suword(volatile void * base,long word)407 suword(volatile void *base, long word)
408 {
409
410 if ((uintptr_t)base + sizeof(word) < (uintptr_t)base ||
411 (uintptr_t)base + sizeof(word) > VM_MAXUSER_ADDRESS)
412 return (-1);
413 if (fast_copyout && suword_fast_tramp(base, word, pmap_get_kcr3()) == 0)
414 return (0);
415 return (cp_slow0((vm_offset_t)base, sizeof(long), true,
416 suword_slow0, &word) != 0 ? -1 : 0);
417 }
418
419 int
suword32(volatile void * base,int32_t word)420 suword32(volatile void *base, int32_t word)
421 {
422
423 return (suword(base, word));
424 }
425
426 struct casueword_arg0 {
427 uint32_t oldval;
428 uint32_t newval;
429 int res;
430 };
431
432 static void
casueword_slow0(void * kva,void * arg)433 casueword_slow0(void *kva, void *arg)
434 {
435 struct casueword_arg0 *ca;
436
437 ca = arg;
438 ca->res = 1 - atomic_fcmpset_int((u_int *)kva, &ca->oldval,
439 ca->newval);
440 }
441
442 int
casueword32(volatile uint32_t * base,uint32_t oldval,uint32_t * oldvalp,uint32_t newval)443 casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp,
444 uint32_t newval)
445 {
446 struct casueword_arg0 ca;
447 int res;
448
449 ca.oldval = oldval;
450 ca.newval = newval;
451 res = cp_slow0((vm_offset_t)base, sizeof(int32_t), true,
452 casueword_slow0, &ca);
453 if (res == 0) {
454 *oldvalp = ca.oldval;
455 return (ca.res);
456 }
457 return (-1);
458 }
459
460 int
casueword(volatile u_long * base,u_long oldval,u_long * oldvalp,u_long newval)461 casueword(volatile u_long *base, u_long oldval, u_long *oldvalp, u_long newval)
462 {
463 struct casueword_arg0 ca;
464 int res;
465
466 ca.oldval = oldval;
467 ca.newval = newval;
468 res = cp_slow0((vm_offset_t)base, sizeof(int32_t), true,
469 casueword_slow0, &ca);
470 if (res == 0) {
471 *oldvalp = ca.oldval;
472 return (ca.res);
473 }
474 return (-1);
475 }
476