1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2007 Seccuris Inc.
5 * All rights reserved.
6 *
7 * This software was developed by Robert N. M. Watson under contract to
8 * Seccuris Inc.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32 #include <sys/cdefs.h>
33 #include "opt_bpf.h"
34
35 #include <sys/param.h>
36 #include <sys/lock.h>
37 #include <sys/malloc.h>
38 #include <sys/mbuf.h>
39 #include <sys/mutex.h>
40 #include <sys/proc.h>
41 #include <sys/sf_buf.h>
42 #include <sys/socket.h>
43 #include <sys/uio.h>
44
45 #include <machine/atomic.h>
46
47 #include <net/if.h>
48 #include <net/bpf.h>
49 #include <net/bpf_zerocopy.h>
50 #include <net/bpfdesc.h>
51
52 #include <vm/vm.h>
53 #include <vm/vm_param.h>
54 #include <vm/pmap.h>
55 #include <vm/vm_extern.h>
56 #include <vm/vm_map.h>
57 #include <vm/vm_page.h>
58
59 /*
60 * Zero-copy buffer scheme for BPF: user space "donates" two buffers, which
61 * are mapped into the kernel address space using sf_bufs and used directly
62 * by BPF. Memory is wired since page faults cannot be tolerated in the
63 * contexts where the buffers are copied to (locks held, interrupt context,
64 * etc). Access to shared memory buffers is synchronized using a header on
65 * each buffer, allowing the number of system calls to go to zero as BPF
66 * reaches saturation (buffers filled as fast as they can be drained by the
67 * user process). Full details of the protocol for communicating between the
68 * user process and BPF may be found in bpf(4).
69 */
70
71 /*
72 * Maximum number of pages per buffer. Since all BPF devices use two, the
73 * maximum per device is 2*BPF_MAX_PAGES. Resource limits on the number of
74 * sf_bufs may be an issue, so do not set this too high. On older systems,
75 * kernel address space limits may also be an issue.
76 */
77 #define BPF_MAX_PAGES 512
78
79 /*
80 * struct zbuf describes a memory buffer loaned by a user process to the
81 * kernel. We represent this as a series of pages managed using an array of
82 * sf_bufs. Even though the memory is contiguous in user space, it may not
83 * be mapped contiguously in the kernel (i.e., a set of physically
84 * non-contiguous pages in the direct map region) so we must implement
85 * scatter-gather copying. One significant mitigating factor is that on
86 * systems with a direct memory map, we can avoid TLB misses.
87 *
88 * At the front of the shared memory region is a bpf_zbuf_header, which
89 * contains shared control data to allow user space and the kernel to
90 * synchronize; this is included in zb_size, but not bpf_bufsize, so that BPF
91 * knows that the space is not available.
92 */
93 struct zbuf {
94 vm_offset_t zb_uaddr; /* User address at time of setup. */
95 size_t zb_size; /* Size of buffer, incl. header. */
96 u_int zb_numpages; /* Number of pages. */
97 int zb_flags; /* Flags on zbuf. */
98 struct sf_buf **zb_pages; /* Pages themselves. */
99 struct bpf_zbuf_header *zb_header; /* Shared header. */
100 };
101
102 /*
103 * When a buffer has been assigned to userspace, flag it as such, as the
104 * buffer may remain in the store position as a result of the user process
105 * not yet having acknowledged the buffer in the hold position yet.
106 */
107 #define ZBUF_FLAG_ASSIGNED 0x00000001 /* Set when owned by user. */
108
109 /*
110 * Release a page we've previously wired.
111 */
112 static void
zbuf_page_free(vm_page_t pp)113 zbuf_page_free(vm_page_t pp)
114 {
115
116 vm_page_unwire(pp, PQ_INACTIVE);
117 }
118
119 /*
120 * Free an sf_buf with attached page.
121 */
122 static void
zbuf_sfbuf_free(struct sf_buf * sf)123 zbuf_sfbuf_free(struct sf_buf *sf)
124 {
125 vm_page_t pp;
126
127 pp = sf_buf_page(sf);
128 sf_buf_free(sf);
129 zbuf_page_free(pp);
130 }
131
132 /*
133 * Free a zbuf, including its page array, sbufs, and pages. Allow partially
134 * allocated zbufs to be freed so that it may be used even during a zbuf
135 * setup.
136 */
137 static void
zbuf_free(struct zbuf * zb)138 zbuf_free(struct zbuf *zb)
139 {
140 int i;
141
142 for (i = 0; i < zb->zb_numpages; i++) {
143 if (zb->zb_pages[i] != NULL)
144 zbuf_sfbuf_free(zb->zb_pages[i]);
145 }
146 free(zb->zb_pages, M_BPF);
147 free(zb, M_BPF);
148 }
149
150 /*
151 * Given a user pointer to a page of user memory, return an sf_buf for the
152 * page. Because we may be requesting quite a few sf_bufs, prefer failure to
153 * deadlock and use SFB_NOWAIT.
154 */
155 static struct sf_buf *
zbuf_sfbuf_get(struct vm_map * map,vm_offset_t uaddr)156 zbuf_sfbuf_get(struct vm_map *map, vm_offset_t uaddr)
157 {
158 struct sf_buf *sf;
159 vm_page_t pp;
160
161 if (vm_fault_quick_hold_pages(map, uaddr, PAGE_SIZE, VM_PROT_READ |
162 VM_PROT_WRITE, &pp, 1) < 0)
163 return (NULL);
164 sf = sf_buf_alloc(pp, SFB_NOWAIT);
165 if (sf == NULL) {
166 zbuf_page_free(pp);
167 return (NULL);
168 }
169 return (sf);
170 }
171
172 /*
173 * Create a zbuf describing a range of user address space memory. Validate
174 * page alignment, size requirements, etc.
175 */
176 static int
zbuf_setup(struct thread * td,vm_offset_t uaddr,size_t len,struct zbuf ** zbp)177 zbuf_setup(struct thread *td, vm_offset_t uaddr, size_t len,
178 struct zbuf **zbp)
179 {
180 struct zbuf *zb;
181 struct vm_map *map;
182 int error, i;
183
184 *zbp = NULL;
185
186 /*
187 * User address must be page-aligned.
188 */
189 if (uaddr & PAGE_MASK)
190 return (EINVAL);
191
192 /*
193 * Length must be an integer number of full pages.
194 */
195 if (len & PAGE_MASK)
196 return (EINVAL);
197
198 /*
199 * Length must not exceed per-buffer resource limit.
200 */
201 if ((len / PAGE_SIZE) > BPF_MAX_PAGES)
202 return (EINVAL);
203
204 /*
205 * Allocate the buffer and set up each page with is own sf_buf.
206 */
207 error = 0;
208 zb = malloc(sizeof(*zb), M_BPF, M_ZERO | M_WAITOK);
209 zb->zb_uaddr = uaddr;
210 zb->zb_size = len;
211 zb->zb_numpages = len / PAGE_SIZE;
212 zb->zb_pages = malloc(sizeof(struct sf_buf *) *
213 zb->zb_numpages, M_BPF, M_ZERO | M_WAITOK);
214 map = &td->td_proc->p_vmspace->vm_map;
215 for (i = 0; i < zb->zb_numpages; i++) {
216 zb->zb_pages[i] = zbuf_sfbuf_get(map,
217 uaddr + (i * PAGE_SIZE));
218 if (zb->zb_pages[i] == NULL) {
219 error = EFAULT;
220 goto error;
221 }
222 }
223 zb->zb_header = sf_buf_kva(zb->zb_pages[0]);
224 bzero(zb->zb_header, sizeof(*zb->zb_header));
225 *zbp = zb;
226 return (0);
227
228 error:
229 zbuf_free(zb);
230 return (error);
231 }
232
233 /*
234 * Copy bytes from a source into the specified zbuf. The caller is
235 * responsible for performing bounds checking, etc.
236 */
237 void
bpf_zerocopy_append_bytes(struct bpf_d * d,caddr_t buf,u_int offset,void * src,u_int len)238 bpf_zerocopy_append_bytes(struct bpf_d *d, caddr_t buf, u_int offset,
239 void *src, u_int len)
240 {
241 u_int count, page, poffset;
242 u_char *src_bytes;
243 struct zbuf *zb;
244
245 KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
246 ("bpf_zerocopy_append_bytes: not in zbuf mode"));
247 KASSERT(buf != NULL, ("bpf_zerocopy_append_bytes: NULL buf"));
248
249 src_bytes = (u_char *)src;
250 zb = (struct zbuf *)buf;
251
252 KASSERT((zb->zb_flags & ZBUF_FLAG_ASSIGNED) == 0,
253 ("bpf_zerocopy_append_bytes: ZBUF_FLAG_ASSIGNED"));
254
255 /*
256 * Scatter-gather copy to user pages mapped into kernel address space
257 * using sf_bufs: copy up to a page at a time.
258 */
259 offset += sizeof(struct bpf_zbuf_header);
260 page = offset / PAGE_SIZE;
261 poffset = offset % PAGE_SIZE;
262 while (len > 0) {
263 KASSERT(page < zb->zb_numpages, ("bpf_zerocopy_append_bytes:"
264 " page overflow (%d p %d np)\n", page, zb->zb_numpages));
265
266 count = min(len, PAGE_SIZE - poffset);
267 bcopy(src_bytes, ((u_char *)sf_buf_kva(zb->zb_pages[page])) +
268 poffset, count);
269 poffset += count;
270 if (poffset == PAGE_SIZE) {
271 poffset = 0;
272 page++;
273 }
274 KASSERT(poffset < PAGE_SIZE,
275 ("bpf_zerocopy_append_bytes: page offset overflow (%d)",
276 poffset));
277 len -= count;
278 src_bytes += count;
279 }
280 }
281
282 /*
283 * Copy bytes from an mbuf chain to the specified zbuf: copying will be
284 * scatter-gather both from mbufs, which may be fragmented over memory, and
285 * to pages, which may not be contiguously mapped in kernel address space.
286 * As with bpf_zerocopy_append_bytes(), the caller is responsible for
287 * checking that this will not exceed the buffer limit.
288 */
289 void
bpf_zerocopy_append_mbuf(struct bpf_d * d,caddr_t buf,u_int offset,void * src,u_int len)290 bpf_zerocopy_append_mbuf(struct bpf_d *d, caddr_t buf, u_int offset,
291 void *src, u_int len)
292 {
293 u_int count, moffset, page, poffset;
294 const struct mbuf *m;
295 struct zbuf *zb;
296
297 KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
298 ("bpf_zerocopy_append_mbuf not in zbuf mode"));
299 KASSERT(buf != NULL, ("bpf_zerocopy_append_mbuf: NULL buf"));
300
301 m = (struct mbuf *)src;
302 zb = (struct zbuf *)buf;
303
304 KASSERT((zb->zb_flags & ZBUF_FLAG_ASSIGNED) == 0,
305 ("bpf_zerocopy_append_mbuf: ZBUF_FLAG_ASSIGNED"));
306
307 /*
308 * Scatter gather both from an mbuf chain and to a user page set
309 * mapped into kernel address space using sf_bufs. If we're lucky,
310 * each mbuf requires one copy operation, but if page alignment and
311 * mbuf alignment work out less well, we'll be doing two copies per
312 * mbuf.
313 */
314 offset += sizeof(struct bpf_zbuf_header);
315 page = offset / PAGE_SIZE;
316 poffset = offset % PAGE_SIZE;
317 moffset = 0;
318 while (len > 0) {
319 KASSERT(page < zb->zb_numpages,
320 ("bpf_zerocopy_append_mbuf: page overflow (%d p %d "
321 "np)\n", page, zb->zb_numpages));
322 KASSERT(m != NULL,
323 ("bpf_zerocopy_append_mbuf: end of mbuf chain"));
324
325 count = min(m->m_len - moffset, len);
326 count = min(count, PAGE_SIZE - poffset);
327 bcopy(mtod(m, u_char *) + moffset,
328 ((u_char *)sf_buf_kva(zb->zb_pages[page])) + poffset,
329 count);
330 poffset += count;
331 if (poffset == PAGE_SIZE) {
332 poffset = 0;
333 page++;
334 }
335 KASSERT(poffset < PAGE_SIZE,
336 ("bpf_zerocopy_append_mbuf: page offset overflow (%d)",
337 poffset));
338 moffset += count;
339 if (moffset == m->m_len) {
340 m = m->m_next;
341 moffset = 0;
342 }
343 len -= count;
344 }
345 }
346
347 /*
348 * Notification from the BPF framework that a buffer in the store position is
349 * rejecting packets and may be considered full. We mark the buffer as
350 * immutable and assign to userspace so that it is immediately available for
351 * the user process to access.
352 */
353 void
bpf_zerocopy_buffull(struct bpf_d * d)354 bpf_zerocopy_buffull(struct bpf_d *d)
355 {
356 struct zbuf *zb;
357
358 KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
359 ("bpf_zerocopy_buffull: not in zbuf mode"));
360
361 zb = (struct zbuf *)d->bd_sbuf;
362 KASSERT(zb != NULL, ("bpf_zerocopy_buffull: zb == NULL"));
363
364 if ((zb->zb_flags & ZBUF_FLAG_ASSIGNED) == 0) {
365 zb->zb_flags |= ZBUF_FLAG_ASSIGNED;
366 zb->zb_header->bzh_kernel_len = d->bd_slen;
367 atomic_add_rel_int(&zb->zb_header->bzh_kernel_gen, 1);
368 }
369 }
370
371 /*
372 * Notification from the BPF framework that a buffer has moved into the held
373 * slot on a descriptor. Zero-copy BPF will update the shared page to let
374 * the user process know and flag the buffer as assigned if it hasn't already
375 * been marked assigned due to filling while it was in the store position.
376 *
377 * Note: identical logic as in bpf_zerocopy_buffull(), except that we operate
378 * on bd_hbuf and bd_hlen.
379 */
380 void
bpf_zerocopy_bufheld(struct bpf_d * d)381 bpf_zerocopy_bufheld(struct bpf_d *d)
382 {
383 struct zbuf *zb;
384
385 KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
386 ("bpf_zerocopy_bufheld: not in zbuf mode"));
387
388 zb = (struct zbuf *)d->bd_hbuf;
389 KASSERT(zb != NULL, ("bpf_zerocopy_bufheld: zb == NULL"));
390
391 if ((zb->zb_flags & ZBUF_FLAG_ASSIGNED) == 0) {
392 zb->zb_flags |= ZBUF_FLAG_ASSIGNED;
393 zb->zb_header->bzh_kernel_len = d->bd_hlen;
394 atomic_add_rel_int(&zb->zb_header->bzh_kernel_gen, 1);
395 }
396 }
397
398 /*
399 * Notification from the BPF framework that the free buffer has been been
400 * rotated out of the held position to the free position. This happens when
401 * the user acknowledges the held buffer.
402 */
403 void
bpf_zerocopy_buf_reclaimed(struct bpf_d * d)404 bpf_zerocopy_buf_reclaimed(struct bpf_d *d)
405 {
406 struct zbuf *zb;
407
408 KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
409 ("bpf_zerocopy_reclaim_buf: not in zbuf mode"));
410
411 KASSERT(d->bd_fbuf != NULL,
412 ("bpf_zerocopy_buf_reclaimed: NULL free buf"));
413 zb = (struct zbuf *)d->bd_fbuf;
414 zb->zb_flags &= ~ZBUF_FLAG_ASSIGNED;
415 }
416
417 /*
418 * Query from the BPF framework regarding whether the buffer currently in the
419 * held position can be moved to the free position, which can be indicated by
420 * the user process making their generation number equal to the kernel
421 * generation number.
422 */
423 int
bpf_zerocopy_canfreebuf(struct bpf_d * d)424 bpf_zerocopy_canfreebuf(struct bpf_d *d)
425 {
426 struct zbuf *zb;
427
428 KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
429 ("bpf_zerocopy_canfreebuf: not in zbuf mode"));
430
431 zb = (struct zbuf *)d->bd_hbuf;
432 if (zb == NULL)
433 return (0);
434 if (zb->zb_header->bzh_kernel_gen ==
435 atomic_load_acq_int(&zb->zb_header->bzh_user_gen))
436 return (1);
437 return (0);
438 }
439
440 /*
441 * Query from the BPF framework as to whether or not the buffer current in
442 * the store position can actually be written to. This may return false if
443 * the store buffer is assigned to userspace before the hold buffer is
444 * acknowledged.
445 */
446 int
bpf_zerocopy_canwritebuf(struct bpf_d * d)447 bpf_zerocopy_canwritebuf(struct bpf_d *d)
448 {
449 struct zbuf *zb;
450
451 KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
452 ("bpf_zerocopy_canwritebuf: not in zbuf mode"));
453
454 zb = (struct zbuf *)d->bd_sbuf;
455 KASSERT(zb != NULL, ("bpf_zerocopy_canwritebuf: bd_sbuf NULL"));
456
457 if (zb->zb_flags & ZBUF_FLAG_ASSIGNED)
458 return (0);
459 return (1);
460 }
461
462 /*
463 * Free zero copy buffers at request of descriptor.
464 */
465 void
bpf_zerocopy_free(struct bpf_d * d)466 bpf_zerocopy_free(struct bpf_d *d)
467 {
468 struct zbuf *zb;
469
470 KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
471 ("bpf_zerocopy_free: not in zbuf mode"));
472
473 zb = (struct zbuf *)d->bd_sbuf;
474 if (zb != NULL)
475 zbuf_free(zb);
476 zb = (struct zbuf *)d->bd_hbuf;
477 if (zb != NULL)
478 zbuf_free(zb);
479 zb = (struct zbuf *)d->bd_fbuf;
480 if (zb != NULL)
481 zbuf_free(zb);
482 }
483
484 /*
485 * Ioctl to return the maximum buffer size.
486 */
487 int
bpf_zerocopy_ioctl_getzmax(struct thread * td,struct bpf_d * d,size_t * i)488 bpf_zerocopy_ioctl_getzmax(struct thread *td, struct bpf_d *d, size_t *i)
489 {
490
491 KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
492 ("bpf_zerocopy_ioctl_getzmax: not in zbuf mode"));
493
494 *i = BPF_MAX_PAGES * PAGE_SIZE;
495 return (0);
496 }
497
498 /*
499 * Ioctl to force rotation of the two buffers, if there's any data available.
500 * This can be used by user space to implement timeouts when waiting for a
501 * buffer to fill.
502 */
503 int
bpf_zerocopy_ioctl_rotzbuf(struct thread * td,struct bpf_d * d,struct bpf_zbuf * bz)504 bpf_zerocopy_ioctl_rotzbuf(struct thread *td, struct bpf_d *d,
505 struct bpf_zbuf *bz)
506 {
507 struct zbuf *bzh;
508
509 bzero(bz, sizeof(*bz));
510 BPFD_LOCK(d);
511 if (d->bd_hbuf == NULL && d->bd_slen != 0) {
512 ROTATE_BUFFERS(d);
513 bzh = (struct zbuf *)d->bd_hbuf;
514 bz->bz_bufa = (void *)bzh->zb_uaddr;
515 bz->bz_buflen = d->bd_hlen;
516 }
517 BPFD_UNLOCK(d);
518 return (0);
519 }
520
521 /*
522 * Ioctl to configure zero-copy buffers -- may be done only once.
523 */
524 int
bpf_zerocopy_ioctl_setzbuf(struct thread * td,struct bpf_d * d,struct bpf_zbuf * bz)525 bpf_zerocopy_ioctl_setzbuf(struct thread *td, struct bpf_d *d,
526 struct bpf_zbuf *bz)
527 {
528 struct zbuf *zba, *zbb;
529 int error;
530
531 KASSERT(d->bd_bufmode == BPF_BUFMODE_ZBUF,
532 ("bpf_zerocopy_ioctl_setzbuf: not in zbuf mode"));
533
534 /*
535 * Must set both buffers. Cannot clear them.
536 */
537 if (bz->bz_bufa == NULL || bz->bz_bufb == NULL)
538 return (EINVAL);
539
540 /*
541 * Buffers must have a size greater than 0. Alignment and other size
542 * validity checking is done in zbuf_setup().
543 */
544 if (bz->bz_buflen == 0)
545 return (EINVAL);
546
547 /*
548 * Allocate new buffers.
549 */
550 error = zbuf_setup(td, (vm_offset_t)bz->bz_bufa, bz->bz_buflen,
551 &zba);
552 if (error)
553 return (error);
554 error = zbuf_setup(td, (vm_offset_t)bz->bz_bufb, bz->bz_buflen,
555 &zbb);
556 if (error) {
557 zbuf_free(zba);
558 return (error);
559 }
560
561 /*
562 * We only allow buffers to be installed once, so atomically check
563 * that no buffers are currently installed and install new buffers.
564 */
565 BPFD_LOCK(d);
566 if (d->bd_hbuf != NULL || d->bd_sbuf != NULL || d->bd_fbuf != NULL ||
567 d->bd_bif != NULL) {
568 BPFD_UNLOCK(d);
569 zbuf_free(zba);
570 zbuf_free(zbb);
571 return (EINVAL);
572 }
573
574 /*
575 * Point BPF descriptor at buffers; initialize sbuf as zba so that
576 * it is always filled first in the sequence, per bpf(4).
577 */
578 d->bd_fbuf = (caddr_t)zbb;
579 d->bd_sbuf = (caddr_t)zba;
580 d->bd_slen = 0;
581 d->bd_hlen = 0;
582
583 /*
584 * We expose only the space left in the buffer after the size of the
585 * shared management region.
586 */
587 d->bd_bufsize = bz->bz_buflen - sizeof(struct bpf_zbuf_header);
588 BPFD_UNLOCK(d);
589 return (0);
590 }
591