1 /* $OpenBSD: criov.c,v 1.9 2002/01/29 15:48:29 jason Exp $ */
2
3 /*-
4 * Copyright (c) 1999 Theo de Raadt
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. The name of the author may not be used to endorse or promote products
16 * derived from this software without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/proc.h>
33 #include <sys/errno.h>
34 #include <sys/malloc.h>
35 #include <sys/kernel.h>
36 #include <sys/mbuf.h>
37 #include <sys/uio.h>
38 #include <sys/limits.h>
39 #include <sys/lock.h>
40 #include <sys/sdt.h>
41
42 #include <machine/vmparam.h>
43
44 #include <vm/vm.h>
45 #include <vm/vm_page.h>
46 #include <vm/pmap.h>
47
48 #include <opencrypto/cryptodev.h>
49
50 SDT_PROVIDER_DECLARE(opencrypto);
51
52 /*
53 * These macros are only for avoiding code duplication, as we need to skip
54 * given number of bytes in the same way in several functions below.
55 */
56 #define CUIO_SKIP() do { \
57 KASSERT(off >= 0, ("%s: off %d < 0", __func__, off)); \
58 KASSERT(len >= 0, ("%s: len %d < 0", __func__, len)); \
59 while (off > 0) { \
60 KASSERT(iol >= 0, ("%s: empty in skip", __func__)); \
61 if (off < iov->iov_len) \
62 break; \
63 off -= iov->iov_len; \
64 iol--; \
65 iov++; \
66 } \
67 } while (0)
68
69 #define CVM_PAGE_SKIP() do { \
70 KASSERT(off >= 0, ("%s: off %d < 0", __func__, off)); \
71 KASSERT(len >= 0, ("%s: len %d < 0", __func__, len)); \
72 while (off > 0) { \
73 if (off < PAGE_SIZE) \
74 break; \
75 processed += PAGE_SIZE - off; \
76 off -= PAGE_SIZE - off; \
77 pages++; \
78 } \
79 } while (0)
80
81 static void
cuio_copydata(struct uio * uio,int off,int len,caddr_t cp)82 cuio_copydata(struct uio* uio, int off, int len, caddr_t cp)
83 {
84 struct iovec *iov = uio->uio_iov;
85 int iol __diagused = uio->uio_iovcnt;
86 unsigned count;
87
88 CUIO_SKIP();
89 while (len > 0) {
90 KASSERT(iol >= 0, ("%s: empty", __func__));
91 count = min(iov->iov_len - off, len);
92 bcopy(((caddr_t)iov->iov_base) + off, cp, count);
93 len -= count;
94 cp += count;
95 off = 0;
96 iol--;
97 iov++;
98 }
99 }
100
101 static void
cuio_copyback(struct uio * uio,int off,int len,c_caddr_t cp)102 cuio_copyback(struct uio* uio, int off, int len, c_caddr_t cp)
103 {
104 struct iovec *iov = uio->uio_iov;
105 int iol __diagused = uio->uio_iovcnt;
106 unsigned count;
107
108 CUIO_SKIP();
109 while (len > 0) {
110 KASSERT(iol >= 0, ("%s: empty", __func__));
111 count = min(iov->iov_len - off, len);
112 bcopy(cp, ((caddr_t)iov->iov_base) + off, count);
113 len -= count;
114 cp += count;
115 off = 0;
116 iol--;
117 iov++;
118 }
119 }
120
121 /*
122 * Return the index and offset of location in iovec list.
123 */
124 static int
cuio_getptr(struct uio * uio,int loc,int * off)125 cuio_getptr(struct uio *uio, int loc, int *off)
126 {
127 int ind, len;
128
129 ind = 0;
130 while (loc >= 0 && ind < uio->uio_iovcnt) {
131 len = uio->uio_iov[ind].iov_len;
132 if (len > loc) {
133 *off = loc;
134 return (ind);
135 }
136 loc -= len;
137 ind++;
138 }
139
140 if (ind > 0 && loc == 0) {
141 ind--;
142 *off = uio->uio_iov[ind].iov_len;
143 return (ind);
144 }
145
146 return (-1);
147 }
148
149 #if CRYPTO_MAY_HAVE_VMPAGE
150 /*
151 * Apply function f to the data in a vm_page_t list starting "off" bytes from
152 * the beginning, continuing for "len" bytes.
153 */
154 static int
cvm_page_apply(vm_page_t * pages,int off,int len,int (* f)(void *,const void *,u_int),void * arg)155 cvm_page_apply(vm_page_t *pages, int off, int len,
156 int (*f)(void *, const void *, u_int), void *arg)
157 {
158 int processed __unused;
159 unsigned count;
160 int rval;
161
162 processed = 0;
163 CVM_PAGE_SKIP();
164 while (len > 0) {
165 char *kaddr = VM_PAGE_TO_DMAP(*pages);
166 count = min(PAGE_SIZE - off, len);
167 rval = (*f)(arg, kaddr + off, count);
168 if (rval)
169 return (rval);
170 len -= count;
171 processed += count;
172 off = 0;
173 pages++;
174 }
175 return (0);
176 }
177
178 static inline void *
cvm_page_contiguous_segment(vm_page_t * pages,size_t skip,int len)179 cvm_page_contiguous_segment(vm_page_t *pages, size_t skip, int len)
180 {
181 if ((skip + len - 1) / PAGE_SIZE > skip / PAGE_SIZE)
182 return (NULL);
183
184 pages += (skip / PAGE_SIZE);
185 skip -= rounddown(skip, PAGE_SIZE);
186 return ((char *)VM_PAGE_TO_DMAP(*pages) + skip);
187 }
188
189 /*
190 * Copy len bytes of data from the vm_page_t array, skipping the first off
191 * bytes, into the pointer cp. Return the number of bytes skipped and copied.
192 * Does not verify the length of the array.
193 */
194 static int
cvm_page_copyback(vm_page_t * pages,int off,int len,c_caddr_t cp)195 cvm_page_copyback(vm_page_t *pages, int off, int len, c_caddr_t cp)
196 {
197 int processed = 0;
198 unsigned count;
199
200 CVM_PAGE_SKIP();
201 while (len > 0) {
202 count = min(PAGE_SIZE - off, len);
203 bcopy(cp, (char *)VM_PAGE_TO_DMAP(*pages) + off, count);
204 len -= count;
205 cp += count;
206 processed += count;
207 off = 0;
208 pages++;
209 }
210 return (processed);
211 }
212
213 /*
214 * Copy len bytes of data from the pointer cp into the vm_page_t array,
215 * skipping the first off bytes, Return the number of bytes skipped and copied.
216 * Does not verify the length of the array.
217 */
218 static int
cvm_page_copydata(vm_page_t * pages,int off,int len,caddr_t cp)219 cvm_page_copydata(vm_page_t *pages, int off, int len, caddr_t cp)
220 {
221 int processed = 0;
222 unsigned count;
223
224 CVM_PAGE_SKIP();
225 while (len > 0) {
226 count = min(PAGE_SIZE - off, len);
227 bcopy((char *)VM_PAGE_TO_DMAP(*pages) + off, cp, count);
228 len -= count;
229 cp += count;
230 processed += count;
231 off = 0;
232 pages++;
233 }
234 return processed;
235 }
236 #endif /* CRYPTO_MAY_HAVE_VMPAGE */
237
238 /*
239 * Given a starting page in an m_epg, determine the length of the
240 * current physically contiguous segment.
241 */
242 static __inline size_t
m_epg_pages_extent(struct mbuf * m,int idx,u_int pglen)243 m_epg_pages_extent(struct mbuf *m, int idx, u_int pglen)
244 {
245 size_t len;
246 u_int i;
247
248 len = pglen;
249 for (i = idx + 1; i < m->m_epg_npgs; i++) {
250 if (m->m_epg_pa[i - 1] + PAGE_SIZE != m->m_epg_pa[i])
251 break;
252 len += m_epg_pagelen(m, i, 0);
253 }
254 return (len);
255 }
256
257 static void *
m_epg_segment(struct mbuf * m,size_t offset,size_t * len)258 m_epg_segment(struct mbuf *m, size_t offset, size_t *len)
259 {
260 u_int i, pglen, pgoff;
261
262 offset += mtod(m, vm_offset_t);
263 if (offset < m->m_epg_hdrlen) {
264 *len = m->m_epg_hdrlen - offset;
265 return (m->m_epg_hdr + offset);
266 }
267 offset -= m->m_epg_hdrlen;
268 pgoff = m->m_epg_1st_off;
269 for (i = 0; i < m->m_epg_npgs; i++) {
270 pglen = m_epg_pagelen(m, i, pgoff);
271 if (offset < pglen) {
272 *len = m_epg_pages_extent(m, i, pglen) - offset;
273 return (PHYS_TO_DMAP(m->m_epg_pa[i] + pgoff +
274 offset));
275 }
276 offset -= pglen;
277 pgoff = 0;
278 }
279 KASSERT(offset <= m->m_epg_trllen, ("%s: offset beyond trailer",
280 __func__));
281 *len = m->m_epg_trllen - offset;
282 return (m->m_epg_trail + offset);
283 }
284
285 static __inline void *
m_epg_contiguous_subsegment(struct mbuf * m,size_t skip,size_t len)286 m_epg_contiguous_subsegment(struct mbuf *m, size_t skip, size_t len)
287 {
288 void *base;
289 size_t seglen;
290
291 base = m_epg_segment(m, skip, &seglen);
292 if (len > seglen)
293 return (NULL);
294 return (base);
295 }
296
297 void
crypto_cursor_init(struct crypto_buffer_cursor * cc,const struct crypto_buffer * cb)298 crypto_cursor_init(struct crypto_buffer_cursor *cc,
299 const struct crypto_buffer *cb)
300 {
301 memset(cc, 0, sizeof(*cc));
302 cc->cc_type = cb->cb_type;
303 switch (cc->cc_type) {
304 case CRYPTO_BUF_CONTIG:
305 cc->cc_buf = cb->cb_buf;
306 cc->cc_buf_len = cb->cb_buf_len;
307 break;
308 case CRYPTO_BUF_MBUF:
309 case CRYPTO_BUF_SINGLE_MBUF:
310 cc->cc_mbuf = cb->cb_mbuf;
311 break;
312 case CRYPTO_BUF_VMPAGE:
313 cc->cc_vmpage = cb->cb_vm_page;
314 cc->cc_buf_len = cb->cb_vm_page_len;
315 cc->cc_offset = cb->cb_vm_page_offset;
316 break;
317 case CRYPTO_BUF_UIO:
318 cc->cc_iov = cb->cb_uio->uio_iov;
319 cc->cc_buf_len = cb->cb_uio->uio_resid;
320 break;
321 default:
322 #ifdef INVARIANTS
323 panic("%s: invalid buffer type %d", __func__, cb->cb_type);
324 #endif
325 break;
326 }
327 }
328
329 SDT_PROBE_DEFINE2(opencrypto, criov, cursor_advance, vmpage, "struct crypto_buffer_cursor*", "size_t");
330
331 void
crypto_cursor_advance(struct crypto_buffer_cursor * cc,size_t amount)332 crypto_cursor_advance(struct crypto_buffer_cursor *cc, size_t amount)
333 {
334 size_t remain;
335
336 switch (cc->cc_type) {
337 case CRYPTO_BUF_CONTIG:
338 MPASS(cc->cc_buf_len >= amount);
339 cc->cc_buf += amount;
340 cc->cc_buf_len -= amount;
341 break;
342 case CRYPTO_BUF_MBUF:
343 for (;;) {
344 remain = cc->cc_mbuf->m_len - cc->cc_offset;
345 if (amount < remain) {
346 cc->cc_offset += amount;
347 break;
348 }
349 amount -= remain;
350 cc->cc_mbuf = cc->cc_mbuf->m_next;
351 cc->cc_offset = 0;
352 if (amount == 0)
353 break;
354 }
355 break;
356 case CRYPTO_BUF_SINGLE_MBUF:
357 MPASS(cc->cc_mbuf->m_len >= cc->cc_offset + amount);
358 cc->cc_offset += amount;
359 break;
360 case CRYPTO_BUF_VMPAGE:
361 for (;;) {
362 SDT_PROBE2(opencrypto, criov, cursor_advance, vmpage,
363 cc, amount);
364 remain = MIN(PAGE_SIZE - cc->cc_offset, cc->cc_buf_len);
365 if (amount < remain) {
366 cc->cc_buf_len -= amount;
367 cc->cc_offset += amount;
368 break;
369 }
370 cc->cc_buf_len -= remain;
371 amount -= remain;
372 cc->cc_vmpage++;
373 cc->cc_offset = 0;
374 if (amount == 0 || cc->cc_buf_len == 0)
375 break;
376 }
377 break;
378 case CRYPTO_BUF_UIO:
379 for (;;) {
380 remain = cc->cc_iov->iov_len - cc->cc_offset;
381 if (amount < remain) {
382 cc->cc_offset += amount;
383 break;
384 }
385 cc->cc_buf_len -= remain;
386 amount -= remain;
387 cc->cc_iov++;
388 cc->cc_offset = 0;
389 if (amount == 0)
390 break;
391 }
392 break;
393 default:
394 #ifdef INVARIANTS
395 panic("%s: invalid buffer type %d", __func__, cc->cc_type);
396 #endif
397 break;
398 }
399 }
400
401 void *
crypto_cursor_segment(struct crypto_buffer_cursor * cc,size_t * len)402 crypto_cursor_segment(struct crypto_buffer_cursor *cc, size_t *len)
403 {
404 switch (cc->cc_type) {
405 case CRYPTO_BUF_CONTIG:
406 case CRYPTO_BUF_UIO:
407 case CRYPTO_BUF_VMPAGE:
408 if (cc->cc_buf_len == 0) {
409 *len = 0;
410 return (NULL);
411 }
412 break;
413 case CRYPTO_BUF_MBUF:
414 case CRYPTO_BUF_SINGLE_MBUF:
415 if (cc->cc_mbuf == NULL) {
416 *len = 0;
417 return (NULL);
418 }
419 break;
420 default:
421 #ifdef INVARIANTS
422 panic("%s: invalid buffer type %d", __func__, cc->cc_type);
423 #endif
424 *len = 0;
425 return (NULL);
426 }
427
428 switch (cc->cc_type) {
429 case CRYPTO_BUF_CONTIG:
430 *len = cc->cc_buf_len;
431 return (cc->cc_buf);
432 case CRYPTO_BUF_MBUF:
433 case CRYPTO_BUF_SINGLE_MBUF:
434 if (cc->cc_mbuf->m_flags & M_EXTPG)
435 return (m_epg_segment(cc->cc_mbuf, cc->cc_offset, len));
436 *len = cc->cc_mbuf->m_len - cc->cc_offset;
437 return (mtod(cc->cc_mbuf, char *) + cc->cc_offset);
438 case CRYPTO_BUF_VMPAGE:
439 *len = PAGE_SIZE - cc->cc_offset;
440 return ((char *)VM_PAGE_TO_DMAP(*cc->cc_vmpage) +
441 cc->cc_offset);
442 case CRYPTO_BUF_UIO:
443 *len = cc->cc_iov->iov_len - cc->cc_offset;
444 return ((char *)cc->cc_iov->iov_base + cc->cc_offset);
445 default:
446 __assert_unreachable();
447 }
448 }
449
450 void
crypto_cursor_copyback(struct crypto_buffer_cursor * cc,int size,const void * vsrc)451 crypto_cursor_copyback(struct crypto_buffer_cursor *cc, int size,
452 const void *vsrc)
453 {
454 size_t remain, todo;
455 const char *src;
456 char *dst;
457
458 src = vsrc;
459 switch (cc->cc_type) {
460 case CRYPTO_BUF_CONTIG:
461 MPASS(cc->cc_buf_len >= size);
462 memcpy(cc->cc_buf, src, size);
463 cc->cc_buf += size;
464 cc->cc_buf_len -= size;
465 break;
466 case CRYPTO_BUF_MBUF:
467 for (;;) {
468 /*
469 * This uses m_copyback() for individual
470 * mbufs so that cc_mbuf and cc_offset are
471 * updated.
472 */
473 remain = cc->cc_mbuf->m_len - cc->cc_offset;
474 todo = MIN(remain, size);
475 m_copyback(cc->cc_mbuf, cc->cc_offset, todo, src);
476 src += todo;
477 if (todo < remain) {
478 cc->cc_offset += todo;
479 break;
480 }
481 size -= todo;
482 cc->cc_mbuf = cc->cc_mbuf->m_next;
483 cc->cc_offset = 0;
484 if (size == 0)
485 break;
486 }
487 break;
488 case CRYPTO_BUF_SINGLE_MBUF:
489 MPASS(cc->cc_mbuf->m_len >= cc->cc_offset + size);
490 m_copyback(cc->cc_mbuf, cc->cc_offset, size, src);
491 cc->cc_offset += size;
492 break;
493 case CRYPTO_BUF_VMPAGE:
494 for (;;) {
495 dst = (char *)VM_PAGE_TO_DMAP(*cc->cc_vmpage) +
496 cc->cc_offset;
497 remain = MIN(PAGE_SIZE - cc->cc_offset, cc->cc_buf_len);
498 todo = MIN(remain, size);
499 memcpy(dst, src, todo);
500 src += todo;
501 cc->cc_buf_len -= todo;
502 if (todo < remain) {
503 cc->cc_offset += todo;
504 break;
505 }
506 size -= todo;
507 cc->cc_vmpage++;
508 cc->cc_offset = 0;
509 if (size == 0)
510 break;
511 }
512 break;
513 case CRYPTO_BUF_UIO:
514 for (;;) {
515 dst = (char *)cc->cc_iov->iov_base + cc->cc_offset;
516 remain = cc->cc_iov->iov_len - cc->cc_offset;
517 todo = MIN(remain, size);
518 memcpy(dst, src, todo);
519 src += todo;
520 cc->cc_buf_len -= todo;
521 if (todo < remain) {
522 cc->cc_offset += todo;
523 break;
524 }
525 size -= todo;
526 cc->cc_iov++;
527 cc->cc_offset = 0;
528 if (size == 0)
529 break;
530 }
531 break;
532 default:
533 #ifdef INVARIANTS
534 panic("%s: invalid buffer type %d", __func__, cc->cc_type);
535 #endif
536 break;
537 }
538 }
539
540 void
crypto_cursor_copydata(struct crypto_buffer_cursor * cc,int size,void * vdst)541 crypto_cursor_copydata(struct crypto_buffer_cursor *cc, int size, void *vdst)
542 {
543 size_t remain, todo;
544 const char *src;
545 char *dst;
546
547 dst = vdst;
548 switch (cc->cc_type) {
549 case CRYPTO_BUF_CONTIG:
550 MPASS(cc->cc_buf_len >= size);
551 memcpy(dst, cc->cc_buf, size);
552 cc->cc_buf += size;
553 cc->cc_buf_len -= size;
554 break;
555 case CRYPTO_BUF_MBUF:
556 for (;;) {
557 /*
558 * This uses m_copydata() for individual
559 * mbufs so that cc_mbuf and cc_offset are
560 * updated.
561 */
562 remain = cc->cc_mbuf->m_len - cc->cc_offset;
563 todo = MIN(remain, size);
564 m_copydata(cc->cc_mbuf, cc->cc_offset, todo, dst);
565 dst += todo;
566 if (todo < remain) {
567 cc->cc_offset += todo;
568 break;
569 }
570 size -= todo;
571 cc->cc_mbuf = cc->cc_mbuf->m_next;
572 cc->cc_offset = 0;
573 if (size == 0)
574 break;
575 }
576 break;
577 case CRYPTO_BUF_SINGLE_MBUF:
578 MPASS(cc->cc_mbuf->m_len >= cc->cc_offset + size);
579 m_copydata(cc->cc_mbuf, cc->cc_offset, size, dst);
580 cc->cc_offset += size;
581 break;
582 case CRYPTO_BUF_VMPAGE:
583 for (;;) {
584 src = (char *)VM_PAGE_TO_DMAP(*cc->cc_vmpage) +
585 cc->cc_offset;
586 remain = MIN(PAGE_SIZE - cc->cc_offset, cc->cc_buf_len);
587 todo = MIN(remain, size);
588 memcpy(dst, src, todo);
589 dst += todo;
590 cc->cc_buf_len -= todo;
591 if (todo < remain) {
592 cc->cc_offset += todo;
593 break;
594 }
595 size -= todo;
596 cc->cc_vmpage++;
597 cc->cc_offset = 0;
598 if (size == 0)
599 break;
600 }
601 break;
602 case CRYPTO_BUF_UIO:
603 for (;;) {
604 src = (const char *)cc->cc_iov->iov_base +
605 cc->cc_offset;
606 remain = cc->cc_iov->iov_len - cc->cc_offset;
607 todo = MIN(remain, size);
608 memcpy(dst, src, todo);
609 dst += todo;
610 cc->cc_buf_len -= todo;
611 if (todo < remain) {
612 cc->cc_offset += todo;
613 break;
614 }
615 size -= todo;
616 cc->cc_iov++;
617 cc->cc_offset = 0;
618 if (size == 0)
619 break;
620 }
621 break;
622 default:
623 #ifdef INVARIANTS
624 panic("%s: invalid buffer type %d", __func__, cc->cc_type);
625 #endif
626 break;
627 }
628 }
629
630 /*
631 * To avoid advancing 'cursor', make a local copy that gets advanced
632 * instead.
633 */
634 void
crypto_cursor_copydata_noadv(struct crypto_buffer_cursor * cc,int size,void * vdst)635 crypto_cursor_copydata_noadv(struct crypto_buffer_cursor *cc, int size,
636 void *vdst)
637 {
638 struct crypto_buffer_cursor copy;
639
640 copy = *cc;
641 crypto_cursor_copydata(©, size, vdst);
642 }
643
644 /*
645 * Apply function f to the data in an iovec list starting "off" bytes from
646 * the beginning, continuing for "len" bytes.
647 */
648 static int
cuio_apply(struct uio * uio,int off,int len,int (* f)(void *,const void *,u_int),void * arg)649 cuio_apply(struct uio *uio, int off, int len,
650 int (*f)(void *, const void *, u_int), void *arg)
651 {
652 struct iovec *iov = uio->uio_iov;
653 int iol __diagused = uio->uio_iovcnt;
654 unsigned count;
655 int rval;
656
657 CUIO_SKIP();
658 while (len > 0) {
659 KASSERT(iol >= 0, ("%s: empty", __func__));
660 count = min(iov->iov_len - off, len);
661 rval = (*f)(arg, ((caddr_t)iov->iov_base) + off, count);
662 if (rval)
663 return (rval);
664 len -= count;
665 off = 0;
666 iol--;
667 iov++;
668 }
669 return (0);
670 }
671
672 void
crypto_copyback(struct cryptop * crp,int off,int size,const void * src)673 crypto_copyback(struct cryptop *crp, int off, int size, const void *src)
674 {
675 struct crypto_buffer *cb;
676
677 if (crp->crp_obuf.cb_type != CRYPTO_BUF_NONE)
678 cb = &crp->crp_obuf;
679 else
680 cb = &crp->crp_buf;
681 switch (cb->cb_type) {
682 case CRYPTO_BUF_MBUF:
683 case CRYPTO_BUF_SINGLE_MBUF:
684 m_copyback(cb->cb_mbuf, off, size, src);
685 break;
686 #if CRYPTO_MAY_HAVE_VMPAGE
687 case CRYPTO_BUF_VMPAGE:
688 MPASS(size <= cb->cb_vm_page_len);
689 MPASS(size + off <=
690 cb->cb_vm_page_len + cb->cb_vm_page_offset);
691 cvm_page_copyback(cb->cb_vm_page,
692 off + cb->cb_vm_page_offset, size, src);
693 break;
694 #endif /* CRYPTO_MAY_HAVE_VMPAGE */
695 case CRYPTO_BUF_UIO:
696 cuio_copyback(cb->cb_uio, off, size, src);
697 break;
698 case CRYPTO_BUF_CONTIG:
699 MPASS(off + size <= cb->cb_buf_len);
700 bcopy(src, cb->cb_buf + off, size);
701 break;
702 default:
703 #ifdef INVARIANTS
704 panic("invalid crp buf type %d", cb->cb_type);
705 #endif
706 break;
707 }
708 }
709
710 void
crypto_copydata(struct cryptop * crp,int off,int size,void * dst)711 crypto_copydata(struct cryptop *crp, int off, int size, void *dst)
712 {
713
714 switch (crp->crp_buf.cb_type) {
715 case CRYPTO_BUF_MBUF:
716 case CRYPTO_BUF_SINGLE_MBUF:
717 m_copydata(crp->crp_buf.cb_mbuf, off, size, dst);
718 break;
719 #if CRYPTO_MAY_HAVE_VMPAGE
720 case CRYPTO_BUF_VMPAGE:
721 MPASS(size <= crp->crp_buf.cb_vm_page_len);
722 MPASS(size + off <= crp->crp_buf.cb_vm_page_len +
723 crp->crp_buf.cb_vm_page_offset);
724 cvm_page_copydata(crp->crp_buf.cb_vm_page,
725 off + crp->crp_buf.cb_vm_page_offset, size, dst);
726 break;
727 #endif /* CRYPTO_MAY_HAVE_VMPAGE */
728 case CRYPTO_BUF_UIO:
729 cuio_copydata(crp->crp_buf.cb_uio, off, size, dst);
730 break;
731 case CRYPTO_BUF_CONTIG:
732 MPASS(off + size <= crp->crp_buf.cb_buf_len);
733 bcopy(crp->crp_buf.cb_buf + off, dst, size);
734 break;
735 default:
736 #ifdef INVARIANTS
737 panic("invalid crp buf type %d", crp->crp_buf.cb_type);
738 #endif
739 break;
740 }
741 }
742
743 int
crypto_apply_buf(struct crypto_buffer * cb,int off,int len,int (* f)(void *,const void *,u_int),void * arg)744 crypto_apply_buf(struct crypto_buffer *cb, int off, int len,
745 int (*f)(void *, const void *, u_int), void *arg)
746 {
747 int error;
748
749 switch (cb->cb_type) {
750 case CRYPTO_BUF_MBUF:
751 case CRYPTO_BUF_SINGLE_MBUF:
752 error = m_apply(cb->cb_mbuf, off, len,
753 (int (*)(void *, void *, u_int))f, arg);
754 break;
755 case CRYPTO_BUF_UIO:
756 error = cuio_apply(cb->cb_uio, off, len, f, arg);
757 break;
758 #if CRYPTO_MAY_HAVE_VMPAGE
759 case CRYPTO_BUF_VMPAGE:
760 error = cvm_page_apply(cb->cb_vm_page,
761 off + cb->cb_vm_page_offset, len, f, arg);
762 break;
763 #endif /* CRYPTO_MAY_HAVE_VMPAGE */
764 case CRYPTO_BUF_CONTIG:
765 MPASS(off + len <= cb->cb_buf_len);
766 error = (*f)(arg, cb->cb_buf + off, len);
767 break;
768 default:
769 #ifdef INVARIANTS
770 panic("invalid crypto buf type %d", cb->cb_type);
771 #endif
772 error = 0;
773 break;
774 }
775 return (error);
776 }
777
778 int
crypto_apply(struct cryptop * crp,int off,int len,int (* f)(void *,const void *,u_int),void * arg)779 crypto_apply(struct cryptop *crp, int off, int len,
780 int (*f)(void *, const void *, u_int), void *arg)
781 {
782 return (crypto_apply_buf(&crp->crp_buf, off, len, f, arg));
783 }
784
785 static inline void *
m_contiguous_subsegment(struct mbuf * m,size_t skip,size_t len)786 m_contiguous_subsegment(struct mbuf *m, size_t skip, size_t len)
787 {
788 int rel_off;
789
790 MPASS(skip <= INT_MAX);
791
792 m = m_getptr(m, (int)skip, &rel_off);
793 if (m == NULL)
794 return (NULL);
795
796 MPASS(rel_off >= 0);
797 skip = rel_off;
798 if (skip + len > m->m_len)
799 return (NULL);
800
801 if (m->m_flags & M_EXTPG)
802 return (m_epg_contiguous_subsegment(m, skip, len));
803 return (mtod(m, char*) + skip);
804 }
805
806 static inline void *
cuio_contiguous_segment(struct uio * uio,size_t skip,size_t len)807 cuio_contiguous_segment(struct uio *uio, size_t skip, size_t len)
808 {
809 int rel_off, idx;
810
811 MPASS(skip <= INT_MAX);
812 idx = cuio_getptr(uio, (int)skip, &rel_off);
813 if (idx < 0)
814 return (NULL);
815
816 MPASS(rel_off >= 0);
817 skip = rel_off;
818 if (skip + len > uio->uio_iov[idx].iov_len)
819 return (NULL);
820 return ((char *)uio->uio_iov[idx].iov_base + skip);
821 }
822
823 void *
crypto_buffer_contiguous_subsegment(struct crypto_buffer * cb,size_t skip,size_t len)824 crypto_buffer_contiguous_subsegment(struct crypto_buffer *cb, size_t skip,
825 size_t len)
826 {
827
828 switch (cb->cb_type) {
829 case CRYPTO_BUF_MBUF:
830 case CRYPTO_BUF_SINGLE_MBUF:
831 return (m_contiguous_subsegment(cb->cb_mbuf, skip, len));
832 case CRYPTO_BUF_UIO:
833 return (cuio_contiguous_segment(cb->cb_uio, skip, len));
834 #if CRYPTO_MAY_HAVE_VMPAGE
835 case CRYPTO_BUF_VMPAGE:
836 MPASS(skip + len <= cb->cb_vm_page_len);
837 return (cvm_page_contiguous_segment(cb->cb_vm_page,
838 skip + cb->cb_vm_page_offset, len));
839 #endif /* CRYPTO_MAY_HAVE_VMPAGE */
840 case CRYPTO_BUF_CONTIG:
841 MPASS(skip + len <= cb->cb_buf_len);
842 return (cb->cb_buf + skip);
843 default:
844 #ifdef INVARIANTS
845 panic("invalid crp buf type %d", cb->cb_type);
846 #endif
847 return (NULL);
848 }
849 }
850
851 void *
crypto_contiguous_subsegment(struct cryptop * crp,size_t skip,size_t len)852 crypto_contiguous_subsegment(struct cryptop *crp, size_t skip, size_t len)
853 {
854 return (crypto_buffer_contiguous_subsegment(&crp->crp_buf, skip, len));
855 }
856