1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2012 EMC Corp. 5 * All rights reserved. 6 * 7 * Copyright (c) 1997, 1998 Justin T. Gibbs. 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include "opt_bus.h" 36 #include "opt_iommu.h" 37 38 #include <sys/param.h> 39 #include <sys/conf.h> 40 #include <sys/systm.h> 41 #include <sys/bio.h> 42 #include <sys/bus.h> 43 #include <sys/callout.h> 44 #include <sys/ktr.h> 45 #include <sys/mbuf.h> 46 #include <sys/memdesc.h> 47 #include <sys/proc.h> 48 #include <sys/uio.h> 49 50 #include <vm/vm.h> 51 #include <vm/vm_page.h> 52 #include <vm/vm_map.h> 53 #include <vm/pmap.h> 54 55 #include <cam/cam.h> 56 #include <cam/cam_ccb.h> 57 58 #include <opencrypto/cryptodev.h> 59 60 #include <machine/bus.h> 61 62 /* 63 * Load up data starting at offset within a region specified by a 64 * list of virtual address ranges until either length or the region 65 * are exhausted. 66 */ 67 static int 68 _bus_dmamap_load_vlist(bus_dma_tag_t dmat, bus_dmamap_t map, 69 bus_dma_segment_t *list, int sglist_cnt, struct pmap *pmap, int *nsegs, 70 int flags, size_t offset, size_t length) 71 { 72 int error; 73 74 error = 0; 75 for (; sglist_cnt > 0 && length != 0; sglist_cnt--, list++) { 76 char *addr; 77 size_t ds_len; 78 79 KASSERT((offset < list->ds_len), 80 ("Invalid mid-segment offset")); 81 addr = (char *)(uintptr_t)list->ds_addr + offset; 82 ds_len = list->ds_len - offset; 83 offset = 0; 84 if (ds_len > length) 85 ds_len = length; 86 length -= ds_len; 87 KASSERT((ds_len != 0), ("Segment length is zero")); 88 error = _bus_dmamap_load_buffer(dmat, map, addr, ds_len, pmap, 89 flags, NULL, nsegs); 90 if (error) 91 break; 92 } 93 return (error); 94 } 95 96 /* 97 * Load a list of physical addresses. 98 */ 99 static int 100 _bus_dmamap_load_plist(bus_dma_tag_t dmat, bus_dmamap_t map, 101 bus_dma_segment_t *list, int sglist_cnt, int *nsegs, int flags) 102 { 103 int error; 104 105 error = 0; 106 for (; sglist_cnt > 0; sglist_cnt--, list++) { 107 error = _bus_dmamap_load_phys(dmat, map, 108 (vm_paddr_t)list->ds_addr, list->ds_len, flags, NULL, 109 nsegs); 110 if (error) 111 break; 112 } 113 return (error); 114 } 115 116 /* 117 * Load an unmapped mbuf 118 */ 119 static int 120 _bus_dmamap_load_mbuf_epg(bus_dma_tag_t dmat, bus_dmamap_t map, 121 struct mbuf *m, bus_dma_segment_t *segs, int *nsegs, int flags) 122 { 123 int error, i, off, len, pglen, pgoff, seglen, segoff; 124 125 M_ASSERTEXTPG(m); 126 127 len = m->m_len; 128 error = 0; 129 130 /* Skip over any data removed from the front. */ 131 off = mtod(m, vm_offset_t); 132 133 if (m->m_epg_hdrlen != 0) { 134 if (off >= m->m_epg_hdrlen) { 135 off -= m->m_epg_hdrlen; 136 } else { 137 seglen = m->m_epg_hdrlen - off; 138 segoff = off; 139 seglen = min(seglen, len); 140 off = 0; 141 len -= seglen; 142 error = _bus_dmamap_load_buffer(dmat, map, 143 &m->m_epg_hdr[segoff], seglen, kernel_pmap, 144 flags, segs, nsegs); 145 } 146 } 147 pgoff = m->m_epg_1st_off; 148 for (i = 0; i < m->m_epg_npgs && error == 0 && len > 0; i++) { 149 pglen = m_epg_pagelen(m, i, pgoff); 150 if (off >= pglen) { 151 off -= pglen; 152 pgoff = 0; 153 continue; 154 } 155 seglen = pglen - off; 156 segoff = pgoff + off; 157 off = 0; 158 seglen = min(seglen, len); 159 len -= seglen; 160 error = _bus_dmamap_load_phys(dmat, map, 161 m->m_epg_pa[i] + segoff, seglen, flags, segs, nsegs); 162 pgoff = 0; 163 }; 164 if (len != 0 && error == 0) { 165 KASSERT((off + len) <= m->m_epg_trllen, 166 ("off + len > trail (%d + %d > %d)", off, len, 167 m->m_epg_trllen)); 168 error = _bus_dmamap_load_buffer(dmat, map, 169 &m->m_epg_trail[off], len, kernel_pmap, flags, segs, 170 nsegs); 171 } 172 return (error); 173 } 174 175 /* 176 * Load a single mbuf. 177 */ 178 static int 179 _bus_dmamap_load_single_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, 180 struct mbuf *m, bus_dma_segment_t *segs, int *nsegs, int flags) 181 { 182 int error; 183 184 error = 0; 185 if ((m->m_flags & M_EXTPG) != 0) 186 error = _bus_dmamap_load_mbuf_epg(dmat, map, m, segs, nsegs, 187 flags); 188 else 189 error = _bus_dmamap_load_buffer(dmat, map, m->m_data, m->m_len, 190 kernel_pmap, flags | BUS_DMA_LOAD_MBUF, segs, nsegs); 191 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 192 __func__, dmat, flags, error, *nsegs); 193 return (error); 194 } 195 196 /* 197 * Load an mbuf chain. 198 */ 199 static int 200 _bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, 201 struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, int flags) 202 { 203 struct mbuf *m; 204 int error; 205 206 error = 0; 207 for (m = m0; m != NULL && error == 0; m = m->m_next) { 208 if (m->m_len > 0) { 209 if ((m->m_flags & M_EXTPG) != 0) 210 error = _bus_dmamap_load_mbuf_epg(dmat, 211 map, m, segs, nsegs, flags); 212 else 213 error = _bus_dmamap_load_buffer(dmat, map, 214 m->m_data, m->m_len, kernel_pmap, 215 flags | BUS_DMA_LOAD_MBUF, segs, nsegs); 216 } 217 } 218 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 219 __func__, dmat, flags, error, *nsegs); 220 return (error); 221 } 222 223 /* 224 * Load from block io. 225 */ 226 static int 227 _bus_dmamap_load_bio(bus_dma_tag_t dmat, bus_dmamap_t map, struct bio *bio, 228 int *nsegs, int flags) 229 { 230 231 if ((bio->bio_flags & BIO_VLIST) != 0) { 232 bus_dma_segment_t *segs = (bus_dma_segment_t *)bio->bio_data; 233 return (_bus_dmamap_load_vlist(dmat, map, segs, bio->bio_ma_n, 234 kernel_pmap, nsegs, flags, bio->bio_ma_offset, 235 bio->bio_bcount)); 236 } 237 238 if ((bio->bio_flags & BIO_UNMAPPED) != 0) 239 return (_bus_dmamap_load_ma(dmat, map, bio->bio_ma, 240 bio->bio_bcount, bio->bio_ma_offset, flags, NULL, nsegs)); 241 242 return (_bus_dmamap_load_buffer(dmat, map, bio->bio_data, 243 bio->bio_bcount, kernel_pmap, flags, NULL, nsegs)); 244 } 245 246 int 247 bus_dmamap_load_ma_triv(bus_dma_tag_t dmat, bus_dmamap_t map, 248 struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags, 249 bus_dma_segment_t *segs, int *segp) 250 { 251 vm_paddr_t paddr; 252 bus_size_t len; 253 int error, i; 254 255 error = 0; 256 for (i = 0; tlen > 0; i++, tlen -= len) { 257 len = min(PAGE_SIZE - ma_offs, tlen); 258 paddr = VM_PAGE_TO_PHYS(ma[i]) + ma_offs; 259 error = _bus_dmamap_load_phys(dmat, map, paddr, len, 260 flags, segs, segp); 261 if (error != 0) 262 break; 263 ma_offs = 0; 264 } 265 return (error); 266 } 267 268 /* 269 * Load a cam control block. 270 */ 271 static int 272 _bus_dmamap_load_ccb(bus_dma_tag_t dmat, bus_dmamap_t map, union ccb *ccb, 273 int *nsegs, int flags) 274 { 275 struct ccb_hdr *ccb_h; 276 void *data_ptr; 277 int error; 278 uint32_t dxfer_len; 279 uint16_t sglist_cnt; 280 281 error = 0; 282 ccb_h = &ccb->ccb_h; 283 switch (ccb_h->func_code) { 284 case XPT_SCSI_IO: { 285 struct ccb_scsiio *csio; 286 287 csio = &ccb->csio; 288 data_ptr = csio->data_ptr; 289 dxfer_len = csio->dxfer_len; 290 sglist_cnt = csio->sglist_cnt; 291 break; 292 } 293 case XPT_CONT_TARGET_IO: { 294 struct ccb_scsiio *ctio; 295 296 ctio = &ccb->ctio; 297 data_ptr = ctio->data_ptr; 298 dxfer_len = ctio->dxfer_len; 299 sglist_cnt = ctio->sglist_cnt; 300 break; 301 } 302 case XPT_ATA_IO: { 303 struct ccb_ataio *ataio; 304 305 ataio = &ccb->ataio; 306 data_ptr = ataio->data_ptr; 307 dxfer_len = ataio->dxfer_len; 308 sglist_cnt = 0; 309 break; 310 } 311 case XPT_NVME_IO: 312 case XPT_NVME_ADMIN: { 313 struct ccb_nvmeio *nvmeio; 314 315 nvmeio = &ccb->nvmeio; 316 data_ptr = nvmeio->data_ptr; 317 dxfer_len = nvmeio->dxfer_len; 318 sglist_cnt = nvmeio->sglist_cnt; 319 break; 320 } 321 default: 322 panic("_bus_dmamap_load_ccb: Unsupported func code %d", 323 ccb_h->func_code); 324 } 325 326 switch ((ccb_h->flags & CAM_DATA_MASK)) { 327 case CAM_DATA_VADDR: 328 error = _bus_dmamap_load_buffer(dmat, map, data_ptr, dxfer_len, 329 kernel_pmap, flags, NULL, nsegs); 330 break; 331 case CAM_DATA_PADDR: 332 error = _bus_dmamap_load_phys(dmat, map, 333 (vm_paddr_t)(uintptr_t)data_ptr, dxfer_len, flags, NULL, 334 nsegs); 335 break; 336 case CAM_DATA_SG: 337 error = _bus_dmamap_load_vlist(dmat, map, 338 (bus_dma_segment_t *)data_ptr, sglist_cnt, kernel_pmap, 339 nsegs, flags, 0, dxfer_len); 340 break; 341 case CAM_DATA_SG_PADDR: 342 error = _bus_dmamap_load_plist(dmat, map, 343 (bus_dma_segment_t *)data_ptr, sglist_cnt, nsegs, flags); 344 break; 345 case CAM_DATA_BIO: 346 error = _bus_dmamap_load_bio(dmat, map, (struct bio *)data_ptr, 347 nsegs, flags); 348 break; 349 default: 350 panic("_bus_dmamap_load_ccb: flags 0x%X unimplemented", 351 ccb_h->flags); 352 } 353 return (error); 354 } 355 356 /* 357 * Load a uio. 358 */ 359 static int 360 _bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, struct uio *uio, 361 int *nsegs, int flags) 362 { 363 bus_size_t resid; 364 bus_size_t minlen; 365 struct iovec *iov; 366 pmap_t pmap; 367 caddr_t addr; 368 int error, i; 369 370 if (uio->uio_segflg == UIO_USERSPACE) { 371 KASSERT(uio->uio_td != NULL, 372 ("bus_dmamap_load_uio: USERSPACE but no proc")); 373 pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace); 374 } else 375 pmap = kernel_pmap; 376 resid = uio->uio_resid; 377 iov = uio->uio_iov; 378 error = 0; 379 380 for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) { 381 /* 382 * Now at the first iovec to load. Load each iovec 383 * until we have exhausted the residual count. 384 */ 385 386 addr = (caddr_t) iov[i].iov_base; 387 minlen = resid < iov[i].iov_len ? resid : iov[i].iov_len; 388 if (minlen > 0) { 389 error = _bus_dmamap_load_buffer(dmat, map, addr, 390 minlen, pmap, flags, NULL, nsegs); 391 resid -= minlen; 392 } 393 } 394 395 return (error); 396 } 397 398 /* 399 * Map the buffer buf into bus space using the dmamap map. 400 */ 401 int 402 bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 403 bus_size_t buflen, bus_dmamap_callback_t *callback, 404 void *callback_arg, int flags) 405 { 406 bus_dma_segment_t *segs; 407 struct memdesc mem; 408 int error; 409 int nsegs; 410 411 if ((flags & BUS_DMA_NOWAIT) == 0) { 412 mem = memdesc_vaddr(buf, buflen); 413 _bus_dmamap_waitok(dmat, map, &mem, callback, callback_arg); 414 } 415 416 nsegs = -1; 417 error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, kernel_pmap, 418 flags, NULL, &nsegs); 419 nsegs++; 420 421 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 422 __func__, dmat, flags, error, nsegs); 423 424 if (error == EINPROGRESS) 425 return (error); 426 427 segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error); 428 if (error) 429 (*callback)(callback_arg, segs, 0, error); 430 else 431 (*callback)(callback_arg, segs, nsegs, 0); 432 433 /* 434 * Return ENOMEM to the caller so that it can pass it up the stack. 435 * This error only happens when NOWAIT is set, so deferral is disabled. 436 */ 437 if (error == ENOMEM) 438 return (error); 439 440 return (0); 441 } 442 443 int 444 bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0, 445 bus_dmamap_callback2_t *callback, void *callback_arg, int flags) 446 { 447 bus_dma_segment_t *segs; 448 int nsegs, error; 449 450 M_ASSERTPKTHDR(m0); 451 452 flags |= BUS_DMA_NOWAIT; 453 nsegs = -1; 454 error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, NULL, &nsegs, flags); 455 ++nsegs; 456 457 segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error); 458 if (error) 459 (*callback)(callback_arg, segs, 0, 0, error); 460 else 461 (*callback)(callback_arg, segs, nsegs, m0->m_pkthdr.len, error); 462 463 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 464 __func__, dmat, flags, error, nsegs); 465 return (error); 466 } 467 468 int 469 bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0, 470 bus_dma_segment_t *segs, int *nsegs, int flags) 471 { 472 int error; 473 474 flags |= BUS_DMA_NOWAIT; 475 *nsegs = -1; 476 error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, segs, nsegs, flags); 477 ++*nsegs; 478 _bus_dmamap_complete(dmat, map, segs, *nsegs, error); 479 return (error); 480 } 481 482 int 483 bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, struct uio *uio, 484 bus_dmamap_callback2_t *callback, void *callback_arg, int flags) 485 { 486 bus_dma_segment_t *segs; 487 int nsegs, error; 488 489 flags |= BUS_DMA_NOWAIT; 490 nsegs = -1; 491 error = _bus_dmamap_load_uio(dmat, map, uio, &nsegs, flags); 492 nsegs++; 493 494 segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error); 495 if (error) 496 (*callback)(callback_arg, segs, 0, 0, error); 497 else 498 (*callback)(callback_arg, segs, nsegs, uio->uio_resid, error); 499 500 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 501 __func__, dmat, flags, error, nsegs); 502 return (error); 503 } 504 505 int 506 bus_dmamap_load_ccb(bus_dma_tag_t dmat, bus_dmamap_t map, union ccb *ccb, 507 bus_dmamap_callback_t *callback, void *callback_arg, 508 int flags) 509 { 510 bus_dma_segment_t *segs; 511 struct ccb_hdr *ccb_h; 512 struct memdesc mem; 513 int error; 514 int nsegs; 515 516 ccb_h = &ccb->ccb_h; 517 if ((ccb_h->flags & CAM_DIR_MASK) == CAM_DIR_NONE) { 518 callback(callback_arg, NULL, 0, 0); 519 return (0); 520 } 521 if ((flags & BUS_DMA_NOWAIT) == 0) { 522 mem = memdesc_ccb(ccb); 523 _bus_dmamap_waitok(dmat, map, &mem, callback, callback_arg); 524 } 525 nsegs = -1; 526 error = _bus_dmamap_load_ccb(dmat, map, ccb, &nsegs, flags); 527 nsegs++; 528 529 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 530 __func__, dmat, flags, error, nsegs); 531 532 if (error == EINPROGRESS) 533 return (error); 534 535 segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error); 536 if (error) 537 (*callback)(callback_arg, segs, 0, error); 538 else 539 (*callback)(callback_arg, segs, nsegs, error); 540 /* 541 * Return ENOMEM to the caller so that it can pass it up the stack. 542 * This error only happens when NOWAIT is set, so deferral is disabled. 543 */ 544 if (error == ENOMEM) 545 return (error); 546 547 return (0); 548 } 549 550 int 551 bus_dmamap_load_bio(bus_dma_tag_t dmat, bus_dmamap_t map, struct bio *bio, 552 bus_dmamap_callback_t *callback, void *callback_arg, 553 int flags) 554 { 555 bus_dma_segment_t *segs; 556 struct memdesc mem; 557 int error; 558 int nsegs; 559 560 if ((flags & BUS_DMA_NOWAIT) == 0) { 561 mem = memdesc_bio(bio); 562 _bus_dmamap_waitok(dmat, map, &mem, callback, callback_arg); 563 } 564 nsegs = -1; 565 error = _bus_dmamap_load_bio(dmat, map, bio, &nsegs, flags); 566 nsegs++; 567 568 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 569 __func__, dmat, flags, error, nsegs); 570 571 if (error == EINPROGRESS) 572 return (error); 573 574 segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error); 575 if (error) 576 (*callback)(callback_arg, segs, 0, error); 577 else 578 (*callback)(callback_arg, segs, nsegs, error); 579 /* 580 * Return ENOMEM to the caller so that it can pass it up the stack. 581 * This error only happens when NOWAIT is set, so deferral is disabled. 582 */ 583 if (error == ENOMEM) 584 return (error); 585 586 return (0); 587 } 588 589 int 590 bus_dmamap_load_mem(bus_dma_tag_t dmat, bus_dmamap_t map, 591 struct memdesc *mem, bus_dmamap_callback_t *callback, 592 void *callback_arg, int flags) 593 { 594 bus_dma_segment_t *segs; 595 int error; 596 int nsegs; 597 598 if ((flags & BUS_DMA_NOWAIT) == 0) 599 _bus_dmamap_waitok(dmat, map, mem, callback, callback_arg); 600 601 nsegs = -1; 602 error = 0; 603 switch (mem->md_type) { 604 case MEMDESC_VADDR: 605 error = _bus_dmamap_load_buffer(dmat, map, mem->u.md_vaddr, 606 mem->md_opaque, kernel_pmap, flags, NULL, &nsegs); 607 break; 608 case MEMDESC_PADDR: 609 error = _bus_dmamap_load_phys(dmat, map, mem->u.md_paddr, 610 mem->md_opaque, flags, NULL, &nsegs); 611 break; 612 case MEMDESC_VLIST: 613 error = _bus_dmamap_load_vlist(dmat, map, mem->u.md_list, 614 mem->md_opaque, kernel_pmap, &nsegs, flags, 0, SIZE_T_MAX); 615 break; 616 case MEMDESC_PLIST: 617 error = _bus_dmamap_load_plist(dmat, map, mem->u.md_list, 618 mem->md_opaque, &nsegs, flags); 619 break; 620 case MEMDESC_BIO: 621 error = _bus_dmamap_load_bio(dmat, map, mem->u.md_bio, 622 &nsegs, flags); 623 break; 624 case MEMDESC_UIO: 625 error = _bus_dmamap_load_uio(dmat, map, mem->u.md_uio, 626 &nsegs, flags); 627 break; 628 case MEMDESC_MBUF: 629 error = _bus_dmamap_load_mbuf_sg(dmat, map, mem->u.md_mbuf, 630 NULL, &nsegs, flags); 631 break; 632 case MEMDESC_CCB: 633 error = _bus_dmamap_load_ccb(dmat, map, mem->u.md_ccb, &nsegs, 634 flags); 635 break; 636 } 637 nsegs++; 638 639 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 640 __func__, dmat, flags, error, nsegs); 641 642 if (error == EINPROGRESS) 643 return (error); 644 645 segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error); 646 if (error) 647 (*callback)(callback_arg, segs, 0, error); 648 else 649 (*callback)(callback_arg, segs, nsegs, 0); 650 651 /* 652 * Return ENOMEM to the caller so that it can pass it up the stack. 653 * This error only happens when NOWAIT is set, so deferral is disabled. 654 */ 655 if (error == ENOMEM) 656 return (error); 657 658 return (0); 659 } 660 661 int 662 bus_dmamap_load_crp_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, 663 struct crypto_buffer *cb, bus_dmamap_callback_t *callback, 664 void *callback_arg, int flags) 665 { 666 bus_dma_segment_t *segs; 667 int error; 668 int nsegs; 669 670 flags |= BUS_DMA_NOWAIT; 671 nsegs = -1; 672 error = 0; 673 switch (cb->cb_type) { 674 case CRYPTO_BUF_CONTIG: 675 error = _bus_dmamap_load_buffer(dmat, map, cb->cb_buf, 676 cb->cb_buf_len, kernel_pmap, flags, NULL, &nsegs); 677 break; 678 case CRYPTO_BUF_MBUF: 679 error = _bus_dmamap_load_mbuf_sg(dmat, map, cb->cb_mbuf, 680 NULL, &nsegs, flags); 681 break; 682 case CRYPTO_BUF_SINGLE_MBUF: 683 error = _bus_dmamap_load_single_mbuf(dmat, map, cb->cb_mbuf, 684 NULL, &nsegs, flags); 685 break; 686 case CRYPTO_BUF_UIO: 687 error = _bus_dmamap_load_uio(dmat, map, cb->cb_uio, &nsegs, 688 flags); 689 break; 690 case CRYPTO_BUF_VMPAGE: 691 error = _bus_dmamap_load_ma(dmat, map, cb->cb_vm_page, 692 cb->cb_vm_page_len, cb->cb_vm_page_offset, flags, NULL, 693 &nsegs); 694 break; 695 default: 696 error = EINVAL; 697 } 698 nsegs++; 699 700 CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", 701 __func__, dmat, flags, error, nsegs); 702 703 if (error == EINPROGRESS) 704 return (error); 705 706 segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error); 707 if (error) 708 (*callback)(callback_arg, segs, 0, error); 709 else 710 (*callback)(callback_arg, segs, nsegs, 0); 711 712 /* 713 * Return ENOMEM to the caller so that it can pass it up the stack. 714 * This error only happens when NOWAIT is set, so deferral is disabled. 715 */ 716 if (error == ENOMEM) 717 return (error); 718 719 return (0); 720 } 721 722 int 723 bus_dmamap_load_crp(bus_dma_tag_t dmat, bus_dmamap_t map, struct cryptop *crp, 724 bus_dmamap_callback_t *callback, void *callback_arg, int flags) 725 { 726 return (bus_dmamap_load_crp_buffer(dmat, map, &crp->crp_buf, callback, 727 callback_arg, flags)); 728 } 729 730 void 731 bus_dma_template_init(bus_dma_template_t *t, bus_dma_tag_t parent) 732 { 733 734 if (t == NULL) 735 return; 736 737 t->parent = parent; 738 t->alignment = 1; 739 t->boundary = 0; 740 t->lowaddr = t->highaddr = BUS_SPACE_MAXADDR; 741 t->maxsize = t->maxsegsize = BUS_SPACE_MAXSIZE; 742 t->nsegments = BUS_SPACE_UNRESTRICTED; 743 t->lockfunc = NULL; 744 t->lockfuncarg = NULL; 745 t->flags = 0; 746 } 747 748 int 749 bus_dma_template_tag(bus_dma_template_t *t, bus_dma_tag_t *dmat) 750 { 751 752 if (t == NULL || dmat == NULL) 753 return (EINVAL); 754 755 return (bus_dma_tag_create(t->parent, t->alignment, t->boundary, 756 t->lowaddr, t->highaddr, NULL, NULL, t->maxsize, 757 t->nsegments, t->maxsegsize, t->flags, t->lockfunc, t->lockfuncarg, 758 dmat)); 759 } 760 761 void 762 bus_dma_template_fill(bus_dma_template_t *t, bus_dma_param_t *kv, u_int count) 763 { 764 bus_dma_param_t *pkv; 765 766 while (count) { 767 pkv = &kv[--count]; 768 switch (pkv->key) { 769 case BD_PARAM_PARENT: 770 t->parent = pkv->ptr; 771 break; 772 case BD_PARAM_ALIGNMENT: 773 t->alignment = pkv->num; 774 break; 775 case BD_PARAM_BOUNDARY: 776 t->boundary = pkv->num; 777 break; 778 case BD_PARAM_LOWADDR: 779 t->lowaddr = pkv->pa; 780 break; 781 case BD_PARAM_HIGHADDR: 782 t->highaddr = pkv->pa; 783 break; 784 case BD_PARAM_MAXSIZE: 785 t->maxsize = pkv->num; 786 break; 787 case BD_PARAM_NSEGMENTS: 788 t->nsegments = pkv->num; 789 break; 790 case BD_PARAM_MAXSEGSIZE: 791 t->maxsegsize = pkv->num; 792 break; 793 case BD_PARAM_FLAGS: 794 t->flags = pkv->num; 795 break; 796 case BD_PARAM_LOCKFUNC: 797 t->lockfunc = pkv->ptr; 798 break; 799 case BD_PARAM_LOCKFUNCARG: 800 t->lockfuncarg = pkv->ptr; 801 break; 802 case BD_PARAM_NAME: 803 t->name = pkv->ptr; 804 break; 805 case BD_PARAM_INVALID: 806 default: 807 KASSERT(0, ("Invalid key %d\n", pkv->key)); 808 break; 809 } 810 } 811 return; 812 } 813 814 #ifndef IOMMU 815 bool bus_dma_iommu_set_buswide(device_t dev); 816 int bus_dma_iommu_load_ident(bus_dma_tag_t dmat, bus_dmamap_t map, 817 vm_paddr_t start, vm_size_t length, int flags); 818 819 bool 820 bus_dma_iommu_set_buswide(device_t dev) 821 { 822 return (false); 823 } 824 825 int 826 bus_dma_iommu_load_ident(bus_dma_tag_t dmat, bus_dmamap_t map, 827 vm_paddr_t start, vm_size_t length, int flags) 828 { 829 return (0); 830 } 831 #endif 832