Lines Matching +full:scatter +full:- +full:gather
1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
16 * 3. Neither the name of the author nor the names of any co-contributors
49 static MALLOC_DEFINE(M_SGLIST, "sglist", "scatter/gather lists");
64 (sgsave).sg_nseg = (sg)->sg_nseg; \
66 (sgsave).ss_len = (sg)->sg_segs[(sgsave).sg_nseg - 1].ss_len; \
72 (sg)->sg_nseg = (sgsave).sg_nseg; \
74 (sg)->sg_segs[(sgsave).sg_nseg - 1].ss_len = (sgsave).ss_len; \
89 if (ss->ss_paddr + ss->ss_len == paddr) in _sglist_append_range()
90 ss->ss_len += len; in _sglist_append_range()
92 if (sg->sg_nseg == sg->sg_maxseg) in _sglist_append_range()
95 ss->ss_paddr = paddr; in _sglist_append_range()
96 ss->ss_len = len; in _sglist_append_range()
97 sg->sg_nseg++; in _sglist_append_range()
105 * user) to a scatter/gather list.
129 seglen = MIN(len, PAGE_SIZE - offset); in _sglist_append_buf()
130 if (sg->sg_nseg == 0) { in _sglist_append_buf()
131 ss = sg->sg_segs; in _sglist_append_buf()
132 ss->ss_paddr = paddr; in _sglist_append_buf()
133 ss->ss_len = seglen; in _sglist_append_buf()
134 sg->sg_nseg = 1; in _sglist_append_buf()
136 ss = &sg->sg_segs[sg->sg_nseg - 1]; in _sglist_append_buf()
142 len -= seglen; in _sglist_append_buf()
156 len -= seglen; in _sglist_append_buf()
165 * Determine the number of scatter/gather list elements needed to
194 * Determine the number of scatter/gather list elements needed to
209 for (i = 1; len > PAGE_SIZE; len -= PAGE_SIZE, i++) { in sglist_count_vmpages()
219 * Determine the number of scatter/gather list elements needed to
233 if (m->m_epg_hdrlen != 0) { in sglist_count_mbuf_epg()
234 if (off >= m->m_epg_hdrlen) { in sglist_count_mbuf_epg()
235 off -= m->m_epg_hdrlen; in sglist_count_mbuf_epg()
237 seglen = m->m_epg_hdrlen - off; in sglist_count_mbuf_epg()
241 len -= seglen; in sglist_count_mbuf_epg()
242 nsegs += sglist_count(&m->m_epg_hdr[segoff], in sglist_count_mbuf_epg()
247 pgoff = m->m_epg_1st_off; in sglist_count_mbuf_epg()
248 for (i = 0; i < m->m_epg_npgs && len > 0; i++) { in sglist_count_mbuf_epg()
251 off -= pglen; in sglist_count_mbuf_epg()
255 seglen = pglen - off; in sglist_count_mbuf_epg()
259 len -= seglen; in sglist_count_mbuf_epg()
260 paddr = m->m_epg_pa[i] + segoff; in sglist_count_mbuf_epg()
267 seglen = MIN(len, m->m_epg_trllen - off); in sglist_count_mbuf_epg()
268 len -= seglen; in sglist_count_mbuf_epg()
269 nsegs += sglist_count(&m->m_epg_trail[off], seglen); in sglist_count_mbuf_epg()
276 * Allocate a scatter/gather list along with 'nsegs' segments. The
294 * Free a scatter/gather list allocated via sglist_allc().
303 if (refcount_release(&sg->sg_refs)) in sglist_free()
309 * range to a scatter/gather list. If there are insufficient
318 if (sg->sg_maxseg == 0) in sglist_append()
328 * Append the segments to describe a bio's data to a scatter/gather list.
338 if ((bp->bio_flags & BIO_UNMAPPED) == 0) in sglist_append_bio()
339 error = sglist_append(sg, bp->bio_data, bp->bio_bcount); in sglist_append_bio()
341 error = sglist_append_vmpages(sg, bp->bio_ma, in sglist_append_bio()
342 bp->bio_ma_offset, bp->bio_bcount); in sglist_append_bio()
347 * Append a single physical address range to a scatter/gather list.
357 if (sg->sg_maxseg == 0) in sglist_append_phys()
362 if (sg->sg_nseg == 0) { in sglist_append_phys()
363 sg->sg_segs[0].ss_paddr = paddr; in sglist_append_phys()
364 sg->sg_segs[0].ss_len = len; in sglist_append_phys()
365 sg->sg_nseg = 1; in sglist_append_phys()
368 ss = &sg->sg_segs[sg->sg_nseg - 1]; in sglist_append_phys()
377 * Append the segments of single multi-page mbuf.
391 if (m->m_epg_hdrlen != 0) { in sglist_append_mbuf_epg()
392 if (off >= m->m_epg_hdrlen) { in sglist_append_mbuf_epg()
393 off -= m->m_epg_hdrlen; in sglist_append_mbuf_epg()
395 seglen = m->m_epg_hdrlen - off; in sglist_append_mbuf_epg()
399 len -= seglen; in sglist_append_mbuf_epg()
401 &m->m_epg_hdr[segoff], seglen); in sglist_append_mbuf_epg()
404 pgoff = m->m_epg_1st_off; in sglist_append_mbuf_epg()
405 for (i = 0; i < m->m_epg_npgs && error == 0 && len > 0; i++) { in sglist_append_mbuf_epg()
408 off -= pglen; in sglist_append_mbuf_epg()
412 seglen = pglen - off; in sglist_append_mbuf_epg()
416 len -= seglen; in sglist_append_mbuf_epg()
417 paddr = m->m_epg_pa[i] + segoff; in sglist_append_mbuf_epg()
422 seglen = MIN(len, m->m_epg_trllen - off); in sglist_append_mbuf_epg()
423 len -= seglen; in sglist_append_mbuf_epg()
425 &m->m_epg_trail[off], seglen); in sglist_append_mbuf_epg()
434 * scatter/gather list. If there are insufficient segments, then this
444 if (sg->sg_maxseg == 0) in sglist_append_mbuf()
449 for (m = m0; m != NULL; m = m->m_next) { in sglist_append_mbuf()
450 if (m->m_len > 0) { in sglist_append_mbuf()
451 if ((m->m_flags & M_EXTPG) != 0) in sglist_append_mbuf()
453 mtod(m, vm_offset_t), m->m_len); in sglist_append_mbuf()
455 error = sglist_append(sg, m->m_data, in sglist_append_mbuf()
456 m->m_len); in sglist_append_mbuf()
467 * Append the segments that describe a single mbuf to a scatter/gather
474 if ((m->m_flags & M_EXTPG) != 0) in sglist_append_single_mbuf()
476 mtod(m, vm_offset_t), m->m_len)); in sglist_append_single_mbuf()
478 return (sglist_append(sg, m->m_data, m->m_len)); in sglist_append_single_mbuf()
496 if (sg->sg_maxseg == 0) in sglist_append_vmpages()
503 if (sg->sg_nseg == 0) { in sglist_append_vmpages()
504 seglen = min(PAGE_SIZE - pgoff, len); in sglist_append_vmpages()
505 sg->sg_segs[0].ss_paddr = VM_PAGE_TO_PHYS(m[0]) + pgoff; in sglist_append_vmpages()
506 sg->sg_segs[0].ss_len = seglen; in sglist_append_vmpages()
507 sg->sg_nseg = 1; in sglist_append_vmpages()
509 len -= seglen; in sglist_append_vmpages()
512 ss = &sg->sg_segs[sg->sg_nseg - 1]; in sglist_append_vmpages()
513 for (; len > 0; i++, len -= seglen) { in sglist_append_vmpages()
514 seglen = min(PAGE_SIZE - pgoff, len); in sglist_append_vmpages()
528 * scatter/gather list. If there are insufficient segments, then this
537 if (sg->sg_maxseg == 0) in sglist_append_user()
541 vmspace_pmap(td->td_proc->p_vmspace), NULL); in sglist_append_user()
548 * Append a subset of an existing scatter/gather list 'source' to a
549 * the scatter/gather list 'sg'. If there are insufficient segments,
561 if (sg->sg_maxseg == 0 || length == 0) in sglist_append_sglist()
565 ss = &sg->sg_segs[sg->sg_nseg - 1]; in sglist_append_sglist()
566 for (i = 0; i < source->sg_nseg; i++) { in sglist_append_sglist()
567 if (offset >= source->sg_segs[i].ss_len) { in sglist_append_sglist()
568 offset -= source->sg_segs[i].ss_len; in sglist_append_sglist()
571 seglen = source->sg_segs[i].ss_len - offset; in sglist_append_sglist()
575 source->sg_segs[i].ss_paddr + offset, seglen); in sglist_append_sglist()
579 length -= seglen; in sglist_append_sglist()
591 * Append the segments that describe a single uio to a scatter/gather
604 if (sg->sg_maxseg == 0) in sglist_append_uio()
607 resid = uio->uio_resid; in sglist_append_uio()
608 iov = uio->uio_iov; in sglist_append_uio()
610 if (uio->uio_segflg == UIO_USERSPACE) { in sglist_append_uio()
611 KASSERT(uio->uio_td != NULL, in sglist_append_uio()
613 pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace); in sglist_append_uio()
619 for (i = 0; i < uio->uio_iovcnt && resid != 0; i++) { in sglist_append_uio()
632 resid -= minlen; in sglist_append_uio()
640 * single uio to a scatter/gather list. If there are insufficient
651 if (sg->sg_maxseg == 0) in sglist_consume_uio()
654 if (uio->uio_segflg == UIO_USERSPACE) { in sglist_consume_uio()
655 KASSERT(uio->uio_td != NULL, in sglist_consume_uio()
657 pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace); in sglist_consume_uio()
662 while (resid > 0 && uio->uio_resid) { in sglist_consume_uio()
663 iov = uio->uio_iov; in sglist_consume_uio()
664 len = iov->iov_len; in sglist_consume_uio()
666 uio->uio_iov++; in sglist_consume_uio()
667 uio->uio_iovcnt--; in sglist_consume_uio()
677 error = _sglist_append_buf(sg, iov->iov_base, len, pmap, &done); in sglist_consume_uio()
678 iov->iov_base = (char *)iov->iov_base + done; in sglist_consume_uio()
679 iov->iov_len -= done; in sglist_consume_uio()
680 uio->uio_resid -= done; in sglist_consume_uio()
681 uio->uio_offset += done; in sglist_consume_uio()
682 resid -= done; in sglist_consume_uio()
690 * Allocate and populate a scatter/gather list to describe a single
714 * Clone a new copy of a scatter/gather list.
723 new = sglist_alloc(sg->sg_maxseg, mflags); in sglist_clone()
726 new->sg_nseg = sg->sg_nseg; in sglist_clone()
727 bcopy(sg->sg_segs, new->sg_segs, sizeof(struct sglist_seg) * in sglist_clone()
728 sg->sg_nseg); in sglist_clone()
734 * scatter/gather list.
743 for (i = 0; i < sg->sg_nseg; i++) in sglist_length()
744 space += sg->sg_segs[i].ss_len; in sglist_length()
749 * Split a scatter/gather list into two lists. The scatter/gather
773 if (original->sg_refs > 1) in sglist_split()
780 for (i = 0; i < original->sg_nseg; i++) { in sglist_split()
781 space += original->sg_segs[i].ss_len; in sglist_split()
786 * scatter/gather list entry, then 'split' in sglist_split()
790 split = space - length; in sglist_split()
806 if (sg->sg_maxseg < count) in sglist_split()
808 if (sg->sg_nseg != 0) in sglist_split()
813 bcopy(original->sg_segs, sg->sg_segs, count * in sglist_split()
815 sg->sg_nseg = count; in sglist_split()
821 * 'count - 1' segments from 'original' now. in sglist_split()
824 count--; in sglist_split()
825 sg->sg_segs[count].ss_len -= split; in sglist_split()
826 original->sg_segs[count].ss_paddr = in sglist_split()
827 sg->sg_segs[count].ss_paddr + split; in sglist_split()
828 original->sg_segs[count].ss_len = split; in sglist_split()
832 original->sg_nseg -= count; in sglist_split()
833 bcopy(original->sg_segs + count, original->sg_segs, count * in sglist_split()
839 * Append the scatter/gather list elements in 'second' to the
840 * scatter/gather list 'first'. If there is not enough space in
850 if (second->sg_nseg == 0) in sglist_join()
858 flast = &first->sg_segs[first->sg_nseg - 1]; in sglist_join()
859 sfirst = &second->sg_segs[0]; in sglist_join()
860 if (first->sg_nseg != 0 && in sglist_join()
861 flast->ss_paddr + flast->ss_len == sfirst->ss_paddr) in sglist_join()
865 if (first->sg_nseg + second->sg_nseg - append > first->sg_maxseg) in sglist_join()
870 flast->ss_len += sfirst->ss_len; in sglist_join()
873 bcopy(first->sg_segs + first->sg_nseg, second->sg_segs + append, in sglist_join()
874 (second->sg_nseg - append) * sizeof(struct sglist_seg)); in sglist_join()
875 first->sg_nseg += second->sg_nseg - append; in sglist_join()
881 * Generate a new scatter/gather list from a range of an existing
882 * scatter/gather list. The 'offset' and 'length' parameters specify
885 * returned. The new scatter/gather list is stored in '*slice'.
914 for (i = 0; i < original->sg_nseg; i++) { in sglist_slice()
915 space += original->sg_segs[i].ss_len; in sglist_slice()
924 foffs = offset - (space - in sglist_slice()
925 original->sg_segs[i].ss_len); in sglist_slice()
937 loffs = space - end; in sglist_slice()
956 if (sg->sg_maxseg < count) in sglist_slice()
958 if (sg->sg_nseg != 0) in sglist_slice()
966 bcopy(original->sg_segs + fseg, sg->sg_segs, in sglist_slice()
968 sg->sg_nseg = count; in sglist_slice()
972 sg->sg_segs[0].ss_paddr += foffs; in sglist_slice()
973 sg->sg_segs[0].ss_len -= foffs; in sglist_slice()
975 (long)sg->sg_segs[0].ss_paddr, sg->sg_segs[0].ss_len); in sglist_slice()
978 sg->sg_segs[count - 1].ss_len -= loffs; in sglist_slice()
979 CTR2(KTR_DEV, "sglist_slice seg[%d]: len %08x", count - 1, in sglist_slice()
980 sg->sg_segs[count - 1].ss_len); in sglist_slice()