1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
16  * 3. Neither the name of the author nor the names of any co-contributors
64 	(sgsave).sg_nseg = (sg)->sg_nseg;				\
66 		(sgsave).ss_len = (sg)->sg_segs[(sgsave).sg_nseg - 1].ss_len; \
72 	(sg)->sg_nseg = (sgsave).sg_nseg;				\
74 		(sg)->sg_segs[(sgsave).sg_nseg - 1].ss_len = (sgsave).ss_len; \
78  * Append a single (paddr, len) to a sglist.  sg is the list and ss is
84     vm_paddr_t paddr, size_t len)  in _sglist_append_range()  argument
89 	if (ss->ss_paddr + ss->ss_len == paddr)  in _sglist_append_range()
90 		ss->ss_len += len;  in _sglist_append_range()
92 		if (sg->sg_nseg == sg->sg_maxseg)  in _sglist_append_range()
95 		ss->ss_paddr = paddr;  in _sglist_append_range()
96 		ss->ss_len = len;  in _sglist_append_range()
97 		sg->sg_nseg++;  in _sglist_append_range()
108 _sglist_append_buf(struct sglist *sg, void *buf, size_t len, pmap_t pmap,  in _sglist_append_buf()  argument
119 	if (len == 0)  in _sglist_append_buf()
129 	seglen = MIN(len, PAGE_SIZE - offset);  in _sglist_append_buf()
130 	if (sg->sg_nseg == 0) {  in _sglist_append_buf()
131 		ss = sg->sg_segs;  in _sglist_append_buf()
132 		ss->ss_paddr = paddr;  in _sglist_append_buf()
133 		ss->ss_len = seglen;  in _sglist_append_buf()
134 		sg->sg_nseg = 1;  in _sglist_append_buf()
136 		ss = &sg->sg_segs[sg->sg_nseg - 1];  in _sglist_append_buf()
142 	len -= seglen;  in _sglist_append_buf()
146 	while (len > 0) {  in _sglist_append_buf()
147 		seglen = MIN(len, PAGE_SIZE);  in _sglist_append_buf()
156 		len -= seglen;  in _sglist_append_buf()
169 sglist_count(void *buf, size_t len)  in sglist_count()  argument
175 	if (len == 0)  in sglist_count()
179 	vendaddr = (vm_offset_t)buf + len;  in sglist_count()
198 sglist_count_vmpages(vm_page_t *m, size_t pgoff, size_t len)  in sglist_count_vmpages()  argument
203 	if (len == 0)  in sglist_count_vmpages()
206 	len += pgoff;  in sglist_count_vmpages()
209 	for (i = 1; len > PAGE_SIZE; len -= PAGE_SIZE, i++) {  in sglist_count_vmpages()
223 sglist_count_mbuf_epg(struct mbuf *m, size_t off, size_t len)  in sglist_count_mbuf_epg()  argument
229 	if (len == 0)  in sglist_count_mbuf_epg()
233 	if (m->m_epg_hdrlen != 0) {  in sglist_count_mbuf_epg()
234 		if (off >= m->m_epg_hdrlen) {  in sglist_count_mbuf_epg()
235 			off -= m->m_epg_hdrlen;  in sglist_count_mbuf_epg()
237 			seglen = m->m_epg_hdrlen - off;  in sglist_count_mbuf_epg()
239 			seglen = MIN(seglen, len);  in sglist_count_mbuf_epg()
241 			len -= seglen;  in sglist_count_mbuf_epg()
242 			nsegs += sglist_count(&m->m_epg_hdr[segoff],  in sglist_count_mbuf_epg()
247 	pgoff = m->m_epg_1st_off;  in sglist_count_mbuf_epg()
248 	for (i = 0; i < m->m_epg_npgs && len > 0; i++) {  in sglist_count_mbuf_epg()
251 			off -= pglen;  in sglist_count_mbuf_epg()
255 		seglen = pglen - off;  in sglist_count_mbuf_epg()
258 		seglen = MIN(seglen, len);  in sglist_count_mbuf_epg()
259 		len -= seglen;  in sglist_count_mbuf_epg()
260 		paddr = m->m_epg_pa[i] + segoff;  in sglist_count_mbuf_epg()
266 	if (len != 0) {  in sglist_count_mbuf_epg()
267 		seglen = MIN(len, m->m_epg_trllen - off);  in sglist_count_mbuf_epg()
268 		len -= seglen;  in sglist_count_mbuf_epg()
269 		nsegs += sglist_count(&m->m_epg_trail[off], seglen);  in sglist_count_mbuf_epg()
271 	KASSERT(len == 0, ("len != 0"));  in sglist_count_mbuf_epg()
303 	if (refcount_release(&sg->sg_refs))  in sglist_free()
313 sglist_append(struct sglist *sg, void *buf, size_t len)  in sglist_append()  argument
318 	if (sg->sg_maxseg == 0)  in sglist_append()
321 	error = _sglist_append_buf(sg, buf, len, NULL, NULL);  in sglist_append()
338 	if ((bp->bio_flags & BIO_UNMAPPED) == 0)  in sglist_append_bio()
339 		error = sglist_append(sg, bp->bio_data, bp->bio_bcount);  in sglist_append_bio()
341 		error = sglist_append_vmpages(sg, bp->bio_ma,  in sglist_append_bio()
342 		    bp->bio_ma_offset, bp->bio_bcount);  in sglist_append_bio()
351 sglist_append_phys(struct sglist *sg, vm_paddr_t paddr, size_t len)  in sglist_append_phys()  argument
357 	if (sg->sg_maxseg == 0)  in sglist_append_phys()
359 	if (len == 0)  in sglist_append_phys()
362 	if (sg->sg_nseg == 0) {  in sglist_append_phys()
363 		sg->sg_segs[0].ss_paddr = paddr;  in sglist_append_phys()
364 		sg->sg_segs[0].ss_len = len;  in sglist_append_phys()
365 		sg->sg_nseg = 1;  in sglist_append_phys()
368 	ss = &sg->sg_segs[sg->sg_nseg - 1];  in sglist_append_phys()
370 	error = _sglist_append_range(sg, &ss, paddr, len);  in sglist_append_phys()
377  * Append the segments of single multi-page mbuf.
382     size_t len)  in sglist_append_mbuf_epg()  argument
391 	if (m->m_epg_hdrlen != 0) {  in sglist_append_mbuf_epg()
392 		if (off >= m->m_epg_hdrlen) {  in sglist_append_mbuf_epg()
393 			off -= m->m_epg_hdrlen;  in sglist_append_mbuf_epg()
395 			seglen = m->m_epg_hdrlen - off;  in sglist_append_mbuf_epg()
397 			seglen = MIN(seglen, len);  in sglist_append_mbuf_epg()
399 			len -= seglen;  in sglist_append_mbuf_epg()
401 			    &m->m_epg_hdr[segoff], seglen);  in sglist_append_mbuf_epg()
404 	pgoff = m->m_epg_1st_off;  in sglist_append_mbuf_epg()
405 	for (i = 0; i < m->m_epg_npgs && error == 0 && len > 0; i++) {  in sglist_append_mbuf_epg()
408 			off -= pglen;  in sglist_append_mbuf_epg()
412 		seglen = pglen - off;  in sglist_append_mbuf_epg()
415 		seglen = MIN(seglen, len);  in sglist_append_mbuf_epg()
416 		len -= seglen;  in sglist_append_mbuf_epg()
417 		paddr = m->m_epg_pa[i] + segoff;  in sglist_append_mbuf_epg()
421 	if (error == 0 && len > 0) {  in sglist_append_mbuf_epg()
422 		seglen = MIN(len, m->m_epg_trllen - off);  in sglist_append_mbuf_epg()
423 		len -= seglen;  in sglist_append_mbuf_epg()
425 		    &m->m_epg_trail[off], seglen);  in sglist_append_mbuf_epg()
428 		KASSERT(len == 0, ("len != 0"));  in sglist_append_mbuf_epg()
444 	if (sg->sg_maxseg == 0)  in sglist_append_mbuf()
449 	for (m = m0; m != NULL; m = m->m_next) {  in sglist_append_mbuf()
450 		if (m->m_len > 0) {  in sglist_append_mbuf()
451 			if ((m->m_flags & M_EXTPG) != 0)  in sglist_append_mbuf()
453 				    mtod(m, vm_offset_t), m->m_len);  in sglist_append_mbuf()
455 				error = sglist_append(sg, m->m_data,  in sglist_append_mbuf()
456 				    m->m_len);  in sglist_append_mbuf()
474 	if ((m->m_flags & M_EXTPG) != 0)  in sglist_append_single_mbuf()
476 		    mtod(m, vm_offset_t), m->m_len));  in sglist_append_single_mbuf()
478 		return (sglist_append(sg, m->m_data, m->m_len));  in sglist_append_single_mbuf()
488     size_t len)  in sglist_append_vmpages()  argument
496 	if (sg->sg_maxseg == 0)  in sglist_append_vmpages()
498 	if (len == 0)  in sglist_append_vmpages()
503 	if (sg->sg_nseg == 0) {  in sglist_append_vmpages()
504 		seglen = min(PAGE_SIZE - pgoff, len);  in sglist_append_vmpages()
505 		sg->sg_segs[0].ss_paddr = VM_PAGE_TO_PHYS(m[0]) + pgoff;  in sglist_append_vmpages()
506 		sg->sg_segs[0].ss_len = seglen;  in sglist_append_vmpages()
507 		sg->sg_nseg = 1;  in sglist_append_vmpages()
509 		len -= seglen;  in sglist_append_vmpages()
512 	ss = &sg->sg_segs[sg->sg_nseg - 1];  in sglist_append_vmpages()
513 	for (; len > 0; i++, len -= seglen) {  in sglist_append_vmpages()
514 		seglen = min(PAGE_SIZE - pgoff, len);  in sglist_append_vmpages()
532 sglist_append_user(struct sglist *sg, void *buf, size_t len, struct thread *td)  in sglist_append_user()  argument
537 	if (sg->sg_maxseg == 0)  in sglist_append_user()
540 	error = _sglist_append_buf(sg, buf, len,  in sglist_append_user()
541 	    vmspace_pmap(td->td_proc->p_vmspace), NULL);  in sglist_append_user()
561 	if (sg->sg_maxseg == 0 || length == 0)  in sglist_append_sglist()
565 	ss = &sg->sg_segs[sg->sg_nseg - 1];  in sglist_append_sglist()
566 	for (i = 0; i < source->sg_nseg; i++) {  in sglist_append_sglist()
567 		if (offset >= source->sg_segs[i].ss_len) {  in sglist_append_sglist()
568 			offset -= source->sg_segs[i].ss_len;  in sglist_append_sglist()
571 		seglen = source->sg_segs[i].ss_len - offset;  in sglist_append_sglist()
575 		    source->sg_segs[i].ss_paddr + offset, seglen);  in sglist_append_sglist()
579 		length -= seglen;  in sglist_append_sglist()
604 	if (sg->sg_maxseg == 0)  in sglist_append_uio()
607 	resid = uio->uio_resid;  in sglist_append_uio()
608 	iov = uio->uio_iov;  in sglist_append_uio()
610 	if (uio->uio_segflg == UIO_USERSPACE) {  in sglist_append_uio()
611 		KASSERT(uio->uio_td != NULL,  in sglist_append_uio()
613 		pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace);  in sglist_append_uio()
619 	for (i = 0; i < uio->uio_iovcnt && resid != 0; i++) {  in sglist_append_uio()
624 		minlen = MIN(resid, iov[i].iov_len);  in sglist_append_uio()
632 			resid -= minlen;  in sglist_append_uio()
649 	int error, len;  in sglist_consume_uio()  local
651 	if (sg->sg_maxseg == 0)  in sglist_consume_uio()
654 	if (uio->uio_segflg == UIO_USERSPACE) {  in sglist_consume_uio()
655 		KASSERT(uio->uio_td != NULL,  in sglist_consume_uio()
657 		pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace);  in sglist_consume_uio()
662 	while (resid > 0 && uio->uio_resid) {  in sglist_consume_uio()
663 		iov = uio->uio_iov;  in sglist_consume_uio()
664 		len = iov->iov_len;  in sglist_consume_uio()
665 		if (len == 0) {  in sglist_consume_uio()
666 			uio->uio_iov++;  in sglist_consume_uio()
667 			uio->uio_iovcnt--;  in sglist_consume_uio()
670 		if (len > resid)  in sglist_consume_uio()
671 			len = resid;  in sglist_consume_uio()
677 		error = _sglist_append_buf(sg, iov->iov_base, len, pmap, &done);  in sglist_consume_uio()
678 		iov->iov_base = (char *)iov->iov_base + done;  in sglist_consume_uio()
679 		iov->iov_len -= done;  in sglist_consume_uio()
680 		uio->uio_resid -= done;  in sglist_consume_uio()
681 		uio->uio_offset += done;  in sglist_consume_uio()
682 		resid -= done;  in sglist_consume_uio()
694 sglist_build(void *buf, size_t len, int mflags)  in sglist_build()  argument
699 	if (len == 0)  in sglist_build()
702 	nsegs = sglist_count(buf, len);  in sglist_build()
706 	if (sglist_append(sg, buf, len) != 0) {  in sglist_build()
723 	new = sglist_alloc(sg->sg_maxseg, mflags);  in sglist_clone()
726 	new->sg_nseg = sg->sg_nseg;  in sglist_clone()
727 	bcopy(sg->sg_segs, new->sg_segs, sizeof(struct sglist_seg) *  in sglist_clone()
728 	    sg->sg_nseg);  in sglist_clone()
743 	for (i = 0; i < sg->sg_nseg; i++)  in sglist_length()
744 		space += sg->sg_segs[i].ss_len;  in sglist_length()
773 	if (original->sg_refs > 1)  in sglist_split()
780 	for (i = 0; i < original->sg_nseg; i++) {  in sglist_split()
781 		space += original->sg_segs[i].ss_len;  in sglist_split()
790 			split = space - length;  in sglist_split()
806 		if (sg->sg_maxseg < count)  in sglist_split()
808 		if (sg->sg_nseg != 0)  in sglist_split()
813 	bcopy(original->sg_segs, sg->sg_segs, count *  in sglist_split()
815 	sg->sg_nseg = count;  in sglist_split()
821 	 * 'count - 1' segments from 'original' now.  in sglist_split()
824 		count--;  in sglist_split()
825 		sg->sg_segs[count].ss_len -= split;  in sglist_split()
826 		original->sg_segs[count].ss_paddr =  in sglist_split()
827 		    sg->sg_segs[count].ss_paddr + split;  in sglist_split()
828 		original->sg_segs[count].ss_len = split;  in sglist_split()
832 	original->sg_nseg -= count;  in sglist_split()
833 	bcopy(original->sg_segs + count, original->sg_segs, count *  in sglist_split()
850 	if (second->sg_nseg == 0)  in sglist_join()
858 	flast = &first->sg_segs[first->sg_nseg - 1];  in sglist_join()
859 	sfirst = &second->sg_segs[0];  in sglist_join()
860 	if (first->sg_nseg != 0 &&  in sglist_join()
861 	    flast->ss_paddr + flast->ss_len == sfirst->ss_paddr)  in sglist_join()
865 	if (first->sg_nseg + second->sg_nseg - append > first->sg_maxseg)  in sglist_join()
870 		flast->ss_len += sfirst->ss_len;  in sglist_join()
873 	bcopy(first->sg_segs + first->sg_nseg, second->sg_segs + append,  in sglist_join()
874 	    (second->sg_nseg - append) * sizeof(struct sglist_seg));  in sglist_join()
875 	first->sg_nseg += second->sg_nseg - append;  in sglist_join()
914 	for (i = 0; i < original->sg_nseg; i++) {  in sglist_slice()
915 		space += original->sg_segs[i].ss_len;  in sglist_slice()
924 				foffs = offset - (space -  in sglist_slice()
925 				    original->sg_segs[i].ss_len);  in sglist_slice()
937 				loffs = space - end;  in sglist_slice()
956 		if (sg->sg_maxseg < count)  in sglist_slice()
958 		if (sg->sg_nseg != 0)  in sglist_slice()
966 	bcopy(original->sg_segs + fseg, sg->sg_segs,  in sglist_slice()
968 	sg->sg_nseg = count;  in sglist_slice()
972 		sg->sg_segs[0].ss_paddr += foffs;  in sglist_slice()
973 		sg->sg_segs[0].ss_len -= foffs;  in sglist_slice()
975 		    (long)sg->sg_segs[0].ss_paddr, sg->sg_segs[0].ss_len);  in sglist_slice()
978 		sg->sg_segs[count - 1].ss_len -= loffs;  in sglist_slice()
979 		CTR2(KTR_DEV, "sglist_slice seg[%d]: len %08x", count - 1,  in sglist_slice()
980 		    sg->sg_segs[count - 1].ss_len);  in sglist_slice()