Lines Matching +full:page +full:- +full:offset
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
52 KASSERT(PMAP_HAS_DMAP, ("direct-map required")); in phys_copyback()
58 todo = min(PAGE_SIZE - page_off, size); in phys_copyback()
61 size -= todo; in phys_copyback()
75 while (vlist->ds_len <= off) { in vlist_copyback()
78 off -= vlist->ds_len; in vlist_copyback()
80 sglist_cnt--; in vlist_copyback()
88 if (todo > vlist->ds_len - off) in vlist_copyback()
89 todo = vlist->ds_len - off; in vlist_copyback()
91 memcpy((char *)(uintptr_t)vlist->ds_addr + off, p, todo); in vlist_copyback()
94 sglist_cnt--; in vlist_copyback()
95 size -= todo; in vlist_copyback()
107 while (plist->ds_len <= off) { in plist_copyback()
110 off -= plist->ds_len; in plist_copyback()
112 sglist_cnt--; in plist_copyback()
120 if (todo > plist->ds_len - off) in plist_copyback()
121 todo = plist->ds_len - off; in plist_copyback()
123 phys_copyback(plist->ds_addr, off, todo, p); in plist_copyback()
126 sglist_cnt--; in plist_copyback()
127 size -= todo; in plist_copyback()
154 KASSERT(off >= 0, ("%s: invalid offset %d", __func__, off)); in memdesc_copyback()
157 switch (mem->md_type) { in memdesc_copyback()
159 KASSERT(off + size <= mem->md_len, ("copy out of bounds")); in memdesc_copyback()
160 memcpy((char *)mem->u.md_vaddr + off, src, size); in memdesc_copyback()
163 KASSERT(off + size <= mem->md_len, ("copy out of bounds")); in memdesc_copyback()
164 phys_copyback(mem->u.md_paddr, off, size, src); in memdesc_copyback()
167 vlist_copyback(mem->u.md_list, mem->md_nseg, off, size, src); in memdesc_copyback()
170 plist_copyback(mem->u.md_list, mem->md_nseg, off, size, src); in memdesc_copyback()
176 m_copyback(mem->u.md_mbuf, off, size, src); in memdesc_copyback()
179 KASSERT(off + size <= mem->md_len, ("copy out of bounds")); in memdesc_copyback()
180 vmpages_copyback(mem->u.md_ma, mem->md_offset + off, size, in memdesc_copyback()
200 KASSERT(PMAP_HAS_DMAP, ("direct-map required")); in phys_copydata()
206 todo = min(PAGE_SIZE - page_off, size); in phys_copydata()
209 size -= todo; in phys_copydata()
223 while (vlist->ds_len <= off) { in vlist_copydata()
226 off -= vlist->ds_len; in vlist_copydata()
228 sglist_cnt--; in vlist_copydata()
236 if (todo > vlist->ds_len - off) in vlist_copydata()
237 todo = vlist->ds_len - off; in vlist_copydata()
239 memcpy(p, (char *)(uintptr_t)vlist->ds_addr + off, todo); in vlist_copydata()
242 sglist_cnt--; in vlist_copydata()
243 size -= todo; in vlist_copydata()
255 while (plist->ds_len <= off) { in plist_copydata()
258 off -= plist->ds_len; in plist_copydata()
260 sglist_cnt--; in plist_copydata()
268 if (todo > plist->ds_len - off) in plist_copydata()
269 todo = plist->ds_len - off; in plist_copydata()
271 phys_copydata(plist->ds_addr, off, todo, p); in plist_copydata()
274 sglist_cnt--; in plist_copydata()
275 size -= todo; in plist_copydata()
302 KASSERT(off >= 0, ("%s: invalid offset %d", __func__, off)); in memdesc_copydata()
305 switch (mem->md_type) { in memdesc_copydata()
307 KASSERT(off + size <= mem->md_len, ("copy out of bounds")); in memdesc_copydata()
308 memcpy(dst, (const char *)mem->u.md_vaddr + off, size); in memdesc_copydata()
311 KASSERT(off + size <= mem->md_len, ("copy out of bounds")); in memdesc_copydata()
312 phys_copydata(mem->u.md_paddr, off, size, dst); in memdesc_copydata()
315 vlist_copydata(mem->u.md_list, mem->md_nseg, off, size, dst); in memdesc_copydata()
318 plist_copydata(mem->u.md_list, mem->md_nseg, off, size, dst); in memdesc_copydata()
324 m_copydata(mem->u.md_mbuf, off, size, dst); in memdesc_copydata()
327 KASSERT(off + size <= mem->md_len, ("copy out of bounds")); in memdesc_copydata()
328 vmpages_copydata(mem->u.md_ma, mem->md_offset + off, size, in memdesc_copydata()
354 if (m->m_epg_npgs == 0) in can_append_paddr()
358 if (m->m_epg_npgs == MBUF_PEXT_MAX_PGS) in can_append_paddr()
361 /* Can't append a non-page-aligned address to a non-empty mbuf. */ in can_append_paddr()
365 /* Can't append if the last page is not a full page. */ in can_append_paddr()
366 last_len = m->m_epg_last_len; in can_append_paddr()
367 if (m->m_epg_npgs == 1) in can_append_paddr()
368 last_len += m->m_epg_1st_off; in can_append_paddr()
382 /* Append the first page. */ in append_paddr_range()
383 if (m->m_epg_npgs == 0) { in append_paddr_range()
384 m->m_epg_pa[0] = trunc_page(pa); in append_paddr_range()
385 m->m_epg_npgs = 1; in append_paddr_range()
386 m->m_epg_1st_off = pa & PAGE_MASK; in append_paddr_range()
387 m->m_epg_last_len = PAGE_SIZE - m->m_epg_1st_off; in append_paddr_range()
388 if (m->m_epg_last_len > len) in append_paddr_range()
389 m->m_epg_last_len = len; in append_paddr_range()
390 m->m_len = m->m_epg_last_len; in append_paddr_range()
391 len -= m->m_epg_last_len; in append_paddr_range()
392 pa += m->m_epg_last_len; in append_paddr_range()
393 appended += m->m_epg_last_len; in append_paddr_range()
399 while (len >= PAGE_SIZE && m->m_epg_npgs < MBUF_PEXT_MAX_PGS) { in append_paddr_range()
400 m->m_epg_pa[m->m_epg_npgs] = pa; in append_paddr_range()
401 m->m_epg_npgs++; in append_paddr_range()
402 m->m_epg_last_len = PAGE_SIZE; in append_paddr_range()
403 m->m_len += PAGE_SIZE; in append_paddr_range()
405 len -= PAGE_SIZE; in append_paddr_range()
409 /* Final partial page. */ in append_paddr_range()
410 if (len > 0 && m->m_epg_npgs < MBUF_PEXT_MAX_PGS) { in append_paddr_range()
411 KASSERT(len < PAGE_SIZE, ("final page is full page")); in append_paddr_range()
412 m->m_epg_pa[m->m_epg_npgs] = pa; in append_paddr_range()
413 m->m_epg_npgs++; in append_paddr_range()
414 m->m_epg_last_len = len; in append_paddr_range()
415 m->m_len += len; in append_paddr_range()
433 * Trim any partial page at the end, but not if it's in paddr_ext_mbuf()
434 * the only page. in paddr_ext_mbuf()
438 len = end - pa; in paddr_ext_mbuf()
448 tail->m_next = extpg_alloc(cb_arg, how); in paddr_ext_mbuf()
449 if (tail->m_next == NULL) in paddr_ext_mbuf()
451 tail = tail->m_next; in paddr_ext_mbuf()
459 len -= appended; in paddr_ext_mbuf()
471 struct bus_dma_segment *vlist, u_int sglist_cnt, size_t offset, in vlist_ext_mbuf() argument
479 while (vlist->ds_len <= offset) { in vlist_ext_mbuf()
482 offset -= vlist->ds_len; in vlist_ext_mbuf()
484 sglist_cnt--; in vlist_ext_mbuf()
492 if (todo > vlist->ds_len - offset) in vlist_ext_mbuf()
493 todo = vlist->ds_len - offset; in vlist_ext_mbuf()
495 n = ext_alloc(cb_arg, how, (char *)(uintptr_t)vlist->ds_addr + in vlist_ext_mbuf()
496 offset, todo); in vlist_ext_mbuf()
504 tail->m_next = n; in vlist_ext_mbuf()
508 offset = 0; in vlist_ext_mbuf()
510 sglist_cnt--; in vlist_ext_mbuf()
511 len -= todo; in vlist_ext_mbuf()
522 struct bus_dma_segment *plist, u_int sglist_cnt, size_t offset, size_t len, in plist_ext_mbuf() argument
529 while (plist->ds_len <= offset) { in plist_ext_mbuf()
532 offset -= plist->ds_len; in plist_ext_mbuf()
534 sglist_cnt--; in plist_ext_mbuf()
544 pa = plist->ds_addr + offset; in plist_ext_mbuf()
546 if (todo > plist->ds_len - offset) in plist_ext_mbuf()
547 todo = plist->ds_len - offset; in plist_ext_mbuf()
551 * partial page, but only if there is more data in plist_ext_mbuf()
553 * some data must be sent, so only drop the final page in plist_ext_mbuf()
564 * page. in plist_ext_mbuf()
569 todo = end - pa; in plist_ext_mbuf()
572 offset = 0; in plist_ext_mbuf()
573 len -= todo; in plist_ext_mbuf()
579 tail->m_next = extpg_alloc(cb_arg, how); in plist_ext_mbuf()
580 if (tail->m_next == NULL) in plist_ext_mbuf()
582 tail = tail->m_next; in plist_ext_mbuf()
589 todo -= appended; in plist_ext_mbuf()
603 vm_page_t *ma, size_t offset, size_t len, size_t *actual_len, in vmpages_ext_mbuf() argument
608 while (offset >= PAGE_SIZE) { in vmpages_ext_mbuf()
610 offset -= PAGE_SIZE; in vmpages_ext_mbuf()
617 * Trim any partial page at the end, but not if it's in vmpages_ext_mbuf()
618 * the only page. in vmpages_ext_mbuf()
620 end = trunc_page(offset + len); in vmpages_ext_mbuf()
621 if (end > offset) in vmpages_ext_mbuf()
622 len = end - offset; in vmpages_ext_mbuf()
630 /* First page. */ in vmpages_ext_mbuf()
631 m->m_epg_pa[0] = VM_PAGE_TO_PHYS(*ma); in vmpages_ext_mbuf()
633 m->m_epg_npgs = 1; in vmpages_ext_mbuf()
634 m->m_epg_1st_off = offset; in vmpages_ext_mbuf()
635 m->m_epg_last_len = PAGE_SIZE - offset; in vmpages_ext_mbuf()
636 if (m->m_epg_last_len > len) in vmpages_ext_mbuf()
637 m->m_epg_last_len = len; in vmpages_ext_mbuf()
638 m->m_len = m->m_epg_last_len; in vmpages_ext_mbuf()
639 len -= m->m_epg_last_len; in vmpages_ext_mbuf()
643 if (tail->m_epg_npgs == MBUF_PEXT_MAX_PGS) { in vmpages_ext_mbuf()
645 tail->m_next = extpg_alloc(cb_arg, how); in vmpages_ext_mbuf()
646 if (tail->m_next == NULL) in vmpages_ext_mbuf()
648 tail = tail->m_next; in vmpages_ext_mbuf()
651 tail->m_epg_pa[tail->m_epg_npgs] = VM_PAGE_TO_PHYS(*ma); in vmpages_ext_mbuf()
653 tail->m_epg_npgs++; in vmpages_ext_mbuf()
654 tail->m_epg_last_len = PAGE_SIZE; in vmpages_ext_mbuf()
655 tail->m_len += PAGE_SIZE; in vmpages_ext_mbuf()
656 len -= PAGE_SIZE; in vmpages_ext_mbuf()
659 /* Last partial page. */ in vmpages_ext_mbuf()
661 if (tail->m_epg_npgs == MBUF_PEXT_MAX_PGS) { in vmpages_ext_mbuf()
663 tail->m_next = extpg_alloc(cb_arg, how); in vmpages_ext_mbuf()
664 if (tail->m_next == NULL) in vmpages_ext_mbuf()
666 tail = tail->m_next; in vmpages_ext_mbuf()
669 tail->m_epg_pa[tail->m_epg_npgs] = VM_PAGE_TO_PHYS(*ma); in vmpages_ext_mbuf()
671 tail->m_epg_npgs++; in vmpages_ext_mbuf()
672 tail->m_epg_last_len = len; in vmpages_ext_mbuf()
673 tail->m_len += len; in vmpages_ext_mbuf()
688 mbuf_subchain(struct mbuf *m0, size_t offset, size_t len, in mbuf_subchain() argument
694 while (offset >= m0->m_len) { in mbuf_subchain()
695 offset -= m0->m_len; in mbuf_subchain()
696 m0 = m0->m_next; in mbuf_subchain()
700 totlen = m0->m_len - offset; in mbuf_subchain()
707 m->m_len = totlen; in mbuf_subchain()
708 if (m0->m_flags & (M_EXT | M_EXTPG)) { in mbuf_subchain()
709 m->m_data = m0->m_data + offset; in mbuf_subchain()
712 memcpy(mtod(m, void *), mtodo(m0, offset), m->m_len); in mbuf_subchain()
715 m0 = m0->m_next; in mbuf_subchain()
716 len -= totlen; in mbuf_subchain()
722 if (can_truncate && m0->m_len > len) in mbuf_subchain()
725 tail->m_next = m_get(how, MT_DATA); in mbuf_subchain()
726 if (tail->m_next == NULL) in mbuf_subchain()
728 tail = tail->m_next; in mbuf_subchain()
729 tail->m_len = m0->m_len; in mbuf_subchain()
730 if (m0->m_flags & (M_EXT | M_EXTPG)) { in mbuf_subchain()
731 tail->m_data = m0->m_data; in mbuf_subchain()
735 tail->m_len); in mbuf_subchain()
737 totlen += tail->m_len; in mbuf_subchain()
738 m0 = m0->m_next; in mbuf_subchain()
739 len -= tail->m_len; in mbuf_subchain()
752 size_t offset, size_t len, size_t *actual_len, bool can_truncate) in memdesc_alloc_ext_mbufs() argument
757 switch (mem->md_type) { in memdesc_alloc_ext_mbufs()
760 (char *)mem->u.md_vaddr + offset, len, &done); in memdesc_alloc_ext_mbufs()
763 m = paddr_ext_mbuf(extpg_alloc, cb_arg, how, mem->u.md_paddr + in memdesc_alloc_ext_mbufs()
764 offset, len, &done, can_truncate); in memdesc_alloc_ext_mbufs()
767 m = vlist_ext_mbuf(ext_alloc, cb_arg, how, mem->u.md_list, in memdesc_alloc_ext_mbufs()
768 mem->md_nseg, offset, len, &done); in memdesc_alloc_ext_mbufs()
771 m = plist_ext_mbuf(extpg_alloc, cb_arg, how, mem->u.md_list, in memdesc_alloc_ext_mbufs()
772 mem->md_nseg, offset, len, &done, can_truncate); in memdesc_alloc_ext_mbufs()
777 m = mbuf_subchain(mem->u.md_mbuf, offset, len, &done, in memdesc_alloc_ext_mbufs()
781 m = vmpages_ext_mbuf(extpg_alloc, cb_arg, how, mem->u.md_ma, in memdesc_alloc_ext_mbufs()
782 mem->md_offset + offset, len, &done, can_truncate); in memdesc_alloc_ext_mbufs()