Lines Matching +full:pga +full:- +full:value
1 /*-
2 * Copyright (c) 2013-2015 Gleb Smirnoff <glebius@FreeBSD.org>
116 KASSERT(sfs->count == 0, ("sendfile sync %p still busy", sfs)); in sendfile_sync_destroy()
118 cv_destroy(&sfs->cv); in sendfile_sync_destroy()
119 mtx_destroy(&sfs->mtx); in sendfile_sync_destroy()
126 mtx_lock(&sfs->mtx); in sendfile_sync_signal()
127 KASSERT(sfs->count > 0, ("sendfile sync %p not busy", sfs)); in sendfile_sync_signal()
128 if (--sfs->count == 0) { in sendfile_sync_signal()
129 if (!sfs->waiting) { in sendfile_sync_signal()
134 cv_signal(&sfs->cv); in sendfile_sync_signal()
137 mtx_unlock(&sfs->mtx); in sendfile_sync_signal()
157 if (req->newptr) in sfstat_sysctl()
173 KASSERT(m->m_flags & M_EXT && m->m_ext.ext_type == EXT_SFBUF, in sendfile_free_mext()
176 sf = m->m_ext.ext_arg1; in sendfile_free_mext()
178 flags = (m->m_ext.ext_flags & EXT_FLAG_NOCACHE) != 0 ? VPR_TRYFREE : 0; in sendfile_free_mext()
183 if (m->m_ext.ext_flags & EXT_FLAG_SYNC) { in sendfile_free_mext()
184 struct sendfile_sync *sfs = m->m_ext.ext_arg2; in sendfile_free_mext()
198 cache_last = m->m_ext.ext_flags & EXT_FLAG_CACHE_LAST; in sendfile_free_mext_pg()
199 flags = (m->m_ext.ext_flags & EXT_FLAG_NOCACHE) != 0 ? VPR_TRYFREE : 0; in sendfile_free_mext_pg()
201 for (i = 0; i < m->m_epg_npgs; i++) { in sendfile_free_mext_pg()
202 if (cache_last && i == m->m_epg_npgs - 1) in sendfile_free_mext_pg()
204 pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]); in sendfile_free_mext_pg()
208 if (m->m_ext.ext_flags & EXT_FLAG_SYNC) { in sendfile_free_mext_pg()
209 struct sendfile_sync *sfs = m->m_ext.ext_arg1; in sendfile_free_mext_pg()
223 return (omin(PAGE_SIZE - (off & PAGE_MASK), len)); in xfsize()
225 if (i == n - 1 && ((off + len) & PAGE_MASK) > 0) in xfsize()
256 *space -= xfsize(old - 1, old, off, *space); in fixspace()
257 old--; in fixspace()
265 *space -= xfsize(0, old, off, *space); in fixspace()
270 *space -= (old - new) * PAGE_SIZE; in fixspace()
276 * Wait for all in-flight ios to complete, we must not unwire pages
282 while (atomic_load_int(&sfio->nios) != 1) in sendfile_iowait()
302 sfio->error = error; in sendfile_iodone()
311 * unbusied the swapped-in pages, they can become in sendfile_iodone()
317 sfio->pa[(pa[0]->pindex - sfio->pindex0) + i] = in sendfile_iodone()
318 pa[i] = vm_page_relookup(sfio->obj, in sendfile_iodone()
319 pa[0]->pindex + i); in sendfile_iodone()
328 if (!refcount_release(&sfio->nios)) in sendfile_iodone()
332 for (i = 1; i < sfio->npages; i++) { in sendfile_iodone()
333 if (sfio->pa[i] == NULL) in sendfile_iodone()
335 KASSERT(vm_page_wired(sfio->pa[i]), in sendfile_iodone()
336 ("sfio %p page %d %p not wired", sfio, i, sfio->pa[i])); in sendfile_iodone()
339 KASSERT(sfio->pa[0]->object == sfio->pa[i]->object, in sendfile_iodone()
341 sfio->pa[i], sfio->pa[0]->object, sfio->pa[i]->object)); in sendfile_iodone()
342 KASSERT(sfio->pa[0]->pindex + i == sfio->pa[i]->pindex, in sendfile_iodone()
344 sfio->pa[i], (uintmax_t)sfio->pa[0]->pindex, in sendfile_iodone()
345 (uintmax_t)sfio->pa[i]->pindex)); in sendfile_iodone()
349 vm_object_pip_wakeup(sfio->obj); in sendfile_iodone()
351 if (sfio->m == NULL) { in sendfile_iodone()
356 * pr_send() hadn't been executed - nothing had been sent in sendfile_iodone()
359 MPASS((curthread->td_pflags & TDP_KTHREAD) == 0); in sendfile_iodone()
365 if ((sfio->m->m_flags & M_EXTPG) != 0) in sendfile_iodone()
366 KASSERT(sfio->tls == sfio->m->m_epg_tls, in sendfile_iodone()
369 KASSERT(sfio->tls == NULL, in sendfile_iodone()
370 ("non-ext_pgs mbuf with TLS session")); in sendfile_iodone()
372 so = sfio->so; in sendfile_iodone()
373 CURVNET_SET_QUIET(so->so_vnet); in sendfile_iodone()
374 if (__predict_false(sfio->error)) { in sendfile_iodone()
386 so->so_proto->pr_abort(so); in sendfile_iodone()
387 so->so_error = EIO; in sendfile_iodone()
389 mb_free_notready(sfio->m, sfio->npages); in sendfile_iodone()
391 } else if (sfio->tls != NULL && sfio->tls->mode == TCP_TLS_MODE_SW) { in sendfile_iodone()
401 ktls_enqueue(sfio->m, so, sfio->npages); in sendfile_iodone()
405 (void)so->so_proto->pr_ready(so, sfio->m, sfio->npages); in sendfile_iodone()
416 * Iterate through pages vector and request paging for non-valid pages.
425 pa = sfio->pa; in sendfile_swapin()
426 npages = sfio->npages; in sendfile_swapin()
429 sfio->pindex0 = OFF_TO_IDX(off); in sendfile_swapin()
470 MPASS(pa[i]->dirty == 0); in sendfile_swapin()
481 count = min(a + 1, npages - i); in sendfile_swapin()
500 for (j = i + count - 1; j > i; j--) { in sendfile_swapin()
505 count--; in sendfile_swapin()
512 * The last page in the run pa[i + count - 1] is in sendfile_swapin()
514 * is not replaced with bogus, thus -1 in the loop end in sendfile_swapin()
517 MPASS(pa[i + count - 1]->valid != VM_PAGE_BITS_ALL); in sendfile_swapin()
518 for (j = i + 1; j < i + count - 1; j++) { in sendfile_swapin()
528 refcount_acquire(&sfio->nios); in sendfile_swapin()
587 if (fp->f_type == DTYPE_VNODE) { in sendfile_getobj()
588 vp = fp->f_vnode; in sendfile_getobj()
590 if (vp->v_type != VREG) { in sendfile_getobj()
594 *bsize = vp->v_mount->mnt_stat.f_iosize; in sendfile_getobj()
595 obj = vp->v_object; in sendfile_getobj()
606 if (obj->type == OBJT_VNODE) { in sendfile_getobj()
608 *obj_size = obj->un_pager.vnp.vnp_size; in sendfile_getobj()
610 error = vn_getsize_locked(vp, obj_size, td->td_ucred); in sendfile_getobj()
615 } else if (fp->f_type == DTYPE_SHM) { in sendfile_getobj()
616 shmfd = fp->f_data; in sendfile_getobj()
617 obj = shmfd->shm_object; in sendfile_getobj()
619 *obj_size = shmfd->shm_size; in sendfile_getobj()
625 if ((obj->flags & OBJ_DEAD) != 0) { in sendfile_getobj()
663 *so = (*sock_fp)->f_data; in sendfile_getsock()
664 if ((*so)->so_type != SOCK_STREAM) in sendfile_getsock()
667 * SCTP one-to-one style sockets currently don't work with in sendfile_getsock()
670 if ((*so)->so_proto->pr_protocol == IPPROTO_SCTP) in sendfile_getsock()
688 * achieve high performance sending with sendfile(2) a non-blocking in sendfile_wait_generic()
701 if (so->so_snd.sb_flags & SB_AUTOLOWAT) { in sendfile_wait_generic()
702 if (so->so_snd.sb_lowat < so->so_snd.sb_hiwat / 2) in sendfile_wait_generic()
703 so->so_snd.sb_lowat = so->so_snd.sb_hiwat / 2; in sendfile_wait_generic()
704 if (so->so_snd.sb_lowat < PAGE_SIZE && in sendfile_wait_generic()
705 so->so_snd.sb_hiwat >= PAGE_SIZE) in sendfile_wait_generic()
706 so->so_snd.sb_lowat = PAGE_SIZE; in sendfile_wait_generic()
709 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { in sendfile_wait_generic()
712 } else if (so->so_error) { in sendfile_wait_generic()
713 error = so->so_error; in sendfile_wait_generic()
714 so->so_error = 0; in sendfile_wait_generic()
717 if ((so->so_state & SS_ISCONNECTED) == 0) { in sendfile_wait_generic()
722 *space = sbspace(&so->so_snd); in sendfile_wait_generic()
723 if (*space < need && (*space <= 0 || *space < so->so_snd.sb_lowat)) { in sendfile_wait_generic()
724 if (so->so_state & SS_NBIO) { in sendfile_wait_generic()
757 vm_page_t pga; in vn_sendfile() local
793 pr = so->so_proto; in vn_sendfile()
796 error = mac_socket_check_send(td->td_ucred, so); in vn_sendfile()
808 td->td_proc->p_comm, td->td_proc->p_pid); in vn_sendfile()
810 mtx_init(&sfs->mtx, "sendfile", NULL, MTX_DEF); in vn_sendfile()
811 cv_init(&sfs->cv, "sendfile"); in vn_sendfile()
812 sfs->waiting = true; in vn_sendfile()
815 rem = nbytes ? omin(nbytes, obj_size - offset) : obj_size - offset; in vn_sendfile()
820 * XXXRW: Historically this has assumed non-interruptibility, so now in vn_sendfile()
826 CURVNET_SET(so->so_vnet); in vn_sendfile()
828 tls = ktls_hold(so->so_snd.sb_tls_info); in vn_sendfile()
849 if ((error = pr->pr_sendfile_wait(so, rem, &space)) != 0) in vn_sendfile()
857 if (hdr_uio != NULL && hdr_uio->uio_resid > 0) { in vn_sendfile()
858 hdr_uio->uio_td = td; in vn_sendfile()
859 hdr_uio->uio_rw = UIO_WRITE; in vn_sendfile()
863 tls->params.max_frame_len, M_EXTPG); in vn_sendfile()
869 space -= hdrlen; in vn_sendfile()
890 if (obj->type == OBJT_VNODE) { in vn_sendfile()
892 nobj_size = obj->un_pager.vnp.vnp_size; in vn_sendfile()
895 error = VOP_GETATTR(vp, &va, td->td_ucred); in vn_sendfile()
910 rem -= off; in vn_sendfile()
922 space -= (PAGE_SIZE - (off & PAGE_MASK)); in vn_sendfile()
925 space += (PAGE_SIZE - (off & PAGE_MASK)); in vn_sendfile()
933 * do any heuristics and use exactly the value supplied by in vn_sendfile()
942 rhpages = howmany(rem + (off & PAGE_MASK), PAGE_SIZE) - in vn_sendfile()
947 rhpages = min(howmany(obj_size - trunc_page(off), PAGE_SIZE) - in vn_sendfile()
952 refcount_init(&sfio->nios, 1); in vn_sendfile()
953 sfio->obj = obj; in vn_sendfile()
954 sfio->error = 0; in vn_sendfile()
955 sfio->m = NULL; in vn_sendfile()
956 sfio->npages = npages; in vn_sendfile()
959 * This doesn't use ktls_hold() because sfio->m will in vn_sendfile()
963 sfio->tls = tls; in vn_sendfile()
979 pa = sfio->pa; in vn_sendfile()
985 * been tested with UNIX-domain sockets. in vn_sendfile()
989 if ((mb_use_ext_pgs && pr->pr_protocol == IPPROTO_TCP) in vn_sendfile()
997 max_pgs = num_pages(tls->params.max_frame_len); in vn_sendfile()
1003 ext_pgs_idx = max_pgs - 1; in vn_sendfile()
1014 sfio->npages = i; in vn_sendfile()
1018 pga = pa[i]; in vn_sendfile()
1019 if (pga == bogus_page) in vn_sendfile()
1020 pga = vm_page_relookup(obj, sfio->pindex0 + i); in vn_sendfile()
1031 m0->m_ext.ext_flags |= in vn_sendfile()
1039 if ((npages - i <= max_pgs) && in vn_sendfile()
1042 m0->m_ext.ext_flags |= in vn_sendfile()
1046 m0->m_ext.ext_flags |= in vn_sendfile()
1048 m0->m_ext.ext_arg1 = sfs; in vn_sendfile()
1049 mtx_lock(&sfs->mtx); in vn_sendfile()
1050 sfs->count++; in vn_sendfile()
1051 mtx_unlock(&sfs->mtx); in vn_sendfile()
1057 mtail->m_next = m0; in vn_sendfile()
1061 m0->m_epg_1st_off = in vn_sendfile()
1065 mtail->m_flags |= M_NOTREADY; in vn_sendfile()
1066 m0->m_epg_nrdy++; in vn_sendfile()
1069 m0->m_epg_pa[ext_pgs_idx] = VM_PAGE_TO_PHYS(pga); in vn_sendfile()
1070 m0->m_epg_npgs++; in vn_sendfile()
1072 m0->m_epg_last_len = xfs; in vn_sendfile()
1074 mtail->m_len += xfs; in vn_sendfile()
1075 mtail->m_ext.ext_size += PAGE_SIZE; in vn_sendfile()
1088 sf = sf_buf_alloc(pga, in vn_sendfile()
1100 sfio->npages = i; in vn_sendfile()
1105 m0->m_ext.ext_buf = (char *)sf_buf_kva(sf); in vn_sendfile()
1106 m0->m_ext.ext_size = PAGE_SIZE; in vn_sendfile()
1107 m0->m_ext.ext_arg1 = sf; in vn_sendfile()
1108 m0->m_ext.ext_type = EXT_SFBUF; in vn_sendfile()
1109 m0->m_ext.ext_flags = EXT_FLAG_EMBREF; in vn_sendfile()
1110 m0->m_ext.ext_free = sendfile_free_mext; in vn_sendfile()
1119 (i != npages - 1 || in vn_sendfile()
1122 m0->m_ext.ext_flags |= EXT_FLAG_NOCACHE; in vn_sendfile()
1124 m0->m_ext.ext_flags |= EXT_FLAG_SYNC; in vn_sendfile()
1125 m0->m_ext.ext_arg2 = sfs; in vn_sendfile()
1126 mtx_lock(&sfs->mtx); in vn_sendfile()
1127 sfs->count++; in vn_sendfile()
1128 mtx_unlock(&sfs->mtx); in vn_sendfile()
1130 m0->m_ext.ext_count = 1; in vn_sendfile()
1131 m0->m_flags |= (M_EXT | M_RDONLY); in vn_sendfile()
1133 m0->m_flags |= M_NOTREADY; in vn_sendfile()
1134 m0->m_data = (char *)sf_buf_kva(sf) + in vn_sendfile()
1136 m0->m_len = xfsize(i, npages, off, space); in vn_sendfile()
1140 mtail->m_next = m0; in vn_sendfile()
1151 rem -= space; in vn_sendfile()
1159 m0 = mhtail->m_next = m; in vn_sendfile()
1192 if (tls != NULL && tls->mode == TCP_TLS_MODE_SW) { in vn_sendfile()
1193 error = pr->pr_send(so, PRUS_NOTREADY, m, NULL, in vn_sendfile()
1203 error = pr->pr_send(so, 0, m, NULL, NULL, td); in vn_sendfile()
1205 sfio->so = so; in vn_sendfile()
1206 sfio->m = m0; in vn_sendfile()
1208 error = pr->pr_send(so, PRUS_NOTREADY, m, NULL, NULL, in vn_sendfile()
1213 if (so->so_proto->pr_protocol == IPPROTO_TCP) { in vn_sendfile()
1238 sbytes += td->td_retval[0]; in vn_sendfile()
1247 * If there was no error we have to clear td->td_retval[0] in vn_sendfile()
1251 td->td_retval[0] = 0; in vn_sendfile()
1268 mtx_lock(&sfs->mtx); in vn_sendfile()
1269 if (sfs->count != 0) in vn_sendfile()
1270 error = cv_wait_sig(&sfs->cv, &sfs->mtx); in vn_sendfile()
1271 if (sfs->count == 0) { in vn_sendfile()
1274 sfs->waiting = false; in vn_sendfile()
1275 mtx_unlock(&sfs->mtx); in vn_sendfile()
1301 if (uap->offset < 0) in sendfile()
1307 if (uap->hdtr != NULL) { in sendfile()
1308 error = copyin(uap->hdtr, &hdtr, sizeof(hdtr)); in sendfile()
1323 if (uap->nbytes > hdr_uio->uio_resid) in sendfile()
1324 uap->nbytes -= hdr_uio->uio_resid; in sendfile()
1326 uap->nbytes = 0; in sendfile()
1338 AUDIT_ARG_FD(uap->fd); in sendfile()
1344 if ((error = fget_read(td, uap->fd, &cap_pread_rights, &fp)) != 0) in sendfile()
1347 error = fo_sendfile(fp, uap->s, hdr_uio, trl_uio, uap->offset, in sendfile()
1348 uap->nbytes, &sbytes, uap->flags, td); in sendfile()
1351 if (uap->sbytes != NULL) in sendfile()
1352 (void)copyout(&sbytes, uap->sbytes, sizeof(off_t)); in sendfile()
1384 args.fd = uap->fd; in freebsd4_sendfile()
1385 args.s = uap->s; in freebsd4_sendfile()
1386 args.offset = uap->offset; in freebsd4_sendfile()
1387 args.nbytes = uap->nbytes; in freebsd4_sendfile()
1388 args.hdtr = uap->hdtr; in freebsd4_sendfile()
1389 args.sbytes = uap->sbytes; in freebsd4_sendfile()
1390 args.flags = uap->flags; in freebsd4_sendfile()