Lines Matching +full:isp +full:- +full:0
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
95 SYSCTL_NODE(_kern_icl, OID_AUTO, soft, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
99 &coalesce, 0, "Try to coalesce PDUs before sending");
102 &partial_receive_len, 0, "Minimum read size for partially received "
106 &max_data_segment_length, 0, "Maximum data segment length");
109 &first_burst_length, 0, "First burst length");
112 &max_burst_length, 0, "Maximum burst length");
115 &sendspace, 0, "Default send socket buffer size");
118 &recvspace, 0, "Default receive socket buffer size");
169 { 0, 0 }
177 if (ic->ic_socket == NULL) in icl_conn_fail()
183 ic->ic_socket->so_error = EDOOFUS; in icl_conn_fail()
184 (ic->ic_error)(ic); in icl_conn_fail()
190 struct icl_soft_pdu *isp = (struct icl_soft_pdu *)ip; in icl_soft_conn_pdu_free() local
192 KASSERT(isp->ref_cnt == 0, ("freeing active PDU")); in icl_soft_conn_pdu_free()
193 m_freem(ip->ip_bhs_mbuf); in icl_soft_conn_pdu_free()
194 m_freem(ip->ip_ahs_mbuf); in icl_soft_conn_pdu_free()
195 m_freem(ip->ip_data_mbuf); in icl_soft_conn_pdu_free()
196 uma_zfree(icl_soft_pdu_zone, isp); in icl_soft_conn_pdu_free()
198 refcount_release(&ic->ic_outstanding_pdus); in icl_soft_conn_pdu_free()
205 struct icl_soft_pdu *isp = (struct icl_soft_pdu *)ip; in icl_soft_pdu_call_cb() local
207 if (isp->cb != NULL) in icl_soft_pdu_call_cb()
208 isp->cb(ip, isp->error); in icl_soft_pdu_call_cb()
210 refcount_release(&ip->ip_conn->ic_outstanding_pdus); in icl_soft_pdu_call_cb()
212 uma_zfree(icl_soft_pdu_zone, isp); in icl_soft_pdu_call_cb()
218 struct icl_soft_pdu *isp = (struct icl_soft_pdu *)ip; in icl_soft_pdu_done() local
220 if (error != 0) in icl_soft_pdu_done()
221 isp->error = error; in icl_soft_pdu_done()
223 m_freem(ip->ip_bhs_mbuf); in icl_soft_pdu_done()
224 ip->ip_bhs_mbuf = NULL; in icl_soft_pdu_done()
225 m_freem(ip->ip_ahs_mbuf); in icl_soft_pdu_done()
226 ip->ip_ahs_mbuf = NULL; in icl_soft_pdu_done()
227 m_freem(ip->ip_data_mbuf); in icl_soft_pdu_done()
228 ip->ip_data_mbuf = NULL; in icl_soft_pdu_done()
230 if (atomic_fetchadd_int(&isp->ref_cnt, -1) == 1) in icl_soft_pdu_done()
237 struct icl_soft_pdu *isp = (struct icl_soft_pdu *)mb->m_ext.ext_arg1; in icl_soft_mbuf_done() local
239 icl_soft_pdu_call_cb(&isp->ip); in icl_soft_mbuf_done()
248 struct icl_soft_pdu *isp; in icl_soft_conn_new_pdu() local
252 refcount_acquire(&ic->ic_outstanding_pdus); in icl_soft_conn_new_pdu()
254 isp = uma_zalloc(icl_soft_pdu_zone, flags | M_ZERO); in icl_soft_conn_new_pdu()
255 if (isp == NULL) { in icl_soft_conn_new_pdu()
258 refcount_release(&ic->ic_outstanding_pdus); in icl_soft_conn_new_pdu()
262 ip = &isp->ip; in icl_soft_conn_new_pdu()
263 ip->ip_conn = ic; in icl_soft_conn_new_pdu()
266 ip->ip_bhs_mbuf = m_gethdr(flags, MT_DATA); in icl_soft_conn_new_pdu()
267 if (ip->ip_bhs_mbuf == NULL) { in icl_soft_conn_new_pdu()
272 ip->ip_bhs = mtod(ip->ip_bhs_mbuf, struct iscsi_bhs *); in icl_soft_conn_new_pdu()
273 memset(ip->ip_bhs, 0, sizeof(struct iscsi_bhs)); in icl_soft_conn_new_pdu()
274 ip->ip_bhs_mbuf->m_len = sizeof(struct iscsi_bhs); in icl_soft_conn_new_pdu()
283 return (request->ip_bhs->bhs_total_ahs_len * 4); in icl_pdu_ahs_length()
289 uint32_t len = 0; in icl_pdu_data_segment_length()
291 len += request->ip_bhs->bhs_data_segment_len[0]; in icl_pdu_data_segment_length()
293 len += request->ip_bhs->bhs_data_segment_len[1]; in icl_pdu_data_segment_length()
295 len += request->ip_bhs->bhs_data_segment_len[2]; in icl_pdu_data_segment_length()
312 response->ip_bhs->bhs_data_segment_len[2] = len; in icl_pdu_set_data_segment_length()
313 response->ip_bhs->bhs_data_segment_len[1] = len >> 8; in icl_pdu_set_data_segment_length()
314 response->ip_bhs->bhs_data_segment_len[0] = len >> 16; in icl_pdu_set_data_segment_length()
321 if ((ip->ip_data_len % 4) != 0) in icl_pdu_padding()
322 return (4 - (ip->ip_data_len % 4)); in icl_pdu_padding()
324 return (0); in icl_pdu_padding()
332 KASSERT(response->ip_ahs_len == 0, ("responding with AHS")); in icl_pdu_size()
334 len = sizeof(struct iscsi_bhs) + response->ip_data_len + in icl_pdu_size()
336 if (response->ip_conn->ic_header_crc32c) in icl_pdu_size()
338 if (response->ip_data_len != 0 && response->ip_conn->ic_data_crc32c) in icl_pdu_size()
348 m_copydata(*r, 0, s, buf); in icl_soft_receive_buf()
350 while ((*r) != NULL && (*r)->m_len == 0) in icl_soft_receive_buf()
352 *rs -= s; in icl_soft_receive_buf()
359 request->ip_ahs_len = icl_pdu_ahs_length(request); in icl_pdu_receive_ahs()
360 if (request->ip_ahs_len == 0) in icl_pdu_receive_ahs()
363 request->ip_ahs_mbuf = *r; in icl_pdu_receive_ahs()
364 *r = m_split(request->ip_ahs_mbuf, request->ip_ahs_len, M_WAITOK); in icl_pdu_receive_ahs()
365 *rs -= request->ip_ahs_len; in icl_pdu_receive_ahs()
374 return (0); in mbuf_crc32c_helper()
380 uint32_t digest = 0xffffffff; in icl_mbuf_to_crc32c()
382 m_apply(m0, 0, len, mbuf_crc32c_helper, &digest); in icl_mbuf_to_crc32c()
383 digest = digest ^ 0xffffffff; in icl_mbuf_to_crc32c()
393 if (request->ip_conn->ic_header_crc32c == false) in icl_pdu_check_header_digest()
394 return (0); in icl_pdu_check_header_digest()
400 request->ip_bhs_mbuf->m_next = request->ip_ahs_mbuf; in icl_pdu_check_header_digest()
401 valid_digest = icl_mbuf_to_crc32c(request->ip_bhs_mbuf, ISCSI_BHS_SIZE); in icl_pdu_check_header_digest()
402 request->ip_bhs_mbuf->m_next = NULL; in icl_pdu_check_header_digest()
404 ICL_WARN("header digest check failed; got 0x%x, " in icl_pdu_check_header_digest()
405 "should be 0x%x", received_digest, valid_digest); in icl_pdu_check_header_digest()
406 return (-1); in icl_pdu_check_header_digest()
409 return (0); in icl_pdu_check_header_digest()
422 if (len == 0) in icl_pdu_data_segment_receive_len()
423 return (0); in icl_pdu_data_segment_receive_len()
429 KASSERT(len > request->ip_data_len, ("len <= request->ip_data_len")); in icl_pdu_data_segment_receive_len()
430 len -= request->ip_data_len; in icl_pdu_data_segment_receive_len()
438 #if 0 in icl_pdu_data_segment_receive_len()
454 if ((len % 4) != 0) in icl_pdu_data_segment_receive_len()
455 len += 4 - (len % 4); in icl_pdu_data_segment_receive_len()
457 #if 0 in icl_pdu_data_segment_receive_len()
469 size_t len, padding = 0; in icl_pdu_receive_data_segment()
472 isc = (struct icl_soft_conn *)request->ip_conn; in icl_pdu_receive_data_segment()
475 isc->receive_len = 0; in icl_pdu_receive_data_segment()
478 if (len == 0) in icl_pdu_receive_data_segment()
479 return (0); in icl_pdu_receive_data_segment()
481 if ((len % 4) != 0) in icl_pdu_receive_data_segment()
482 padding = 4 - (len % 4); in icl_pdu_receive_data_segment()
487 KASSERT(len > request->ip_data_len, ("len <= request->ip_data_len")); in icl_pdu_receive_data_segment()
488 len -= request->ip_data_len; in icl_pdu_receive_data_segment()
496 #if 0 in icl_pdu_receive_data_segment()
498 len + padding, *rs - padding)); in icl_pdu_receive_data_segment()
500 len = *rs - padding; in icl_pdu_receive_data_segment()
502 padding = 0; in icl_pdu_receive_data_segment()
509 if (len > 0) { in icl_pdu_receive_data_segment()
512 *rs -= len + padding; in icl_pdu_receive_data_segment()
514 if (request->ip_data_mbuf == NULL) in icl_pdu_receive_data_segment()
515 request->ip_data_mbuf = m; in icl_pdu_receive_data_segment()
517 m_cat(request->ip_data_mbuf, m); in icl_pdu_receive_data_segment()
519 request->ip_data_len += len; in icl_pdu_receive_data_segment()
521 ICL_DEBUG("len 0"); in icl_pdu_receive_data_segment()
524 isc->receive_len = icl_pdu_data_segment_receive_len(request); in icl_pdu_receive_data_segment()
526 return (0); in icl_pdu_receive_data_segment()
534 if (request->ip_conn->ic_data_crc32c == false) in icl_pdu_check_data_digest()
535 return (0); in icl_pdu_check_data_digest()
537 if (request->ip_data_len == 0) in icl_pdu_check_data_digest()
538 return (0); in icl_pdu_check_data_digest()
548 valid_digest = icl_mbuf_to_crc32c(request->ip_data_mbuf, in icl_pdu_check_data_digest()
549 roundup2(request->ip_data_len, 4)); in icl_pdu_check_data_digest()
551 ICL_WARN("data digest check failed; got 0x%x, " in icl_pdu_check_data_digest()
552 "should be 0x%x", received_digest, valid_digest); in icl_pdu_check_data_digest()
553 return (-1); in icl_pdu_check_data_digest()
556 return (0); in icl_pdu_check_data_digest()
561 * "part" of PDU at a time; call it repeatedly until it returns non-NULL.
566 struct icl_conn *ic = &isc->ic; in icl_conn_receive_pdu()
569 int error = 0; in icl_conn_receive_pdu()
572 if (isc->receive_state == ICL_CONN_STATE_BHS) { in icl_conn_receive_pdu()
573 KASSERT(isc->receive_pdu == NULL, in icl_conn_receive_pdu()
574 ("isc->receive_pdu != NULL")); in icl_conn_receive_pdu()
582 isc->receive_pdu = request; in icl_conn_receive_pdu()
584 KASSERT(isc->receive_pdu != NULL, in icl_conn_receive_pdu()
585 ("isc->receive_pdu == NULL")); in icl_conn_receive_pdu()
586 request = isc->receive_pdu; in icl_conn_receive_pdu()
589 switch (isc->receive_state) { in icl_conn_receive_pdu()
592 icl_soft_receive_buf(r, rs, request->ip_bhs, in icl_conn_receive_pdu()
601 if (len > ic->ic_max_recv_data_segment_length) { in icl_conn_receive_pdu()
609 isc->receive_state = ICL_CONN_STATE_AHS; in icl_conn_receive_pdu()
610 isc->receive_len = icl_pdu_ahs_length(request); in icl_conn_receive_pdu()
616 isc->receive_state = ICL_CONN_STATE_HEADER_DIGEST; in icl_conn_receive_pdu()
617 if (ic->ic_header_crc32c == false) in icl_conn_receive_pdu()
618 isc->receive_len = 0; in icl_conn_receive_pdu()
620 isc->receive_len = ISCSI_HEADER_DIGEST_SIZE; in icl_conn_receive_pdu()
626 if (error != 0) { in icl_conn_receive_pdu()
632 isc->receive_state = ICL_CONN_STATE_DATA; in icl_conn_receive_pdu()
633 isc->receive_len = icl_pdu_data_segment_receive_len(request); in icl_conn_receive_pdu()
640 if (error != 0) { in icl_conn_receive_pdu()
649 isc->receive_state = ICL_CONN_STATE_DATA_DIGEST; in icl_conn_receive_pdu()
650 if (request->ip_data_len == 0 || ic->ic_data_crc32c == false) in icl_conn_receive_pdu()
651 isc->receive_len = 0; in icl_conn_receive_pdu()
653 isc->receive_len = ISCSI_DATA_DIGEST_SIZE; in icl_conn_receive_pdu()
659 if (error != 0) { in icl_conn_receive_pdu()
669 isc->receive_state = ICL_CONN_STATE_BHS; in icl_conn_receive_pdu()
670 isc->receive_len = sizeof(struct iscsi_bhs); in icl_conn_receive_pdu()
671 isc->receive_pdu = NULL; in icl_conn_receive_pdu()
675 panic("invalid receive_state %d\n", isc->receive_state); in icl_conn_receive_pdu()
678 if (error != 0) { in icl_conn_receive_pdu()
680 * Don't free the PDU; it's pointed to by isc->receive_pdu in icl_conn_receive_pdu()
692 struct icl_conn *ic = &isc->ic; in icl_conn_receive_pdus()
696 if (ic->ic_disconnecting) in icl_conn_receive_pdus()
703 if (*rs < isc->receive_len) { in icl_conn_receive_pdus()
704 #if 0 in icl_conn_receive_pdus()
706 *rs, isc->receive_len); in icl_conn_receive_pdus()
715 if (response->ip_ahs_len > 0) { in icl_conn_receive_pdus()
717 "AHS; opcode 0x%x; dropping connection", in icl_conn_receive_pdus()
718 response->ip_bhs->bhs_opcode); in icl_conn_receive_pdus()
724 (ic->ic_receive)(response); in icl_conn_receive_pdus()
732 struct icl_conn *ic = &isc->ic; in icl_receive_thread()
733 size_t available, read = 0; in icl_receive_thread()
739 so = ic->ic_socket; in icl_receive_thread()
742 SOCKBUF_LOCK(&so->so_rcv); in icl_receive_thread()
743 if (ic->ic_disconnecting) { in icl_receive_thread()
744 SOCKBUF_UNLOCK(&so->so_rcv); in icl_receive_thread()
754 available = sbavail(&so->so_rcv); in icl_receive_thread()
755 if (read + available < isc->receive_len) { in icl_receive_thread()
756 so->so_rcv.sb_lowat = isc->receive_len - read; in icl_receive_thread()
757 cv_wait(&isc->receive_cv, SOCKBUF_MTX(&so->so_rcv)); in icl_receive_thread()
758 so->so_rcv.sb_lowat = so->so_rcv.sb_hiwat + 1; in icl_receive_thread()
759 available = sbavail(&so->so_rcv); in icl_receive_thread()
761 SOCKBUF_UNLOCK(&so->so_rcv); in icl_receive_thread()
763 if (available == 0) { in icl_receive_thread()
764 if (so->so_error != 0) { in icl_receive_thread()
766 "dropping connection", so->so_error); in icl_receive_thread()
773 memset(&uio, 0, sizeof(uio)); in icl_receive_thread()
777 if (error != 0) { in icl_receive_thread()
781 if (uio.uio_resid != 0) { in icl_receive_thread()
799 isc->receive_running = false; in icl_receive_thread()
800 cv_signal(&isc->send_cv); in icl_receive_thread()
814 cv_signal(&isc->receive_cv); in icl_soupcall_receive()
822 uint32_t digest, zero = 0; in icl_pdu_finalize()
826 ic = request->ip_conn; in icl_pdu_finalize()
828 icl_pdu_set_data_segment_length(request, request->ip_data_len); in icl_pdu_finalize()
832 if (ic->ic_header_crc32c) { in icl_pdu_finalize()
833 digest = icl_mbuf_to_crc32c(request->ip_bhs_mbuf, in icl_pdu_finalize()
835 ok = m_append(request->ip_bhs_mbuf, sizeof(digest), in icl_pdu_finalize()
843 if (request->ip_data_len != 0) { in icl_pdu_finalize()
845 if (padding > 0) { in icl_pdu_finalize()
846 ok = m_append(request->ip_data_mbuf, padding, in icl_pdu_finalize()
854 if (ic->ic_data_crc32c) { in icl_pdu_finalize()
855 digest = icl_mbuf_to_crc32c(request->ip_data_mbuf, in icl_pdu_finalize()
856 roundup2(request->ip_data_len, 4)); in icl_pdu_finalize()
858 ok = m_append(request->ip_data_mbuf, sizeof(digest), in icl_pdu_finalize()
866 m_cat(request->ip_bhs_mbuf, request->ip_data_mbuf); in icl_pdu_finalize()
867 request->ip_data_mbuf = NULL; in icl_pdu_finalize()
870 request->ip_bhs_mbuf->m_pkthdr.len = pdu_len; in icl_pdu_finalize()
872 return (0); in icl_pdu_finalize()
878 struct icl_conn *ic = &isc->ic; in icl_conn_send_pdus()
890 so = ic->ic_socket; in icl_conn_send_pdus()
892 SOCKBUF_LOCK(&so->so_snd); in icl_conn_send_pdus()
899 available = sbspace(&so->so_snd); in icl_conn_send_pdus()
900 isc->check_send_space = false; in icl_conn_send_pdus()
906 so->so_snd.sb_lowat = so->so_snd.sb_hiwat + 1; in icl_conn_send_pdus()
907 SOCKBUF_UNLOCK(&so->so_snd); in icl_conn_send_pdus()
919 SOCKBUF_LOCK(&so->so_snd); in icl_conn_send_pdus()
920 available = sbspace(&so->so_snd); in icl_conn_send_pdus()
927 so->so_snd.sb_lowat = max(size, in icl_conn_send_pdus()
928 so->so_snd.sb_hiwat / 8); in icl_conn_send_pdus()
929 SOCKBUF_UNLOCK(&so->so_snd); in icl_conn_send_pdus()
932 SOCKBUF_UNLOCK(&so->so_snd); in icl_conn_send_pdus()
936 if (error != 0) { in icl_conn_send_pdus()
944 m = request->ip_bhs_mbuf; in icl_conn_send_pdus()
962 if (error != 0) { in icl_conn_send_pdus()
970 while (m->m_next) in icl_conn_send_pdus()
971 m = m->m_next; in icl_conn_send_pdus()
972 m_cat(m, request2->ip_bhs_mbuf); in icl_conn_send_pdus()
973 request2->ip_bhs_mbuf = NULL; in icl_conn_send_pdus()
974 request->ip_bhs_mbuf->m_pkthdr.len += size2; in icl_conn_send_pdus()
976 icl_soft_pdu_done(request2, 0); in icl_conn_send_pdus()
985 available -= size; in icl_conn_send_pdus()
986 error = sosend(so, NULL, NULL, request->ip_bhs_mbuf, in icl_conn_send_pdus()
988 request->ip_bhs_mbuf = NULL; /* Sosend consumes the mbuf. */ in icl_conn_send_pdus()
989 if (error != 0) { in icl_conn_send_pdus()
996 icl_soft_pdu_done(request, 0); in icl_conn_send_pdus()
1008 ic = &isc->ic; in icl_send_thread()
1020 if (STAILQ_EMPTY(&queue) || isc->check_send_space) in icl_send_thread()
1021 STAILQ_CONCAT(&queue, &isc->to_send); in icl_send_thread()
1031 if (isc->check_send_space) in icl_send_thread()
1039 !STAILQ_EMPTY(&isc->to_send)) in icl_send_thread()
1050 if (ic->ic_disconnecting) { in icl_send_thread()
1055 cv_wait(&isc->send_cv, ic->ic_lock); in icl_send_thread()
1062 STAILQ_CONCAT(&isc->to_send, &queue); in icl_send_thread()
1064 isc->send_running = false; in icl_send_thread()
1065 cv_signal(&isc->send_cv); in icl_send_thread()
1080 ic = &isc->ic; in icl_soupcall_send()
1083 isc->check_send_space = true; in icl_soupcall_send()
1086 cv_signal(&isc->send_cv); in icl_soupcall_send()
1094 struct icl_soft_pdu *isp; in icl_soft_free_mext_pg() local
1104 isp = m->m_ext.ext_arg1; in icl_soft_free_mext_pg()
1105 if (atomic_fetchadd_int(&isp->ref_cnt, -1) == 1) in icl_soft_free_mext_pg()
1106 icl_soft_pdu_call_cb(&isp->ip); in icl_soft_free_mext_pg()
1113 struct icl_soft_pdu *isp = (struct icl_soft_pdu *)request; in icl_soft_conn_pdu_append_bio() local
1119 KASSERT(len > 0, ("len == 0")); in icl_soft_conn_pdu_append_bio()
1121 m_tail = request->ip_data_mbuf; in icl_soft_conn_pdu_append_bio()
1123 for (; m_tail->m_next != NULL; m_tail = m_tail->m_next) in icl_soft_conn_pdu_append_bio()
1126 MPASS(bp->bio_flags & BIO_UNMAPPED); in icl_soft_conn_pdu_append_bio()
1127 if (offset < PAGE_SIZE - bp->bio_ma_offset) { in icl_soft_conn_pdu_append_bio()
1128 page_offset = bp->bio_ma_offset + offset; in icl_soft_conn_pdu_append_bio()
1129 i = 0; in icl_soft_conn_pdu_append_bio()
1131 offset -= PAGE_SIZE - bp->bio_ma_offset; in icl_soft_conn_pdu_append_bio()
1133 offset -= PAGE_SIZE; in icl_soft_conn_pdu_append_bio()
1139 while (len > 0) { in icl_soft_conn_pdu_append_bio()
1142 icl_soft_free_mext_pg, 0); in icl_soft_conn_pdu_append_bio()
1145 atomic_add_int(&isp->ref_cnt, 1); in icl_soft_conn_pdu_append_bio()
1146 m->m_ext.ext_arg1 = isp; in icl_soft_conn_pdu_append_bio()
1147 m->m_epg_1st_off = page_offset; in icl_soft_conn_pdu_append_bio()
1150 todo = MIN(len, PAGE_SIZE - page_offset); in icl_soft_conn_pdu_append_bio()
1152 m->m_epg_pa[m->m_epg_npgs] = in icl_soft_conn_pdu_append_bio()
1153 VM_PAGE_TO_PHYS(bp->bio_ma[i]); in icl_soft_conn_pdu_append_bio()
1154 m->m_epg_npgs++; in icl_soft_conn_pdu_append_bio()
1155 m->m_epg_last_len = todo; in icl_soft_conn_pdu_append_bio()
1156 m->m_len += todo; in icl_soft_conn_pdu_append_bio()
1157 m->m_ext.ext_size += PAGE_SIZE; in icl_soft_conn_pdu_append_bio()
1160 if (m->m_epg_npgs == MBUF_PEXT_MAX_PGS) { in icl_soft_conn_pdu_append_bio()
1162 m_tail->m_next = m; in icl_soft_conn_pdu_append_bio()
1164 request->ip_data_mbuf = m; in icl_soft_conn_pdu_append_bio()
1166 request->ip_data_len += m->m_len; in icl_soft_conn_pdu_append_bio()
1170 page_offset = 0; in icl_soft_conn_pdu_append_bio()
1171 len -= todo; in icl_soft_conn_pdu_append_bio()
1177 m_tail->m_next = m; in icl_soft_conn_pdu_append_bio()
1179 request->ip_data_mbuf = m; in icl_soft_conn_pdu_append_bio()
1180 request->ip_data_len += m->m_len; in icl_soft_conn_pdu_append_bio()
1182 return (0); in icl_soft_conn_pdu_append_bio()
1185 m = m_getm2(NULL, len, flags, MT_DATA, 0); in icl_soft_conn_pdu_append_bio()
1189 if (request->ip_data_mbuf == NULL) { in icl_soft_conn_pdu_append_bio()
1190 request->ip_data_mbuf = m; in icl_soft_conn_pdu_append_bio()
1191 request->ip_data_len = len; in icl_soft_conn_pdu_append_bio()
1193 m_tail->m_next = m; in icl_soft_conn_pdu_append_bio()
1194 request->ip_data_len += len; in icl_soft_conn_pdu_append_bio()
1197 while (len > 0) { in icl_soft_conn_pdu_append_bio()
1198 todo = MIN(len, PAGE_SIZE - page_offset); in icl_soft_conn_pdu_append_bio()
1199 vaddr = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(bp->bio_ma[i])); in icl_soft_conn_pdu_append_bio()
1202 mtodo = min(todo, M_SIZE(m) - m->m_len); in icl_soft_conn_pdu_append_bio()
1203 memcpy(mtod(m, char *) + m->m_len, (char *)vaddr + in icl_soft_conn_pdu_append_bio()
1205 m->m_len += mtodo; in icl_soft_conn_pdu_append_bio()
1206 if (m->m_len == M_SIZE(m)) in icl_soft_conn_pdu_append_bio()
1207 m = m->m_next; in icl_soft_conn_pdu_append_bio()
1209 todo -= mtodo; in icl_soft_conn_pdu_append_bio()
1210 } while (todo > 0); in icl_soft_conn_pdu_append_bio()
1212 page_offset = 0; in icl_soft_conn_pdu_append_bio()
1213 len -= todo; in icl_soft_conn_pdu_append_bio()
1217 return (0); in icl_soft_conn_pdu_append_bio()
1224 struct icl_soft_pdu *isp = (struct icl_soft_pdu *)request; in icl_soft_conn_pdu_append_data() local
1226 size_t copylen, off = 0; in icl_soft_conn_pdu_append_data()
1228 KASSERT(len > 0, ("len == 0")); in icl_soft_conn_pdu_append_data()
1237 newmb->m_flags |= M_RDONLY; in icl_soft_conn_pdu_append_data()
1238 m_extaddref(newmb, __DECONST(char *, addr), len, &isp->ref_cnt, in icl_soft_conn_pdu_append_data()
1239 icl_soft_mbuf_done, isp, NULL); in icl_soft_conn_pdu_append_data()
1240 newmb->m_len = len; in icl_soft_conn_pdu_append_data()
1242 newmb = m_getm2(NULL, len, flags, MT_DATA, 0); in icl_soft_conn_pdu_append_data()
1248 for (mb = newmb; mb != NULL; mb = mb->m_next) { in icl_soft_conn_pdu_append_data()
1249 copylen = min(M_TRAILINGSPACE(mb), len - off); in icl_soft_conn_pdu_append_data()
1251 mb->m_len = copylen; in icl_soft_conn_pdu_append_data()
1257 if (request->ip_data_mbuf == NULL) { in icl_soft_conn_pdu_append_data()
1258 request->ip_data_mbuf = newmb; in icl_soft_conn_pdu_append_data()
1259 request->ip_data_len = len; in icl_soft_conn_pdu_append_data()
1261 m_cat(request->ip_data_mbuf, newmb); in icl_soft_conn_pdu_append_data()
1262 request->ip_data_len += len; in icl_soft_conn_pdu_append_data()
1265 return (0); in icl_soft_conn_pdu_append_data()
1276 MPASS(bp->bio_flags & BIO_UNMAPPED); in icl_soft_conn_pdu_get_bio()
1277 if (bio_off < PAGE_SIZE - bp->bio_ma_offset) { in icl_soft_conn_pdu_get_bio()
1278 page_offset = bp->bio_ma_offset + bio_off; in icl_soft_conn_pdu_get_bio()
1279 i = 0; in icl_soft_conn_pdu_get_bio()
1281 bio_off -= PAGE_SIZE - bp->bio_ma_offset; in icl_soft_conn_pdu_get_bio()
1283 bio_off -= PAGE_SIZE; in icl_soft_conn_pdu_get_bio()
1287 while (len > 0) { in icl_soft_conn_pdu_get_bio()
1288 todo = MIN(len, PAGE_SIZE - page_offset); in icl_soft_conn_pdu_get_bio()
1290 vaddr = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(bp->bio_ma[i])); in icl_soft_conn_pdu_get_bio()
1291 m_copydata(ip->ip_data_mbuf, pdu_off, todo, (char *)vaddr + in icl_soft_conn_pdu_get_bio()
1294 page_offset = 0; in icl_soft_conn_pdu_get_bio()
1296 len -= todo; in icl_soft_conn_pdu_get_bio()
1306 m_copydata(ip->ip_data_mbuf, off, len, addr); in icl_soft_conn_pdu_get_data()
1321 struct icl_soft_pdu *isp = (struct icl_soft_pdu *)ip; in icl_soft_conn_pdu_queue_cb() local
1324 isp->ref_cnt++; in icl_soft_conn_pdu_queue_cb()
1325 isp->cb = cb; in icl_soft_conn_pdu_queue_cb()
1327 if (ic->ic_disconnecting || ic->ic_socket == NULL) { in icl_soft_conn_pdu_queue_cb()
1333 if (!STAILQ_EMPTY(&isc->to_send)) { in icl_soft_conn_pdu_queue_cb()
1334 STAILQ_INSERT_TAIL(&isc->to_send, ip, ip_next); in icl_soft_conn_pdu_queue_cb()
1343 STAILQ_INSERT_TAIL(&isc->to_send, ip, ip_next); in icl_soft_conn_pdu_queue_cb()
1344 cv_signal(&isc->send_cv); in icl_soft_conn_pdu_queue_cb()
1358 STAILQ_INIT(&isc->to_send); in icl_soft_new_conn()
1359 cv_init(&isc->send_cv, "icl_tx"); in icl_soft_new_conn()
1360 cv_init(&isc->receive_cv, "icl_rx"); in icl_soft_new_conn()
1362 ic = &isc->ic; in icl_soft_new_conn()
1363 ic->ic_lock = lock; in icl_soft_new_conn()
1365 refcount_init(&ic->ic_outstanding_pdus, 0); in icl_soft_new_conn()
1367 ic->ic_name = name; in icl_soft_new_conn()
1368 ic->ic_offload = "None"; in icl_soft_new_conn()
1369 ic->ic_unmapped = PMAP_HAS_DMAP; in icl_soft_new_conn()
1380 KASSERT(ic->ic_outstanding_pdus == 0, in icl_soft_conn_free()
1382 ic->ic_outstanding_pdus)); in icl_soft_conn_free()
1384 cv_destroy(&isc->send_cv); in icl_soft_conn_free()
1385 cv_destroy(&isc->receive_cv); in icl_soft_conn_free()
1403 if (ic->ic_socket == NULL) { in icl_conn_start()
1408 isc->receive_state = ICL_CONN_STATE_BHS; in icl_conn_start()
1409 isc->receive_len = sizeof(struct iscsi_bhs); in icl_conn_start()
1410 ic->ic_disconnecting = false; in icl_conn_start()
1420 ic->ic_max_send_data_segment_length + in icl_conn_start()
1428 ic->ic_max_recv_data_segment_length + in icl_conn_start()
1436 error = soreserve(ic->ic_socket, sendspace, recvspace); in icl_conn_start()
1437 if (error != 0) { in icl_conn_start()
1442 ic->ic_socket->so_snd.sb_flags |= SB_AUTOSIZE; in icl_conn_start()
1443 ic->ic_socket->so_rcv.sb_flags |= SB_AUTOSIZE; in icl_conn_start()
1454 error = sosetopt(ic->ic_socket, &opt); in icl_conn_start()
1455 if (error != 0) { in icl_conn_start()
1465 SOCKBUF_LOCK(&ic->ic_socket->so_snd); in icl_conn_start()
1466 soupcall_set(ic->ic_socket, SO_SND, icl_soupcall_send, isc); in icl_conn_start()
1467 SOCKBUF_UNLOCK(&ic->ic_socket->so_snd); in icl_conn_start()
1468 SOCKBUF_LOCK(&ic->ic_socket->so_rcv); in icl_conn_start()
1469 soupcall_set(ic->ic_socket, SO_RCV, icl_soupcall_receive, isc); in icl_conn_start()
1470 SOCKBUF_UNLOCK(&ic->ic_socket->so_rcv); in icl_conn_start()
1476 isc->send_running = isc->receive_running = true; in icl_conn_start()
1478 error = kthread_add(icl_send_thread, ic, NULL, NULL, 0, 0, "%stx", in icl_conn_start()
1479 ic->ic_name); in icl_conn_start()
1480 if (error != 0) { in icl_conn_start()
1483 isc->send_running = isc->receive_running = false; in icl_conn_start()
1484 cv_signal(&isc->send_cv); in icl_conn_start()
1489 error = kthread_add(icl_receive_thread, ic, NULL, NULL, 0, 0, "%srx", in icl_conn_start()
1490 ic->ic_name); in icl_conn_start()
1491 if (error != 0) { in icl_conn_start()
1494 isc->receive_running = false; in icl_conn_start()
1495 cv_signal(&isc->send_cv); in icl_conn_start()
1501 return (0); in icl_conn_start()
1519 if (fd == 0) { in icl_soft_conn_handoff()
1521 if (ic->ic_socket == NULL) { in icl_soft_conn_handoff()
1527 return (0); in icl_soft_conn_handoff()
1536 if (error != 0) in icl_soft_conn_handoff()
1538 if (fp->f_type != DTYPE_SOCKET) { in icl_soft_conn_handoff()
1542 so = fp->f_data; in icl_soft_conn_handoff()
1543 if (so->so_type != SOCK_STREAM) { in icl_soft_conn_handoff()
1550 if (ic->ic_socket != NULL) { in icl_soft_conn_handoff()
1556 ic->ic_socket = fp->f_data; in icl_soft_conn_handoff()
1557 fp->f_ops = &badfileops; in icl_soft_conn_handoff()
1558 fp->f_data = NULL; in icl_soft_conn_handoff()
1576 * Receive thread sleeps on so->so_rcv lock, send on ic->ic_lock. in icl_soft_conn_close()
1579 if (!ic->ic_disconnecting) { in icl_soft_conn_close()
1580 so = ic->ic_socket; in icl_soft_conn_close()
1582 SOCKBUF_LOCK(&so->so_rcv); in icl_soft_conn_close()
1583 ic->ic_disconnecting = true; in icl_soft_conn_close()
1585 SOCKBUF_UNLOCK(&so->so_rcv); in icl_soft_conn_close()
1587 while (isc->receive_running || isc->send_running) { in icl_soft_conn_close()
1588 cv_signal(&isc->receive_cv); in icl_soft_conn_close()
1589 cv_signal(&isc->send_cv); in icl_soft_conn_close()
1590 cv_wait(&isc->send_cv, ic->ic_lock); in icl_soft_conn_close()
1594 so = ic->ic_socket; in icl_soft_conn_close()
1599 ic->ic_socket = NULL; in icl_soft_conn_close()
1605 SOCKBUF_LOCK(&so->so_snd); in icl_soft_conn_close()
1606 if (so->so_snd.sb_upcall != NULL) in icl_soft_conn_close()
1608 SOCKBUF_UNLOCK(&so->so_snd); in icl_soft_conn_close()
1609 SOCKBUF_LOCK(&so->so_rcv); in icl_soft_conn_close()
1610 if (so->so_rcv.sb_upcall != NULL) in icl_soft_conn_close()
1612 SOCKBUF_UNLOCK(&so->so_rcv); in icl_soft_conn_close()
1616 if (isc->receive_pdu != NULL) { in icl_soft_conn_close()
1618 icl_soft_conn_pdu_free(ic, isc->receive_pdu); in icl_soft_conn_close()
1619 isc->receive_pdu = NULL; in icl_soft_conn_close()
1625 while (!STAILQ_EMPTY(&isc->to_send)) { in icl_soft_conn_close()
1626 pdu = STAILQ_FIRST(&isc->to_send); in icl_soft_conn_close()
1627 STAILQ_REMOVE_HEAD(&isc->to_send, ip_next); in icl_soft_conn_close()
1631 KASSERT(STAILQ_EMPTY(&isc->to_send), in icl_soft_conn_close()
1632 ("destroying session with non-empty send queue")); in icl_soft_conn_close()
1641 return (0); in icl_soft_conn_task_setup()
1654 return (0); in icl_soft_conn_transfer_setup()
1666 idl->idl_max_recv_data_segment_length = max_data_segment_length; in icl_soft_limits()
1667 idl->idl_max_send_data_segment_length = max_data_segment_length; in icl_soft_limits()
1668 idl->idl_max_burst_length = max_burst_length; in icl_soft_limits()
1669 idl->idl_first_burst_length = first_burst_length; in icl_soft_limits()
1671 return (0); in icl_soft_limits()
1691 if (so->so_type != SOCK_STREAM) in icl_soft_handoff_sock()
1695 if (ic->ic_socket != NULL) { in icl_soft_handoff_sock()
1699 ic->ic_socket = so; in icl_soft_handoff_sock()
1715 UMA_ALIGN_PTR, 0); in icl_soft_load()
1716 refcount_init(&icl_ncons, 0); in icl_soft_load()
1723 error = icl_register("none", false, 0, in icl_soft_load()
1725 KASSERT(error == 0, ("failed to register")); in icl_soft_load()
1727 #if defined(ICL_KERNEL_PROXY) && 0 in icl_soft_load()
1731 error = icl_register("proxytest", true, 0, in icl_soft_load()
1733 KASSERT(error == 0, ("failed to register")); in icl_soft_load()
1743 if (icl_ncons != 0) in icl_soft_unload()
1747 #if defined(ICL_KERNEL_PROXY) && 0 in icl_soft_unload()
1753 return (0); in icl_soft_unload()
1773 0