1 /*- 2 * Copyright (c) 2012 The FreeBSD Foundation 3 * All rights reserved. 4 * 5 * This software was developed by Edward Tomasz Napierala under sponsorship 6 * from the FreeBSD Foundation. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 */ 30 31 /* 32 * Software implementation of iSCSI Common Layer kobj(9) interface. 33 */ 34 35 #include <sys/cdefs.h> 36 __FBSDID("$FreeBSD$"); 37 38 #include <sys/param.h> 39 #include <sys/capsicum.h> 40 #include <sys/condvar.h> 41 #include <sys/conf.h> 42 #include <sys/file.h> 43 #include <sys/kernel.h> 44 #include <sys/kthread.h> 45 #include <sys/lock.h> 46 #include <sys/mbuf.h> 47 #include <sys/mutex.h> 48 #include <sys/module.h> 49 #include <sys/protosw.h> 50 #include <sys/socket.h> 51 #include <sys/socketvar.h> 52 #include <sys/sysctl.h> 53 #include <sys/systm.h> 54 #include <sys/sx.h> 55 #include <sys/uio.h> 56 #include <vm/uma.h> 57 #include <netinet/in.h> 58 #include <netinet/tcp.h> 59 60 #include <dev/iscsi/icl.h> 61 #include <dev/iscsi/iscsi_proto.h> 62 #include <icl_conn_if.h> 63 64 static int coalesce = 1; 65 SYSCTL_INT(_kern_icl, OID_AUTO, coalesce, CTLFLAG_RWTUN, 66 &coalesce, 0, "Try to coalesce PDUs before sending"); 67 static int partial_receive_len = 128 * 1024; 68 SYSCTL_INT(_kern_icl, OID_AUTO, partial_receive_len, CTLFLAG_RWTUN, 69 &partial_receive_len, 0, "Minimum read size for partially received " 70 "data segment"); 71 static int sendspace = 1048576; 72 SYSCTL_INT(_kern_icl, OID_AUTO, sendspace, CTLFLAG_RWTUN, 73 &sendspace, 0, "Default send socket buffer size"); 74 static int recvspace = 1048576; 75 SYSCTL_INT(_kern_icl, OID_AUTO, recvspace, CTLFLAG_RWTUN, 76 &recvspace, 0, "Default receive socket buffer size"); 77 78 static MALLOC_DEFINE(M_ICL_SOFT, "icl_soft", "iSCSI software backend"); 79 static uma_zone_t icl_pdu_zone; 80 81 static volatile u_int icl_ncons; 82 83 #define ICL_CONN_LOCK(X) mtx_lock(X->ic_lock) 84 #define ICL_CONN_UNLOCK(X) mtx_unlock(X->ic_lock) 85 #define ICL_CONN_LOCK_ASSERT(X) mtx_assert(X->ic_lock, MA_OWNED) 86 #define ICL_CONN_LOCK_ASSERT_NOT(X) mtx_assert(X->ic_lock, MA_NOTOWNED) 87 88 STAILQ_HEAD(icl_pdu_stailq, icl_pdu); 89 90 static icl_conn_new_pdu_t icl_soft_conn_new_pdu; 91 static icl_conn_pdu_free_t icl_soft_conn_pdu_free; 92 static icl_conn_pdu_data_segment_length_t 93 icl_soft_conn_pdu_data_segment_length; 94 static icl_conn_pdu_append_data_t icl_soft_conn_pdu_append_data; 95 static icl_conn_pdu_get_data_t icl_soft_conn_pdu_get_data; 96 static icl_conn_pdu_queue_t icl_soft_conn_pdu_queue; 97 static icl_conn_handoff_t icl_soft_conn_handoff; 98 static icl_conn_free_t icl_soft_conn_free; 99 static icl_conn_close_t icl_soft_conn_close; 100 static icl_conn_task_setup_t icl_soft_conn_task_setup; 101 static icl_conn_task_done_t icl_soft_conn_task_done; 102 static icl_conn_transfer_setup_t icl_soft_conn_transfer_setup; 103 static icl_conn_transfer_done_t icl_soft_conn_transfer_done; 104 #ifdef ICL_KERNEL_PROXY 105 static icl_conn_connect_t icl_soft_conn_connect; 106 #endif 107 108 static kobj_method_t icl_soft_methods[] = { 109 KOBJMETHOD(icl_conn_new_pdu, icl_soft_conn_new_pdu), 110 KOBJMETHOD(icl_conn_pdu_free, icl_soft_conn_pdu_free), 111 KOBJMETHOD(icl_conn_pdu_data_segment_length, 112 icl_soft_conn_pdu_data_segment_length), 113 KOBJMETHOD(icl_conn_pdu_append_data, icl_soft_conn_pdu_append_data), 114 KOBJMETHOD(icl_conn_pdu_get_data, icl_soft_conn_pdu_get_data), 115 KOBJMETHOD(icl_conn_pdu_queue, icl_soft_conn_pdu_queue), 116 KOBJMETHOD(icl_conn_handoff, icl_soft_conn_handoff), 117 KOBJMETHOD(icl_conn_free, icl_soft_conn_free), 118 KOBJMETHOD(icl_conn_close, icl_soft_conn_close), 119 KOBJMETHOD(icl_conn_task_setup, icl_soft_conn_task_setup), 120 KOBJMETHOD(icl_conn_task_done, icl_soft_conn_task_done), 121 KOBJMETHOD(icl_conn_transfer_setup, icl_soft_conn_transfer_setup), 122 KOBJMETHOD(icl_conn_transfer_done, icl_soft_conn_transfer_done), 123 #ifdef ICL_KERNEL_PROXY 124 KOBJMETHOD(icl_conn_connect, icl_soft_conn_connect), 125 #endif 126 { 0, 0 } 127 }; 128 129 DEFINE_CLASS(icl_soft, icl_soft_methods, sizeof(struct icl_conn)); 130 131 static void 132 icl_conn_fail(struct icl_conn *ic) 133 { 134 if (ic->ic_socket == NULL) 135 return; 136 137 /* 138 * XXX 139 */ 140 ic->ic_socket->so_error = EDOOFUS; 141 (ic->ic_error)(ic); 142 } 143 144 static struct mbuf * 145 icl_conn_receive(struct icl_conn *ic, size_t len) 146 { 147 struct uio uio; 148 struct socket *so; 149 struct mbuf *m; 150 int error, flags; 151 152 so = ic->ic_socket; 153 154 memset(&uio, 0, sizeof(uio)); 155 uio.uio_resid = len; 156 157 flags = MSG_DONTWAIT; 158 error = soreceive(so, NULL, &uio, &m, NULL, &flags); 159 if (error != 0) { 160 ICL_DEBUG("soreceive error %d", error); 161 return (NULL); 162 } 163 if (uio.uio_resid != 0) { 164 m_freem(m); 165 ICL_DEBUG("short read"); 166 return (NULL); 167 } 168 169 return (m); 170 } 171 172 static int 173 icl_conn_receive_buf(struct icl_conn *ic, void *buf, size_t len) 174 { 175 struct iovec iov[1]; 176 struct uio uio; 177 struct socket *so; 178 int error, flags; 179 180 so = ic->ic_socket; 181 182 memset(&uio, 0, sizeof(uio)); 183 iov[0].iov_base = buf; 184 iov[0].iov_len = len; 185 uio.uio_iov = iov; 186 uio.uio_iovcnt = 1; 187 uio.uio_offset = 0; 188 uio.uio_resid = len; 189 uio.uio_segflg = UIO_SYSSPACE; 190 uio.uio_rw = UIO_READ; 191 192 flags = MSG_DONTWAIT; 193 error = soreceive(so, NULL, &uio, NULL, NULL, &flags); 194 if (error != 0) { 195 ICL_DEBUG("soreceive error %d", error); 196 return (-1); 197 } 198 if (uio.uio_resid != 0) { 199 ICL_DEBUG("short read"); 200 return (-1); 201 } 202 203 return (0); 204 } 205 206 static void 207 icl_soft_conn_pdu_free(struct icl_conn *ic, struct icl_pdu *ip) 208 { 209 210 m_freem(ip->ip_bhs_mbuf); 211 m_freem(ip->ip_ahs_mbuf); 212 m_freem(ip->ip_data_mbuf); 213 uma_zfree(icl_pdu_zone, ip); 214 #ifdef DIAGNOSTIC 215 refcount_release(&ic->ic_outstanding_pdus); 216 #endif 217 } 218 219 /* 220 * Allocate icl_pdu with empty BHS to fill up by the caller. 221 */ 222 struct icl_pdu * 223 icl_soft_conn_new_pdu(struct icl_conn *ic, int flags) 224 { 225 struct icl_pdu *ip; 226 227 #ifdef DIAGNOSTIC 228 refcount_acquire(&ic->ic_outstanding_pdus); 229 #endif 230 ip = uma_zalloc(icl_pdu_zone, flags | M_ZERO); 231 if (ip == NULL) { 232 ICL_WARN("failed to allocate %zd bytes", sizeof(*ip)); 233 #ifdef DIAGNOSTIC 234 refcount_release(&ic->ic_outstanding_pdus); 235 #endif 236 return (NULL); 237 } 238 ip->ip_conn = ic; 239 240 CTASSERT(sizeof(struct iscsi_bhs) <= MHLEN); 241 ip->ip_bhs_mbuf = m_gethdr(flags, MT_DATA); 242 if (ip->ip_bhs_mbuf == NULL) { 243 ICL_WARN("failed to allocate BHS mbuf"); 244 icl_soft_conn_pdu_free(ic, ip); 245 return (NULL); 246 } 247 ip->ip_bhs = mtod(ip->ip_bhs_mbuf, struct iscsi_bhs *); 248 memset(ip->ip_bhs, 0, sizeof(struct iscsi_bhs)); 249 ip->ip_bhs_mbuf->m_len = sizeof(struct iscsi_bhs); 250 251 return (ip); 252 } 253 254 static int 255 icl_pdu_ahs_length(const struct icl_pdu *request) 256 { 257 258 return (request->ip_bhs->bhs_total_ahs_len * 4); 259 } 260 261 static size_t 262 icl_pdu_data_segment_length(const struct icl_pdu *request) 263 { 264 uint32_t len = 0; 265 266 len += request->ip_bhs->bhs_data_segment_len[0]; 267 len <<= 8; 268 len += request->ip_bhs->bhs_data_segment_len[1]; 269 len <<= 8; 270 len += request->ip_bhs->bhs_data_segment_len[2]; 271 272 return (len); 273 } 274 275 size_t 276 icl_soft_conn_pdu_data_segment_length(struct icl_conn *ic, 277 const struct icl_pdu *request) 278 { 279 280 return (icl_pdu_data_segment_length(request)); 281 } 282 283 static void 284 icl_pdu_set_data_segment_length(struct icl_pdu *response, uint32_t len) 285 { 286 287 response->ip_bhs->bhs_data_segment_len[2] = len; 288 response->ip_bhs->bhs_data_segment_len[1] = len >> 8; 289 response->ip_bhs->bhs_data_segment_len[0] = len >> 16; 290 } 291 292 static size_t 293 icl_pdu_padding(const struct icl_pdu *ip) 294 { 295 296 if ((ip->ip_data_len % 4) != 0) 297 return (4 - (ip->ip_data_len % 4)); 298 299 return (0); 300 } 301 302 static size_t 303 icl_pdu_size(const struct icl_pdu *response) 304 { 305 size_t len; 306 307 KASSERT(response->ip_ahs_len == 0, ("responding with AHS")); 308 309 len = sizeof(struct iscsi_bhs) + response->ip_data_len + 310 icl_pdu_padding(response); 311 if (response->ip_conn->ic_header_crc32c) 312 len += ISCSI_HEADER_DIGEST_SIZE; 313 if (response->ip_data_len != 0 && response->ip_conn->ic_data_crc32c) 314 len += ISCSI_DATA_DIGEST_SIZE; 315 316 return (len); 317 } 318 319 static int 320 icl_pdu_receive_bhs(struct icl_pdu *request, size_t *availablep) 321 { 322 323 if (icl_conn_receive_buf(request->ip_conn, 324 request->ip_bhs, sizeof(struct iscsi_bhs))) { 325 ICL_DEBUG("failed to receive BHS"); 326 return (-1); 327 } 328 329 *availablep -= sizeof(struct iscsi_bhs); 330 return (0); 331 } 332 333 static int 334 icl_pdu_receive_ahs(struct icl_pdu *request, size_t *availablep) 335 { 336 337 request->ip_ahs_len = icl_pdu_ahs_length(request); 338 if (request->ip_ahs_len == 0) 339 return (0); 340 341 request->ip_ahs_mbuf = icl_conn_receive(request->ip_conn, 342 request->ip_ahs_len); 343 if (request->ip_ahs_mbuf == NULL) { 344 ICL_DEBUG("failed to receive AHS"); 345 return (-1); 346 } 347 348 *availablep -= request->ip_ahs_len; 349 return (0); 350 } 351 352 static uint32_t 353 icl_mbuf_to_crc32c(const struct mbuf *m0) 354 { 355 uint32_t digest = 0xffffffff; 356 const struct mbuf *m; 357 358 for (m = m0; m != NULL; m = m->m_next) 359 digest = calculate_crc32c(digest, 360 mtod(m, const void *), m->m_len); 361 362 digest = digest ^ 0xffffffff; 363 364 return (digest); 365 } 366 367 static int 368 icl_pdu_check_header_digest(struct icl_pdu *request, size_t *availablep) 369 { 370 uint32_t received_digest, valid_digest; 371 372 if (request->ip_conn->ic_header_crc32c == false) 373 return (0); 374 375 CTASSERT(sizeof(received_digest) == ISCSI_HEADER_DIGEST_SIZE); 376 if (icl_conn_receive_buf(request->ip_conn, 377 &received_digest, ISCSI_HEADER_DIGEST_SIZE)) { 378 ICL_DEBUG("failed to receive header digest"); 379 return (-1); 380 } 381 *availablep -= ISCSI_HEADER_DIGEST_SIZE; 382 383 /* Temporary attach AHS to BHS to calculate header digest. */ 384 request->ip_bhs_mbuf->m_next = request->ip_ahs_mbuf; 385 valid_digest = icl_mbuf_to_crc32c(request->ip_bhs_mbuf); 386 request->ip_bhs_mbuf->m_next = NULL; 387 if (received_digest != valid_digest) { 388 ICL_WARN("header digest check failed; got 0x%x, " 389 "should be 0x%x", received_digest, valid_digest); 390 return (-1); 391 } 392 393 return (0); 394 } 395 396 /* 397 * Return the number of bytes that should be waiting in the receive socket 398 * before icl_pdu_receive_data_segment() gets called. 399 */ 400 static size_t 401 icl_pdu_data_segment_receive_len(const struct icl_pdu *request) 402 { 403 size_t len; 404 405 len = icl_pdu_data_segment_length(request); 406 if (len == 0) 407 return (0); 408 409 /* 410 * Account for the parts of data segment already read from 411 * the socket buffer. 412 */ 413 KASSERT(len > request->ip_data_len, ("len <= request->ip_data_len")); 414 len -= request->ip_data_len; 415 416 /* 417 * Don't always wait for the full data segment to be delivered 418 * to the socket; this might badly affect performance due to 419 * TCP window scaling. 420 */ 421 if (len > partial_receive_len) { 422 #if 0 423 ICL_DEBUG("need %zd bytes of data, limiting to %zd", 424 len, partial_receive_len)); 425 #endif 426 len = partial_receive_len; 427 428 return (len); 429 } 430 431 /* 432 * Account for padding. Note that due to the way code is written, 433 * the icl_pdu_receive_data_segment() must always receive padding 434 * along with the last part of data segment, because it would be 435 * impossible to tell whether we've already received the full data 436 * segment including padding, or without it. 437 */ 438 if ((len % 4) != 0) 439 len += 4 - (len % 4); 440 441 #if 0 442 ICL_DEBUG("need %zd bytes of data", len)); 443 #endif 444 445 return (len); 446 } 447 448 static int 449 icl_pdu_receive_data_segment(struct icl_pdu *request, 450 size_t *availablep, bool *more_neededp) 451 { 452 struct icl_conn *ic; 453 size_t len, padding = 0; 454 struct mbuf *m; 455 456 ic = request->ip_conn; 457 458 *more_neededp = false; 459 ic->ic_receive_len = 0; 460 461 len = icl_pdu_data_segment_length(request); 462 if (len == 0) 463 return (0); 464 465 if ((len % 4) != 0) 466 padding = 4 - (len % 4); 467 468 /* 469 * Account for already received parts of data segment. 470 */ 471 KASSERT(len > request->ip_data_len, ("len <= request->ip_data_len")); 472 len -= request->ip_data_len; 473 474 if (len + padding > *availablep) { 475 /* 476 * Not enough data in the socket buffer. Receive as much 477 * as we can. Don't receive padding, since, obviously, it's 478 * not the end of data segment yet. 479 */ 480 #if 0 481 ICL_DEBUG("limited from %zd to %zd", 482 len + padding, *availablep - padding)); 483 #endif 484 len = *availablep - padding; 485 *more_neededp = true; 486 padding = 0; 487 } 488 489 /* 490 * Must not try to receive padding without at least one byte 491 * of actual data segment. 492 */ 493 if (len > 0) { 494 m = icl_conn_receive(request->ip_conn, len + padding); 495 if (m == NULL) { 496 ICL_DEBUG("failed to receive data segment"); 497 return (-1); 498 } 499 500 if (request->ip_data_mbuf == NULL) 501 request->ip_data_mbuf = m; 502 else 503 m_cat(request->ip_data_mbuf, m); 504 505 request->ip_data_len += len; 506 *availablep -= len + padding; 507 } else 508 ICL_DEBUG("len 0"); 509 510 if (*more_neededp) 511 ic->ic_receive_len = 512 icl_pdu_data_segment_receive_len(request); 513 514 return (0); 515 } 516 517 static int 518 icl_pdu_check_data_digest(struct icl_pdu *request, size_t *availablep) 519 { 520 uint32_t received_digest, valid_digest; 521 522 if (request->ip_conn->ic_data_crc32c == false) 523 return (0); 524 525 if (request->ip_data_len == 0) 526 return (0); 527 528 CTASSERT(sizeof(received_digest) == ISCSI_DATA_DIGEST_SIZE); 529 if (icl_conn_receive_buf(request->ip_conn, 530 &received_digest, ISCSI_DATA_DIGEST_SIZE)) { 531 ICL_DEBUG("failed to receive data digest"); 532 return (-1); 533 } 534 *availablep -= ISCSI_DATA_DIGEST_SIZE; 535 536 /* 537 * Note that ip_data_mbuf also contains padding; since digest 538 * calculation is supposed to include that, we iterate over 539 * the entire ip_data_mbuf chain, not just ip_data_len bytes of it. 540 */ 541 valid_digest = icl_mbuf_to_crc32c(request->ip_data_mbuf); 542 if (received_digest != valid_digest) { 543 ICL_WARN("data digest check failed; got 0x%x, " 544 "should be 0x%x", received_digest, valid_digest); 545 return (-1); 546 } 547 548 return (0); 549 } 550 551 /* 552 * Somewhat contrary to the name, this attempts to receive only one 553 * "part" of PDU at a time; call it repeatedly until it returns non-NULL. 554 */ 555 static struct icl_pdu * 556 icl_conn_receive_pdu(struct icl_conn *ic, size_t *availablep) 557 { 558 struct icl_pdu *request; 559 struct socket *so; 560 size_t len; 561 int error; 562 bool more_needed; 563 564 so = ic->ic_socket; 565 566 if (ic->ic_receive_state == ICL_CONN_STATE_BHS) { 567 KASSERT(ic->ic_receive_pdu == NULL, 568 ("ic->ic_receive_pdu != NULL")); 569 request = icl_soft_conn_new_pdu(ic, M_NOWAIT); 570 if (request == NULL) { 571 ICL_DEBUG("failed to allocate PDU; " 572 "dropping connection"); 573 icl_conn_fail(ic); 574 return (NULL); 575 } 576 ic->ic_receive_pdu = request; 577 } else { 578 KASSERT(ic->ic_receive_pdu != NULL, 579 ("ic->ic_receive_pdu == NULL")); 580 request = ic->ic_receive_pdu; 581 } 582 583 if (*availablep < ic->ic_receive_len) { 584 #if 0 585 ICL_DEBUG("not enough data; need %zd, " 586 "have %zd", ic->ic_receive_len, *availablep); 587 #endif 588 return (NULL); 589 } 590 591 switch (ic->ic_receive_state) { 592 case ICL_CONN_STATE_BHS: 593 //ICL_DEBUG("receiving BHS"); 594 error = icl_pdu_receive_bhs(request, availablep); 595 if (error != 0) { 596 ICL_DEBUG("failed to receive BHS; " 597 "dropping connection"); 598 break; 599 } 600 601 /* 602 * We don't enforce any limit for AHS length; 603 * its length is stored in 8 bit field. 604 */ 605 606 len = icl_pdu_data_segment_length(request); 607 if (len > ic->ic_max_data_segment_length) { 608 ICL_WARN("received data segment " 609 "length %zd is larger than negotiated " 610 "MaxDataSegmentLength %zd; " 611 "dropping connection", 612 len, ic->ic_max_data_segment_length); 613 error = EINVAL; 614 break; 615 } 616 617 ic->ic_receive_state = ICL_CONN_STATE_AHS; 618 ic->ic_receive_len = icl_pdu_ahs_length(request); 619 break; 620 621 case ICL_CONN_STATE_AHS: 622 //ICL_DEBUG("receiving AHS"); 623 error = icl_pdu_receive_ahs(request, availablep); 624 if (error != 0) { 625 ICL_DEBUG("failed to receive AHS; " 626 "dropping connection"); 627 break; 628 } 629 ic->ic_receive_state = ICL_CONN_STATE_HEADER_DIGEST; 630 if (ic->ic_header_crc32c == false) 631 ic->ic_receive_len = 0; 632 else 633 ic->ic_receive_len = ISCSI_HEADER_DIGEST_SIZE; 634 break; 635 636 case ICL_CONN_STATE_HEADER_DIGEST: 637 //ICL_DEBUG("receiving header digest"); 638 error = icl_pdu_check_header_digest(request, availablep); 639 if (error != 0) { 640 ICL_DEBUG("header digest failed; " 641 "dropping connection"); 642 break; 643 } 644 645 ic->ic_receive_state = ICL_CONN_STATE_DATA; 646 ic->ic_receive_len = 647 icl_pdu_data_segment_receive_len(request); 648 break; 649 650 case ICL_CONN_STATE_DATA: 651 //ICL_DEBUG("receiving data segment"); 652 error = icl_pdu_receive_data_segment(request, availablep, 653 &more_needed); 654 if (error != 0) { 655 ICL_DEBUG("failed to receive data segment;" 656 "dropping connection"); 657 break; 658 } 659 660 if (more_needed) 661 break; 662 663 ic->ic_receive_state = ICL_CONN_STATE_DATA_DIGEST; 664 if (request->ip_data_len == 0 || ic->ic_data_crc32c == false) 665 ic->ic_receive_len = 0; 666 else 667 ic->ic_receive_len = ISCSI_DATA_DIGEST_SIZE; 668 break; 669 670 case ICL_CONN_STATE_DATA_DIGEST: 671 //ICL_DEBUG("receiving data digest"); 672 error = icl_pdu_check_data_digest(request, availablep); 673 if (error != 0) { 674 ICL_DEBUG("data digest failed; " 675 "dropping connection"); 676 break; 677 } 678 679 /* 680 * We've received complete PDU; reset the receive state machine 681 * and return the PDU. 682 */ 683 ic->ic_receive_state = ICL_CONN_STATE_BHS; 684 ic->ic_receive_len = sizeof(struct iscsi_bhs); 685 ic->ic_receive_pdu = NULL; 686 return (request); 687 688 default: 689 panic("invalid ic_receive_state %d\n", ic->ic_receive_state); 690 } 691 692 if (error != 0) { 693 /* 694 * Don't free the PDU; it's pointed to by ic->ic_receive_pdu 695 * and will get freed in icl_soft_conn_close(). 696 */ 697 icl_conn_fail(ic); 698 } 699 700 return (NULL); 701 } 702 703 static void 704 icl_conn_receive_pdus(struct icl_conn *ic, size_t available) 705 { 706 struct icl_pdu *response; 707 struct socket *so; 708 709 so = ic->ic_socket; 710 711 /* 712 * This can never happen; we're careful to only mess with ic->ic_socket 713 * pointer when the send/receive threads are not running. 714 */ 715 KASSERT(so != NULL, ("NULL socket")); 716 717 for (;;) { 718 if (ic->ic_disconnecting) 719 return; 720 721 if (so->so_error != 0) { 722 ICL_DEBUG("connection error %d; " 723 "dropping connection", so->so_error); 724 icl_conn_fail(ic); 725 return; 726 } 727 728 /* 729 * Loop until we have a complete PDU or there is not enough 730 * data in the socket buffer. 731 */ 732 if (available < ic->ic_receive_len) { 733 #if 0 734 ICL_DEBUG("not enough data; have %zd, " 735 "need %zd", available, 736 ic->ic_receive_len); 737 #endif 738 return; 739 } 740 741 response = icl_conn_receive_pdu(ic, &available); 742 if (response == NULL) 743 continue; 744 745 if (response->ip_ahs_len > 0) { 746 ICL_WARN("received PDU with unsupported " 747 "AHS; opcode 0x%x; dropping connection", 748 response->ip_bhs->bhs_opcode); 749 icl_soft_conn_pdu_free(ic, response); 750 icl_conn_fail(ic); 751 return; 752 } 753 754 (ic->ic_receive)(response); 755 } 756 } 757 758 static void 759 icl_receive_thread(void *arg) 760 { 761 struct icl_conn *ic; 762 size_t available; 763 struct socket *so; 764 765 ic = arg; 766 so = ic->ic_socket; 767 768 for (;;) { 769 if (ic->ic_disconnecting) { 770 //ICL_DEBUG("terminating"); 771 break; 772 } 773 774 /* 775 * Set the low watermark, to be checked by 776 * soreadable() in icl_soupcall_receive() 777 * to avoid unnecessary wakeups until there 778 * is enough data received to read the PDU. 779 */ 780 SOCKBUF_LOCK(&so->so_rcv); 781 available = sbavail(&so->so_rcv); 782 if (available < ic->ic_receive_len) { 783 so->so_rcv.sb_lowat = ic->ic_receive_len; 784 cv_wait(&ic->ic_receive_cv, &so->so_rcv.sb_mtx); 785 } else 786 so->so_rcv.sb_lowat = so->so_rcv.sb_hiwat + 1; 787 SOCKBUF_UNLOCK(&so->so_rcv); 788 789 icl_conn_receive_pdus(ic, available); 790 } 791 792 ICL_CONN_LOCK(ic); 793 ic->ic_receive_running = false; 794 cv_signal(&ic->ic_send_cv); 795 ICL_CONN_UNLOCK(ic); 796 kthread_exit(); 797 } 798 799 static int 800 icl_soupcall_receive(struct socket *so, void *arg, int waitflag) 801 { 802 struct icl_conn *ic; 803 804 if (!soreadable(so)) 805 return (SU_OK); 806 807 ic = arg; 808 cv_signal(&ic->ic_receive_cv); 809 return (SU_OK); 810 } 811 812 static int 813 icl_pdu_finalize(struct icl_pdu *request) 814 { 815 size_t padding, pdu_len; 816 uint32_t digest, zero = 0; 817 int ok; 818 struct icl_conn *ic; 819 820 ic = request->ip_conn; 821 822 icl_pdu_set_data_segment_length(request, request->ip_data_len); 823 824 pdu_len = icl_pdu_size(request); 825 826 if (ic->ic_header_crc32c) { 827 digest = icl_mbuf_to_crc32c(request->ip_bhs_mbuf); 828 ok = m_append(request->ip_bhs_mbuf, sizeof(digest), 829 (void *)&digest); 830 if (ok != 1) { 831 ICL_WARN("failed to append header digest"); 832 return (1); 833 } 834 } 835 836 if (request->ip_data_len != 0) { 837 padding = icl_pdu_padding(request); 838 if (padding > 0) { 839 ok = m_append(request->ip_data_mbuf, padding, 840 (void *)&zero); 841 if (ok != 1) { 842 ICL_WARN("failed to append padding"); 843 return (1); 844 } 845 } 846 847 if (ic->ic_data_crc32c) { 848 digest = icl_mbuf_to_crc32c(request->ip_data_mbuf); 849 850 ok = m_append(request->ip_data_mbuf, sizeof(digest), 851 (void *)&digest); 852 if (ok != 1) { 853 ICL_WARN("failed to append data digest"); 854 return (1); 855 } 856 } 857 858 m_cat(request->ip_bhs_mbuf, request->ip_data_mbuf); 859 request->ip_data_mbuf = NULL; 860 } 861 862 request->ip_bhs_mbuf->m_pkthdr.len = pdu_len; 863 864 return (0); 865 } 866 867 static void 868 icl_conn_send_pdus(struct icl_conn *ic, struct icl_pdu_stailq *queue) 869 { 870 struct icl_pdu *request, *request2; 871 struct socket *so; 872 long available, size, size2; 873 int coalesced, error; 874 875 ICL_CONN_LOCK_ASSERT_NOT(ic); 876 877 so = ic->ic_socket; 878 879 SOCKBUF_LOCK(&so->so_snd); 880 /* 881 * Check how much space do we have for transmit. We can't just 882 * call sosend() and retry when we get EWOULDBLOCK or EMSGSIZE, 883 * as it always frees the mbuf chain passed to it, even in case 884 * of error. 885 */ 886 available = sbspace(&so->so_snd); 887 888 /* 889 * Notify the socket upcall that we don't need wakeups 890 * for the time being. 891 */ 892 so->so_snd.sb_lowat = so->so_snd.sb_hiwat + 1; 893 SOCKBUF_UNLOCK(&so->so_snd); 894 895 while (!STAILQ_EMPTY(queue)) { 896 request = STAILQ_FIRST(queue); 897 size = icl_pdu_size(request); 898 if (available < size) { 899 900 /* 901 * Set the low watermark, to be checked by 902 * sowriteable() in icl_soupcall_send() 903 * to avoid unnecessary wakeups until there 904 * is enough space for the PDU to fit. 905 */ 906 SOCKBUF_LOCK(&so->so_snd); 907 available = sbspace(&so->so_snd); 908 if (available < size) { 909 #if 1 910 ICL_DEBUG("no space to send; " 911 "have %ld, need %ld", 912 available, size); 913 #endif 914 so->so_snd.sb_lowat = size; 915 SOCKBUF_UNLOCK(&so->so_snd); 916 return; 917 } 918 SOCKBUF_UNLOCK(&so->so_snd); 919 } 920 STAILQ_REMOVE_HEAD(queue, ip_next); 921 error = icl_pdu_finalize(request); 922 if (error != 0) { 923 ICL_DEBUG("failed to finalize PDU; " 924 "dropping connection"); 925 icl_soft_conn_pdu_free(ic, request); 926 icl_conn_fail(ic); 927 return; 928 } 929 if (coalesce) { 930 coalesced = 1; 931 for (;;) { 932 request2 = STAILQ_FIRST(queue); 933 if (request2 == NULL) 934 break; 935 size2 = icl_pdu_size(request2); 936 if (available < size + size2) 937 break; 938 STAILQ_REMOVE_HEAD(queue, ip_next); 939 error = icl_pdu_finalize(request2); 940 if (error != 0) { 941 ICL_DEBUG("failed to finalize PDU; " 942 "dropping connection"); 943 icl_soft_conn_pdu_free(ic, request); 944 icl_soft_conn_pdu_free(ic, request2); 945 icl_conn_fail(ic); 946 return; 947 } 948 m_cat(request->ip_bhs_mbuf, request2->ip_bhs_mbuf); 949 request2->ip_bhs_mbuf = NULL; 950 request->ip_bhs_mbuf->m_pkthdr.len += size2; 951 size += size2; 952 STAILQ_REMOVE_AFTER(queue, request, ip_next); 953 icl_soft_conn_pdu_free(ic, request2); 954 coalesced++; 955 } 956 #if 0 957 if (coalesced > 1) { 958 ICL_DEBUG("coalesced %d PDUs into %ld bytes", 959 coalesced, size); 960 } 961 #endif 962 } 963 available -= size; 964 error = sosend(so, NULL, NULL, request->ip_bhs_mbuf, 965 NULL, MSG_DONTWAIT, curthread); 966 request->ip_bhs_mbuf = NULL; /* Sosend consumes the mbuf. */ 967 if (error != 0) { 968 ICL_DEBUG("failed to send PDU, error %d; " 969 "dropping connection", error); 970 icl_soft_conn_pdu_free(ic, request); 971 icl_conn_fail(ic); 972 return; 973 } 974 icl_soft_conn_pdu_free(ic, request); 975 } 976 } 977 978 static void 979 icl_send_thread(void *arg) 980 { 981 struct icl_conn *ic; 982 struct icl_pdu_stailq queue; 983 984 ic = arg; 985 986 STAILQ_INIT(&queue); 987 988 ICL_CONN_LOCK(ic); 989 for (;;) { 990 for (;;) { 991 /* 992 * If the local queue is empty, populate it from 993 * the main one. This way the icl_conn_send_pdus() 994 * can go through all the queued PDUs without holding 995 * any locks. 996 */ 997 if (STAILQ_EMPTY(&queue)) 998 STAILQ_SWAP(&ic->ic_to_send, &queue, icl_pdu); 999 1000 ic->ic_check_send_space = false; 1001 ICL_CONN_UNLOCK(ic); 1002 icl_conn_send_pdus(ic, &queue); 1003 ICL_CONN_LOCK(ic); 1004 1005 /* 1006 * The icl_soupcall_send() was called since the last 1007 * call to sbspace(); go around; 1008 */ 1009 if (ic->ic_check_send_space) 1010 continue; 1011 1012 /* 1013 * Local queue is empty, but we still have PDUs 1014 * in the main one; go around. 1015 */ 1016 if (STAILQ_EMPTY(&queue) && 1017 !STAILQ_EMPTY(&ic->ic_to_send)) 1018 continue; 1019 1020 /* 1021 * There might be some stuff in the local queue, 1022 * which didn't get sent due to not having enough send 1023 * space. Wait for socket upcall. 1024 */ 1025 break; 1026 } 1027 1028 if (ic->ic_disconnecting) { 1029 //ICL_DEBUG("terminating"); 1030 break; 1031 } 1032 1033 cv_wait(&ic->ic_send_cv, ic->ic_lock); 1034 } 1035 1036 /* 1037 * We're exiting; move PDUs back to the main queue, so they can 1038 * get freed properly. At this point ordering doesn't matter. 1039 */ 1040 STAILQ_CONCAT(&ic->ic_to_send, &queue); 1041 1042 ic->ic_send_running = false; 1043 cv_signal(&ic->ic_send_cv); 1044 ICL_CONN_UNLOCK(ic); 1045 kthread_exit(); 1046 } 1047 1048 static int 1049 icl_soupcall_send(struct socket *so, void *arg, int waitflag) 1050 { 1051 struct icl_conn *ic; 1052 1053 if (!sowriteable(so)) 1054 return (SU_OK); 1055 1056 ic = arg; 1057 1058 ICL_CONN_LOCK(ic); 1059 ic->ic_check_send_space = true; 1060 ICL_CONN_UNLOCK(ic); 1061 1062 cv_signal(&ic->ic_send_cv); 1063 1064 return (SU_OK); 1065 } 1066 1067 static int 1068 icl_soft_conn_pdu_append_data(struct icl_conn *ic, struct icl_pdu *request, 1069 const void *addr, size_t len, int flags) 1070 { 1071 struct mbuf *mb, *newmb; 1072 size_t copylen, off = 0; 1073 1074 KASSERT(len > 0, ("len == 0")); 1075 1076 newmb = m_getm2(NULL, len, flags, MT_DATA, 0); 1077 if (newmb == NULL) { 1078 ICL_WARN("failed to allocate mbuf for %zd bytes", len); 1079 return (ENOMEM); 1080 } 1081 1082 for (mb = newmb; mb != NULL; mb = mb->m_next) { 1083 copylen = min(M_TRAILINGSPACE(mb), len - off); 1084 memcpy(mtod(mb, char *), (const char *)addr + off, copylen); 1085 mb->m_len = copylen; 1086 off += copylen; 1087 } 1088 KASSERT(off == len, ("%s: off != len", __func__)); 1089 1090 if (request->ip_data_mbuf == NULL) { 1091 request->ip_data_mbuf = newmb; 1092 request->ip_data_len = len; 1093 } else { 1094 m_cat(request->ip_data_mbuf, newmb); 1095 request->ip_data_len += len; 1096 } 1097 1098 return (0); 1099 } 1100 1101 void 1102 icl_soft_conn_pdu_get_data(struct icl_conn *ic, struct icl_pdu *ip, 1103 size_t off, void *addr, size_t len) 1104 { 1105 1106 m_copydata(ip->ip_data_mbuf, off, len, addr); 1107 } 1108 1109 static void 1110 icl_pdu_queue(struct icl_pdu *ip) 1111 { 1112 struct icl_conn *ic; 1113 1114 ic = ip->ip_conn; 1115 1116 ICL_CONN_LOCK_ASSERT(ic); 1117 1118 if (ic->ic_disconnecting || ic->ic_socket == NULL) { 1119 ICL_DEBUG("icl_pdu_queue on closed connection"); 1120 icl_soft_conn_pdu_free(ic, ip); 1121 return; 1122 } 1123 1124 if (!STAILQ_EMPTY(&ic->ic_to_send)) { 1125 STAILQ_INSERT_TAIL(&ic->ic_to_send, ip, ip_next); 1126 /* 1127 * If the queue is not empty, someone else had already 1128 * signaled the send thread; no need to do that again, 1129 * just return. 1130 */ 1131 return; 1132 } 1133 1134 STAILQ_INSERT_TAIL(&ic->ic_to_send, ip, ip_next); 1135 cv_signal(&ic->ic_send_cv); 1136 } 1137 1138 void 1139 icl_soft_conn_pdu_queue(struct icl_conn *ic, struct icl_pdu *ip) 1140 { 1141 1142 icl_pdu_queue(ip); 1143 } 1144 1145 static struct icl_conn * 1146 icl_soft_new_conn(const char *name, struct mtx *lock) 1147 { 1148 struct icl_conn *ic; 1149 1150 refcount_acquire(&icl_ncons); 1151 1152 ic = (struct icl_conn *)kobj_create(&icl_soft_class, M_ICL_SOFT, M_WAITOK | M_ZERO); 1153 1154 STAILQ_INIT(&ic->ic_to_send); 1155 ic->ic_lock = lock; 1156 cv_init(&ic->ic_send_cv, "icl_tx"); 1157 cv_init(&ic->ic_receive_cv, "icl_rx"); 1158 #ifdef DIAGNOSTIC 1159 refcount_init(&ic->ic_outstanding_pdus, 0); 1160 #endif 1161 ic->ic_max_data_segment_length = ICL_MAX_DATA_SEGMENT_LENGTH; 1162 ic->ic_name = name; 1163 ic->ic_offload = "None"; 1164 ic->ic_unmapped = false; 1165 1166 return (ic); 1167 } 1168 1169 void 1170 icl_soft_conn_free(struct icl_conn *ic) 1171 { 1172 1173 cv_destroy(&ic->ic_send_cv); 1174 cv_destroy(&ic->ic_receive_cv); 1175 kobj_delete((struct kobj *)ic, M_ICL_SOFT); 1176 refcount_release(&icl_ncons); 1177 } 1178 1179 static int 1180 icl_conn_start(struct icl_conn *ic) 1181 { 1182 size_t minspace; 1183 struct sockopt opt; 1184 int error, one = 1; 1185 1186 ICL_CONN_LOCK(ic); 1187 1188 /* 1189 * XXX: Ugly hack. 1190 */ 1191 if (ic->ic_socket == NULL) { 1192 ICL_CONN_UNLOCK(ic); 1193 return (EINVAL); 1194 } 1195 1196 ic->ic_receive_state = ICL_CONN_STATE_BHS; 1197 ic->ic_receive_len = sizeof(struct iscsi_bhs); 1198 ic->ic_disconnecting = false; 1199 1200 ICL_CONN_UNLOCK(ic); 1201 1202 /* 1203 * For sendspace, this is required because the current code cannot 1204 * send a PDU in pieces; thus, the minimum buffer size is equal 1205 * to the maximum PDU size. "+4" is to account for possible padding. 1206 * 1207 * What we should actually do here is to use autoscaling, but set 1208 * some minimal buffer size to "minspace". I don't know a way to do 1209 * that, though. 1210 */ 1211 minspace = sizeof(struct iscsi_bhs) + ic->ic_max_data_segment_length + 1212 ISCSI_HEADER_DIGEST_SIZE + ISCSI_DATA_DIGEST_SIZE + 4; 1213 if (sendspace < minspace) { 1214 ICL_WARN("kern.icl.sendspace too low; must be at least %zd", 1215 minspace); 1216 sendspace = minspace; 1217 } 1218 if (recvspace < minspace) { 1219 ICL_WARN("kern.icl.recvspace too low; must be at least %zd", 1220 minspace); 1221 recvspace = minspace; 1222 } 1223 1224 error = soreserve(ic->ic_socket, sendspace, recvspace); 1225 if (error != 0) { 1226 ICL_WARN("soreserve failed with error %d", error); 1227 icl_soft_conn_close(ic); 1228 return (error); 1229 } 1230 ic->ic_socket->so_snd.sb_flags |= SB_AUTOSIZE; 1231 ic->ic_socket->so_rcv.sb_flags |= SB_AUTOSIZE; 1232 1233 /* 1234 * Disable Nagle. 1235 */ 1236 bzero(&opt, sizeof(opt)); 1237 opt.sopt_dir = SOPT_SET; 1238 opt.sopt_level = IPPROTO_TCP; 1239 opt.sopt_name = TCP_NODELAY; 1240 opt.sopt_val = &one; 1241 opt.sopt_valsize = sizeof(one); 1242 error = sosetopt(ic->ic_socket, &opt); 1243 if (error != 0) { 1244 ICL_WARN("disabling TCP_NODELAY failed with error %d", error); 1245 icl_soft_conn_close(ic); 1246 return (error); 1247 } 1248 1249 /* 1250 * Register socket upcall, to get notified about incoming PDUs 1251 * and free space to send outgoing ones. 1252 */ 1253 SOCKBUF_LOCK(&ic->ic_socket->so_snd); 1254 soupcall_set(ic->ic_socket, SO_SND, icl_soupcall_send, ic); 1255 SOCKBUF_UNLOCK(&ic->ic_socket->so_snd); 1256 SOCKBUF_LOCK(&ic->ic_socket->so_rcv); 1257 soupcall_set(ic->ic_socket, SO_RCV, icl_soupcall_receive, ic); 1258 SOCKBUF_UNLOCK(&ic->ic_socket->so_rcv); 1259 1260 /* 1261 * Start threads. 1262 */ 1263 ICL_CONN_LOCK(ic); 1264 ic->ic_send_running = ic->ic_receive_running = true; 1265 ICL_CONN_UNLOCK(ic); 1266 error = kthread_add(icl_send_thread, ic, NULL, NULL, 0, 0, "%stx", 1267 ic->ic_name); 1268 if (error != 0) { 1269 ICL_WARN("kthread_add(9) failed with error %d", error); 1270 ICL_CONN_LOCK(ic); 1271 ic->ic_send_running = ic->ic_receive_running = false; 1272 cv_signal(&ic->ic_send_cv); 1273 ICL_CONN_UNLOCK(ic); 1274 icl_soft_conn_close(ic); 1275 return (error); 1276 } 1277 error = kthread_add(icl_receive_thread, ic, NULL, NULL, 0, 0, "%srx", 1278 ic->ic_name); 1279 if (error != 0) { 1280 ICL_WARN("kthread_add(9) failed with error %d", error); 1281 ICL_CONN_LOCK(ic); 1282 ic->ic_receive_running = false; 1283 cv_signal(&ic->ic_send_cv); 1284 ICL_CONN_UNLOCK(ic); 1285 icl_soft_conn_close(ic); 1286 return (error); 1287 } 1288 1289 return (0); 1290 } 1291 1292 int 1293 icl_soft_conn_handoff(struct icl_conn *ic, int fd) 1294 { 1295 struct file *fp; 1296 struct socket *so; 1297 cap_rights_t rights; 1298 int error; 1299 1300 ICL_CONN_LOCK_ASSERT_NOT(ic); 1301 1302 #ifdef ICL_KERNEL_PROXY 1303 /* 1304 * We're transitioning to Full Feature phase, and we don't 1305 * really care. 1306 */ 1307 if (fd == 0) { 1308 ICL_CONN_LOCK(ic); 1309 if (ic->ic_socket == NULL) { 1310 ICL_CONN_UNLOCK(ic); 1311 ICL_WARN("proxy handoff without connect"); 1312 return (EINVAL); 1313 } 1314 ICL_CONN_UNLOCK(ic); 1315 return (0); 1316 } 1317 #endif 1318 1319 /* 1320 * Steal the socket from userland. 1321 */ 1322 error = fget(curthread, fd, 1323 cap_rights_init(&rights, CAP_SOCK_CLIENT), &fp); 1324 if (error != 0) 1325 return (error); 1326 if (fp->f_type != DTYPE_SOCKET) { 1327 fdrop(fp, curthread); 1328 return (EINVAL); 1329 } 1330 so = fp->f_data; 1331 if (so->so_type != SOCK_STREAM) { 1332 fdrop(fp, curthread); 1333 return (EINVAL); 1334 } 1335 1336 ICL_CONN_LOCK(ic); 1337 1338 if (ic->ic_socket != NULL) { 1339 ICL_CONN_UNLOCK(ic); 1340 fdrop(fp, curthread); 1341 return (EBUSY); 1342 } 1343 1344 ic->ic_socket = fp->f_data; 1345 fp->f_ops = &badfileops; 1346 fp->f_data = NULL; 1347 fdrop(fp, curthread); 1348 ICL_CONN_UNLOCK(ic); 1349 1350 error = icl_conn_start(ic); 1351 1352 return (error); 1353 } 1354 1355 void 1356 icl_soft_conn_close(struct icl_conn *ic) 1357 { 1358 struct icl_pdu *pdu; 1359 struct socket *so; 1360 1361 ICL_CONN_LOCK(ic); 1362 1363 /* 1364 * Wake up the threads, so they can properly terminate. 1365 */ 1366 ic->ic_disconnecting = true; 1367 while (ic->ic_receive_running || ic->ic_send_running) { 1368 cv_signal(&ic->ic_receive_cv); 1369 cv_signal(&ic->ic_send_cv); 1370 cv_wait(&ic->ic_send_cv, ic->ic_lock); 1371 } 1372 1373 /* Some other thread could close the connection same time. */ 1374 so = ic->ic_socket; 1375 if (so == NULL) { 1376 ICL_CONN_UNLOCK(ic); 1377 return; 1378 } 1379 ic->ic_socket = NULL; 1380 1381 /* 1382 * Deregister socket upcalls. 1383 */ 1384 ICL_CONN_UNLOCK(ic); 1385 SOCKBUF_LOCK(&so->so_snd); 1386 if (so->so_snd.sb_upcall != NULL) 1387 soupcall_clear(so, SO_SND); 1388 SOCKBUF_UNLOCK(&so->so_snd); 1389 SOCKBUF_LOCK(&so->so_rcv); 1390 if (so->so_rcv.sb_upcall != NULL) 1391 soupcall_clear(so, SO_RCV); 1392 SOCKBUF_UNLOCK(&so->so_rcv); 1393 soclose(so); 1394 ICL_CONN_LOCK(ic); 1395 1396 if (ic->ic_receive_pdu != NULL) { 1397 //ICL_DEBUG("freeing partially received PDU"); 1398 icl_soft_conn_pdu_free(ic, ic->ic_receive_pdu); 1399 ic->ic_receive_pdu = NULL; 1400 } 1401 1402 /* 1403 * Remove any outstanding PDUs from the send queue. 1404 */ 1405 while (!STAILQ_EMPTY(&ic->ic_to_send)) { 1406 pdu = STAILQ_FIRST(&ic->ic_to_send); 1407 STAILQ_REMOVE_HEAD(&ic->ic_to_send, ip_next); 1408 icl_soft_conn_pdu_free(ic, pdu); 1409 } 1410 1411 KASSERT(STAILQ_EMPTY(&ic->ic_to_send), 1412 ("destroying session with non-empty send queue")); 1413 #ifdef DIAGNOSTIC 1414 KASSERT(ic->ic_outstanding_pdus == 0, 1415 ("destroying session with %d outstanding PDUs", 1416 ic->ic_outstanding_pdus)); 1417 #endif 1418 ICL_CONN_UNLOCK(ic); 1419 } 1420 1421 int 1422 icl_soft_conn_task_setup(struct icl_conn *ic, struct icl_pdu *ip, 1423 struct ccb_scsiio *csio, uint32_t *task_tagp, void **prvp) 1424 { 1425 1426 return (0); 1427 } 1428 1429 void 1430 icl_soft_conn_task_done(struct icl_conn *ic, void *prv) 1431 { 1432 } 1433 1434 int 1435 icl_soft_conn_transfer_setup(struct icl_conn *ic, union ctl_io *io, 1436 uint32_t *transfer_tag, void **prvp) 1437 { 1438 1439 return (0); 1440 } 1441 1442 void 1443 icl_soft_conn_transfer_done(struct icl_conn *ic, void *prv) 1444 { 1445 } 1446 1447 static int 1448 icl_soft_limits(struct icl_drv_limits *idl) 1449 { 1450 1451 idl->idl_max_recv_data_segment_length = 128 * 1024; 1452 1453 return (0); 1454 } 1455 1456 #ifdef ICL_KERNEL_PROXY 1457 int 1458 icl_soft_conn_connect(struct icl_conn *ic, int domain, int socktype, 1459 int protocol, struct sockaddr *from_sa, struct sockaddr *to_sa) 1460 { 1461 1462 return (icl_soft_proxy_connect(ic, domain, socktype, protocol, 1463 from_sa, to_sa)); 1464 } 1465 1466 int 1467 icl_soft_handoff_sock(struct icl_conn *ic, struct socket *so) 1468 { 1469 int error; 1470 1471 ICL_CONN_LOCK_ASSERT_NOT(ic); 1472 1473 if (so->so_type != SOCK_STREAM) 1474 return (EINVAL); 1475 1476 ICL_CONN_LOCK(ic); 1477 if (ic->ic_socket != NULL) { 1478 ICL_CONN_UNLOCK(ic); 1479 return (EBUSY); 1480 } 1481 ic->ic_socket = so; 1482 ICL_CONN_UNLOCK(ic); 1483 1484 error = icl_conn_start(ic); 1485 1486 return (error); 1487 } 1488 #endif /* ICL_KERNEL_PROXY */ 1489 1490 static int 1491 icl_soft_load(void) 1492 { 1493 int error; 1494 1495 icl_pdu_zone = uma_zcreate("icl_pdu", 1496 sizeof(struct icl_pdu), NULL, NULL, NULL, NULL, 1497 UMA_ALIGN_PTR, 0); 1498 refcount_init(&icl_ncons, 0); 1499 1500 /* 1501 * The reason we call this "none" is that to the user, 1502 * it's known as "offload driver"; "offload driver: soft" 1503 * doesn't make much sense. 1504 */ 1505 error = icl_register("none", false, 0, 1506 icl_soft_limits, icl_soft_new_conn); 1507 KASSERT(error == 0, ("failed to register")); 1508 1509 #if defined(ICL_KERNEL_PROXY) && 0 1510 /* 1511 * Debugging aid for kernel proxy functionality. 1512 */ 1513 error = icl_register("proxytest", true, 0, 1514 icl_soft_limits, icl_soft_new_conn); 1515 KASSERT(error == 0, ("failed to register")); 1516 #endif 1517 1518 return (error); 1519 } 1520 1521 static int 1522 icl_soft_unload(void) 1523 { 1524 1525 if (icl_ncons != 0) 1526 return (EBUSY); 1527 1528 icl_unregister("none", false); 1529 #if defined(ICL_KERNEL_PROXY) && 0 1530 icl_unregister("proxytest", true); 1531 #endif 1532 1533 uma_zdestroy(icl_pdu_zone); 1534 1535 return (0); 1536 } 1537 1538 static int 1539 icl_soft_modevent(module_t mod, int what, void *arg) 1540 { 1541 1542 switch (what) { 1543 case MOD_LOAD: 1544 return (icl_soft_load()); 1545 case MOD_UNLOAD: 1546 return (icl_soft_unload()); 1547 default: 1548 return (EINVAL); 1549 } 1550 } 1551 1552 moduledata_t icl_soft_data = { 1553 "icl_soft", 1554 icl_soft_modevent, 1555 0 1556 }; 1557 1558 DECLARE_MODULE(icl_soft, icl_soft_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE); 1559 MODULE_DEPEND(icl_soft, icl, 1, 1, 1); 1560 MODULE_VERSION(icl_soft, 1); 1561