1 /*- 2 * Copyright (c) 2012 The FreeBSD Foundation 3 * All rights reserved. 4 * 5 * This software was developed by Edward Tomasz Napierala under sponsorship 6 * from the FreeBSD Foundation. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 */ 30 31 /* 32 * Software implementation of iSCSI Common Layer kobj(9) interface. 33 */ 34 35 #include <sys/cdefs.h> 36 __FBSDID("$FreeBSD$"); 37 38 #include <sys/param.h> 39 #include <sys/capsicum.h> 40 #include <sys/condvar.h> 41 #include <sys/conf.h> 42 #include <sys/file.h> 43 #include <sys/kernel.h> 44 #include <sys/kthread.h> 45 #include <sys/lock.h> 46 #include <sys/mbuf.h> 47 #include <sys/mutex.h> 48 #include <sys/module.h> 49 #include <sys/protosw.h> 50 #include <sys/socket.h> 51 #include <sys/socketvar.h> 52 #include <sys/sysctl.h> 53 #include <sys/systm.h> 54 #include <sys/sx.h> 55 #include <sys/uio.h> 56 #include <vm/uma.h> 57 #include <netinet/in.h> 58 #include <netinet/tcp.h> 59 60 #include <dev/iscsi/icl.h> 61 #include <dev/iscsi/iscsi_proto.h> 62 #include <icl_conn_if.h> 63 64 static int coalesce = 1; 65 SYSCTL_INT(_kern_icl, OID_AUTO, coalesce, CTLFLAG_RWTUN, 66 &coalesce, 0, "Try to coalesce PDUs before sending"); 67 static int partial_receive_len = 128 * 1024; 68 SYSCTL_INT(_kern_icl, OID_AUTO, partial_receive_len, CTLFLAG_RWTUN, 69 &partial_receive_len, 0, "Minimum read size for partially received " 70 "data segment"); 71 static int sendspace = 1048576; 72 SYSCTL_INT(_kern_icl, OID_AUTO, sendspace, CTLFLAG_RWTUN, 73 &sendspace, 0, "Default send socket buffer size"); 74 static int recvspace = 1048576; 75 SYSCTL_INT(_kern_icl, OID_AUTO, recvspace, CTLFLAG_RWTUN, 76 &recvspace, 0, "Default receive socket buffer size"); 77 78 static MALLOC_DEFINE(M_ICL_SOFT, "icl_soft", "iSCSI software backend"); 79 static uma_zone_t icl_pdu_zone; 80 81 static volatile u_int icl_ncons; 82 83 #define ICL_CONN_LOCK(X) mtx_lock(X->ic_lock) 84 #define ICL_CONN_UNLOCK(X) mtx_unlock(X->ic_lock) 85 #define ICL_CONN_LOCK_ASSERT(X) mtx_assert(X->ic_lock, MA_OWNED) 86 #define ICL_CONN_LOCK_ASSERT_NOT(X) mtx_assert(X->ic_lock, MA_NOTOWNED) 87 88 STAILQ_HEAD(icl_pdu_stailq, icl_pdu); 89 90 static icl_conn_new_pdu_t icl_soft_conn_new_pdu; 91 static icl_conn_pdu_free_t icl_soft_conn_pdu_free; 92 static icl_conn_pdu_data_segment_length_t 93 icl_soft_conn_pdu_data_segment_length; 94 static icl_conn_pdu_append_data_t icl_soft_conn_pdu_append_data; 95 static icl_conn_pdu_get_data_t icl_soft_conn_pdu_get_data; 96 static icl_conn_pdu_queue_t icl_soft_conn_pdu_queue; 97 static icl_conn_handoff_t icl_soft_conn_handoff; 98 static icl_conn_free_t icl_soft_conn_free; 99 static icl_conn_close_t icl_soft_conn_close; 100 static icl_conn_task_setup_t icl_soft_conn_task_setup; 101 static icl_conn_task_done_t icl_soft_conn_task_done; 102 static icl_conn_transfer_setup_t icl_soft_conn_transfer_setup; 103 static icl_conn_transfer_done_t icl_soft_conn_transfer_done; 104 105 static kobj_method_t icl_soft_methods[] = { 106 KOBJMETHOD(icl_conn_new_pdu, icl_soft_conn_new_pdu), 107 KOBJMETHOD(icl_conn_pdu_free, icl_soft_conn_pdu_free), 108 KOBJMETHOD(icl_conn_pdu_data_segment_length, 109 icl_soft_conn_pdu_data_segment_length), 110 KOBJMETHOD(icl_conn_pdu_append_data, icl_soft_conn_pdu_append_data), 111 KOBJMETHOD(icl_conn_pdu_get_data, icl_soft_conn_pdu_get_data), 112 KOBJMETHOD(icl_conn_pdu_queue, icl_soft_conn_pdu_queue), 113 KOBJMETHOD(icl_conn_handoff, icl_soft_conn_handoff), 114 KOBJMETHOD(icl_conn_free, icl_soft_conn_free), 115 KOBJMETHOD(icl_conn_close, icl_soft_conn_close), 116 KOBJMETHOD(icl_conn_task_setup, icl_soft_conn_task_setup), 117 KOBJMETHOD(icl_conn_task_done, icl_soft_conn_task_done), 118 KOBJMETHOD(icl_conn_transfer_setup, icl_soft_conn_transfer_setup), 119 KOBJMETHOD(icl_conn_transfer_done, icl_soft_conn_transfer_done), 120 { 0, 0 } 121 }; 122 123 DEFINE_CLASS(icl_soft, icl_soft_methods, sizeof(struct icl_conn)); 124 125 static void 126 icl_conn_fail(struct icl_conn *ic) 127 { 128 if (ic->ic_socket == NULL) 129 return; 130 131 /* 132 * XXX 133 */ 134 ic->ic_socket->so_error = EDOOFUS; 135 (ic->ic_error)(ic); 136 } 137 138 static struct mbuf * 139 icl_conn_receive(struct icl_conn *ic, size_t len) 140 { 141 struct uio uio; 142 struct socket *so; 143 struct mbuf *m; 144 int error, flags; 145 146 so = ic->ic_socket; 147 148 memset(&uio, 0, sizeof(uio)); 149 uio.uio_resid = len; 150 151 flags = MSG_DONTWAIT; 152 error = soreceive(so, NULL, &uio, &m, NULL, &flags); 153 if (error != 0) { 154 ICL_DEBUG("soreceive error %d", error); 155 return (NULL); 156 } 157 if (uio.uio_resid != 0) { 158 m_freem(m); 159 ICL_DEBUG("short read"); 160 return (NULL); 161 } 162 163 return (m); 164 } 165 166 static struct icl_pdu * 167 icl_pdu_new_empty(struct icl_conn *ic, int flags) 168 { 169 struct icl_pdu *ip; 170 171 #ifdef DIAGNOSTIC 172 refcount_acquire(&ic->ic_outstanding_pdus); 173 #endif 174 ip = uma_zalloc(icl_pdu_zone, flags | M_ZERO); 175 if (ip == NULL) { 176 ICL_WARN("failed to allocate %zd bytes", sizeof(*ip)); 177 #ifdef DIAGNOSTIC 178 refcount_release(&ic->ic_outstanding_pdus); 179 #endif 180 return (NULL); 181 } 182 183 ip->ip_conn = ic; 184 185 return (ip); 186 } 187 188 static void 189 icl_pdu_free(struct icl_pdu *ip) 190 { 191 struct icl_conn *ic; 192 193 ic = ip->ip_conn; 194 195 m_freem(ip->ip_bhs_mbuf); 196 m_freem(ip->ip_ahs_mbuf); 197 m_freem(ip->ip_data_mbuf); 198 uma_zfree(icl_pdu_zone, ip); 199 #ifdef DIAGNOSTIC 200 refcount_release(&ic->ic_outstanding_pdus); 201 #endif 202 } 203 204 void 205 icl_soft_conn_pdu_free(struct icl_conn *ic, struct icl_pdu *ip) 206 { 207 208 icl_pdu_free(ip); 209 } 210 211 /* 212 * Allocate icl_pdu with empty BHS to fill up by the caller. 213 */ 214 struct icl_pdu * 215 icl_soft_conn_new_pdu(struct icl_conn *ic, int flags) 216 { 217 struct icl_pdu *ip; 218 219 ip = icl_pdu_new_empty(ic, flags); 220 if (ip == NULL) 221 return (NULL); 222 223 ip->ip_bhs_mbuf = m_getm2(NULL, sizeof(struct iscsi_bhs), 224 flags, MT_DATA, M_PKTHDR); 225 if (ip->ip_bhs_mbuf == NULL) { 226 ICL_WARN("failed to allocate %zd bytes", sizeof(*ip)); 227 icl_pdu_free(ip); 228 return (NULL); 229 } 230 ip->ip_bhs = mtod(ip->ip_bhs_mbuf, struct iscsi_bhs *); 231 memset(ip->ip_bhs, 0, sizeof(struct iscsi_bhs)); 232 ip->ip_bhs_mbuf->m_len = sizeof(struct iscsi_bhs); 233 234 return (ip); 235 } 236 237 static int 238 icl_pdu_ahs_length(const struct icl_pdu *request) 239 { 240 241 return (request->ip_bhs->bhs_total_ahs_len * 4); 242 } 243 244 static size_t 245 icl_pdu_data_segment_length(const struct icl_pdu *request) 246 { 247 uint32_t len = 0; 248 249 len += request->ip_bhs->bhs_data_segment_len[0]; 250 len <<= 8; 251 len += request->ip_bhs->bhs_data_segment_len[1]; 252 len <<= 8; 253 len += request->ip_bhs->bhs_data_segment_len[2]; 254 255 return (len); 256 } 257 258 size_t 259 icl_soft_conn_pdu_data_segment_length(struct icl_conn *ic, 260 const struct icl_pdu *request) 261 { 262 263 return (icl_pdu_data_segment_length(request)); 264 } 265 266 static void 267 icl_pdu_set_data_segment_length(struct icl_pdu *response, uint32_t len) 268 { 269 270 response->ip_bhs->bhs_data_segment_len[2] = len; 271 response->ip_bhs->bhs_data_segment_len[1] = len >> 8; 272 response->ip_bhs->bhs_data_segment_len[0] = len >> 16; 273 } 274 275 static size_t 276 icl_pdu_padding(const struct icl_pdu *ip) 277 { 278 279 if ((ip->ip_data_len % 4) != 0) 280 return (4 - (ip->ip_data_len % 4)); 281 282 return (0); 283 } 284 285 static size_t 286 icl_pdu_size(const struct icl_pdu *response) 287 { 288 size_t len; 289 290 KASSERT(response->ip_ahs_len == 0, ("responding with AHS")); 291 292 len = sizeof(struct iscsi_bhs) + response->ip_data_len + 293 icl_pdu_padding(response); 294 if (response->ip_conn->ic_header_crc32c) 295 len += ISCSI_HEADER_DIGEST_SIZE; 296 if (response->ip_data_len != 0 && response->ip_conn->ic_data_crc32c) 297 len += ISCSI_DATA_DIGEST_SIZE; 298 299 return (len); 300 } 301 302 static int 303 icl_pdu_receive_bhs(struct icl_pdu *request, size_t *availablep) 304 { 305 struct mbuf *m; 306 307 m = icl_conn_receive(request->ip_conn, sizeof(struct iscsi_bhs)); 308 if (m == NULL) { 309 ICL_DEBUG("failed to receive BHS"); 310 return (-1); 311 } 312 313 request->ip_bhs_mbuf = m_pullup(m, sizeof(struct iscsi_bhs)); 314 if (request->ip_bhs_mbuf == NULL) { 315 ICL_WARN("m_pullup failed"); 316 return (-1); 317 } 318 request->ip_bhs = mtod(request->ip_bhs_mbuf, struct iscsi_bhs *); 319 320 /* 321 * XXX: For architectures with strict alignment requirements 322 * we may need to allocate ip_bhs and copy the data into it. 323 * For some reason, though, not doing this doesn't seem 324 * to cause problems; tested on sparc64. 325 */ 326 327 *availablep -= sizeof(struct iscsi_bhs); 328 return (0); 329 } 330 331 static int 332 icl_pdu_receive_ahs(struct icl_pdu *request, size_t *availablep) 333 { 334 335 request->ip_ahs_len = icl_pdu_ahs_length(request); 336 if (request->ip_ahs_len == 0) 337 return (0); 338 339 request->ip_ahs_mbuf = icl_conn_receive(request->ip_conn, 340 request->ip_ahs_len); 341 if (request->ip_ahs_mbuf == NULL) { 342 ICL_DEBUG("failed to receive AHS"); 343 return (-1); 344 } 345 346 *availablep -= request->ip_ahs_len; 347 return (0); 348 } 349 350 static uint32_t 351 icl_mbuf_to_crc32c(const struct mbuf *m0) 352 { 353 uint32_t digest = 0xffffffff; 354 const struct mbuf *m; 355 356 for (m = m0; m != NULL; m = m->m_next) 357 digest = calculate_crc32c(digest, 358 mtod(m, const void *), m->m_len); 359 360 digest = digest ^ 0xffffffff; 361 362 return (digest); 363 } 364 365 static int 366 icl_pdu_check_header_digest(struct icl_pdu *request, size_t *availablep) 367 { 368 struct mbuf *m; 369 uint32_t received_digest, valid_digest; 370 371 if (request->ip_conn->ic_header_crc32c == false) 372 return (0); 373 374 m = icl_conn_receive(request->ip_conn, ISCSI_HEADER_DIGEST_SIZE); 375 if (m == NULL) { 376 ICL_DEBUG("failed to receive header digest"); 377 return (-1); 378 } 379 380 CTASSERT(sizeof(received_digest) == ISCSI_HEADER_DIGEST_SIZE); 381 m_copydata(m, 0, ISCSI_HEADER_DIGEST_SIZE, (void *)&received_digest); 382 m_freem(m); 383 384 *availablep -= ISCSI_HEADER_DIGEST_SIZE; 385 386 /* 387 * XXX: Handle AHS. 388 */ 389 valid_digest = icl_mbuf_to_crc32c(request->ip_bhs_mbuf); 390 if (received_digest != valid_digest) { 391 ICL_WARN("header digest check failed; got 0x%x, " 392 "should be 0x%x", received_digest, valid_digest); 393 return (-1); 394 } 395 396 return (0); 397 } 398 399 /* 400 * Return the number of bytes that should be waiting in the receive socket 401 * before icl_pdu_receive_data_segment() gets called. 402 */ 403 static size_t 404 icl_pdu_data_segment_receive_len(const struct icl_pdu *request) 405 { 406 size_t len; 407 408 len = icl_pdu_data_segment_length(request); 409 if (len == 0) 410 return (0); 411 412 /* 413 * Account for the parts of data segment already read from 414 * the socket buffer. 415 */ 416 KASSERT(len > request->ip_data_len, ("len <= request->ip_data_len")); 417 len -= request->ip_data_len; 418 419 /* 420 * Don't always wait for the full data segment to be delivered 421 * to the socket; this might badly affect performance due to 422 * TCP window scaling. 423 */ 424 if (len > partial_receive_len) { 425 #if 0 426 ICL_DEBUG("need %zd bytes of data, limiting to %zd", 427 len, partial_receive_len)); 428 #endif 429 len = partial_receive_len; 430 431 return (len); 432 } 433 434 /* 435 * Account for padding. Note that due to the way code is written, 436 * the icl_pdu_receive_data_segment() must always receive padding 437 * along with the last part of data segment, because it would be 438 * impossible to tell whether we've already received the full data 439 * segment including padding, or without it. 440 */ 441 if ((len % 4) != 0) 442 len += 4 - (len % 4); 443 444 #if 0 445 ICL_DEBUG("need %zd bytes of data", len)); 446 #endif 447 448 return (len); 449 } 450 451 static int 452 icl_pdu_receive_data_segment(struct icl_pdu *request, 453 size_t *availablep, bool *more_neededp) 454 { 455 struct icl_conn *ic; 456 size_t len, padding = 0; 457 struct mbuf *m; 458 459 ic = request->ip_conn; 460 461 *more_neededp = false; 462 ic->ic_receive_len = 0; 463 464 len = icl_pdu_data_segment_length(request); 465 if (len == 0) 466 return (0); 467 468 if ((len % 4) != 0) 469 padding = 4 - (len % 4); 470 471 /* 472 * Account for already received parts of data segment. 473 */ 474 KASSERT(len > request->ip_data_len, ("len <= request->ip_data_len")); 475 len -= request->ip_data_len; 476 477 if (len + padding > *availablep) { 478 /* 479 * Not enough data in the socket buffer. Receive as much 480 * as we can. Don't receive padding, since, obviously, it's 481 * not the end of data segment yet. 482 */ 483 #if 0 484 ICL_DEBUG("limited from %zd to %zd", 485 len + padding, *availablep - padding)); 486 #endif 487 len = *availablep - padding; 488 *more_neededp = true; 489 padding = 0; 490 } 491 492 /* 493 * Must not try to receive padding without at least one byte 494 * of actual data segment. 495 */ 496 if (len > 0) { 497 m = icl_conn_receive(request->ip_conn, len + padding); 498 if (m == NULL) { 499 ICL_DEBUG("failed to receive data segment"); 500 return (-1); 501 } 502 503 if (request->ip_data_mbuf == NULL) 504 request->ip_data_mbuf = m; 505 else 506 m_cat(request->ip_data_mbuf, m); 507 508 request->ip_data_len += len; 509 *availablep -= len + padding; 510 } else 511 ICL_DEBUG("len 0"); 512 513 if (*more_neededp) 514 ic->ic_receive_len = 515 icl_pdu_data_segment_receive_len(request); 516 517 return (0); 518 } 519 520 static int 521 icl_pdu_check_data_digest(struct icl_pdu *request, size_t *availablep) 522 { 523 struct mbuf *m; 524 uint32_t received_digest, valid_digest; 525 526 if (request->ip_conn->ic_data_crc32c == false) 527 return (0); 528 529 if (request->ip_data_len == 0) 530 return (0); 531 532 m = icl_conn_receive(request->ip_conn, ISCSI_DATA_DIGEST_SIZE); 533 if (m == NULL) { 534 ICL_DEBUG("failed to receive data digest"); 535 return (-1); 536 } 537 538 CTASSERT(sizeof(received_digest) == ISCSI_DATA_DIGEST_SIZE); 539 m_copydata(m, 0, ISCSI_DATA_DIGEST_SIZE, (void *)&received_digest); 540 m_freem(m); 541 542 *availablep -= ISCSI_DATA_DIGEST_SIZE; 543 544 /* 545 * Note that ip_data_mbuf also contains padding; since digest 546 * calculation is supposed to include that, we iterate over 547 * the entire ip_data_mbuf chain, not just ip_data_len bytes of it. 548 */ 549 valid_digest = icl_mbuf_to_crc32c(request->ip_data_mbuf); 550 if (received_digest != valid_digest) { 551 ICL_WARN("data digest check failed; got 0x%x, " 552 "should be 0x%x", received_digest, valid_digest); 553 return (-1); 554 } 555 556 return (0); 557 } 558 559 /* 560 * Somewhat contrary to the name, this attempts to receive only one 561 * "part" of PDU at a time; call it repeatedly until it returns non-NULL. 562 */ 563 static struct icl_pdu * 564 icl_conn_receive_pdu(struct icl_conn *ic, size_t *availablep) 565 { 566 struct icl_pdu *request; 567 struct socket *so; 568 size_t len; 569 int error; 570 bool more_needed; 571 572 so = ic->ic_socket; 573 574 if (ic->ic_receive_state == ICL_CONN_STATE_BHS) { 575 KASSERT(ic->ic_receive_pdu == NULL, 576 ("ic->ic_receive_pdu != NULL")); 577 request = icl_pdu_new_empty(ic, M_NOWAIT); 578 if (request == NULL) { 579 ICL_DEBUG("failed to allocate PDU; " 580 "dropping connection"); 581 icl_conn_fail(ic); 582 return (NULL); 583 } 584 ic->ic_receive_pdu = request; 585 } else { 586 KASSERT(ic->ic_receive_pdu != NULL, 587 ("ic->ic_receive_pdu == NULL")); 588 request = ic->ic_receive_pdu; 589 } 590 591 if (*availablep < ic->ic_receive_len) { 592 #if 0 593 ICL_DEBUG("not enough data; need %zd, " 594 "have %zd", ic->ic_receive_len, *availablep); 595 #endif 596 return (NULL); 597 } 598 599 switch (ic->ic_receive_state) { 600 case ICL_CONN_STATE_BHS: 601 //ICL_DEBUG("receiving BHS"); 602 error = icl_pdu_receive_bhs(request, availablep); 603 if (error != 0) { 604 ICL_DEBUG("failed to receive BHS; " 605 "dropping connection"); 606 break; 607 } 608 609 /* 610 * We don't enforce any limit for AHS length; 611 * its length is stored in 8 bit field. 612 */ 613 614 len = icl_pdu_data_segment_length(request); 615 if (len > ic->ic_max_data_segment_length) { 616 ICL_WARN("received data segment " 617 "length %zd is larger than negotiated " 618 "MaxDataSegmentLength %zd; " 619 "dropping connection", 620 len, ic->ic_max_data_segment_length); 621 error = EINVAL; 622 break; 623 } 624 625 ic->ic_receive_state = ICL_CONN_STATE_AHS; 626 ic->ic_receive_len = icl_pdu_ahs_length(request); 627 break; 628 629 case ICL_CONN_STATE_AHS: 630 //ICL_DEBUG("receiving AHS"); 631 error = icl_pdu_receive_ahs(request, availablep); 632 if (error != 0) { 633 ICL_DEBUG("failed to receive AHS; " 634 "dropping connection"); 635 break; 636 } 637 ic->ic_receive_state = ICL_CONN_STATE_HEADER_DIGEST; 638 if (ic->ic_header_crc32c == false) 639 ic->ic_receive_len = 0; 640 else 641 ic->ic_receive_len = ISCSI_HEADER_DIGEST_SIZE; 642 break; 643 644 case ICL_CONN_STATE_HEADER_DIGEST: 645 //ICL_DEBUG("receiving header digest"); 646 error = icl_pdu_check_header_digest(request, availablep); 647 if (error != 0) { 648 ICL_DEBUG("header digest failed; " 649 "dropping connection"); 650 break; 651 } 652 653 ic->ic_receive_state = ICL_CONN_STATE_DATA; 654 ic->ic_receive_len = 655 icl_pdu_data_segment_receive_len(request); 656 break; 657 658 case ICL_CONN_STATE_DATA: 659 //ICL_DEBUG("receiving data segment"); 660 error = icl_pdu_receive_data_segment(request, availablep, 661 &more_needed); 662 if (error != 0) { 663 ICL_DEBUG("failed to receive data segment;" 664 "dropping connection"); 665 break; 666 } 667 668 if (more_needed) 669 break; 670 671 ic->ic_receive_state = ICL_CONN_STATE_DATA_DIGEST; 672 if (request->ip_data_len == 0 || ic->ic_data_crc32c == false) 673 ic->ic_receive_len = 0; 674 else 675 ic->ic_receive_len = ISCSI_DATA_DIGEST_SIZE; 676 break; 677 678 case ICL_CONN_STATE_DATA_DIGEST: 679 //ICL_DEBUG("receiving data digest"); 680 error = icl_pdu_check_data_digest(request, availablep); 681 if (error != 0) { 682 ICL_DEBUG("data digest failed; " 683 "dropping connection"); 684 break; 685 } 686 687 /* 688 * We've received complete PDU; reset the receive state machine 689 * and return the PDU. 690 */ 691 ic->ic_receive_state = ICL_CONN_STATE_BHS; 692 ic->ic_receive_len = sizeof(struct iscsi_bhs); 693 ic->ic_receive_pdu = NULL; 694 return (request); 695 696 default: 697 panic("invalid ic_receive_state %d\n", ic->ic_receive_state); 698 } 699 700 if (error != 0) { 701 /* 702 * Don't free the PDU; it's pointed to by ic->ic_receive_pdu 703 * and will get freed in icl_soft_conn_close(). 704 */ 705 icl_conn_fail(ic); 706 } 707 708 return (NULL); 709 } 710 711 static void 712 icl_conn_receive_pdus(struct icl_conn *ic, size_t available) 713 { 714 struct icl_pdu *response; 715 struct socket *so; 716 717 so = ic->ic_socket; 718 719 /* 720 * This can never happen; we're careful to only mess with ic->ic_socket 721 * pointer when the send/receive threads are not running. 722 */ 723 KASSERT(so != NULL, ("NULL socket")); 724 725 for (;;) { 726 if (ic->ic_disconnecting) 727 return; 728 729 if (so->so_error != 0) { 730 ICL_DEBUG("connection error %d; " 731 "dropping connection", so->so_error); 732 icl_conn_fail(ic); 733 return; 734 } 735 736 /* 737 * Loop until we have a complete PDU or there is not enough 738 * data in the socket buffer. 739 */ 740 if (available < ic->ic_receive_len) { 741 #if 0 742 ICL_DEBUG("not enough data; have %zd, " 743 "need %zd", available, 744 ic->ic_receive_len); 745 #endif 746 return; 747 } 748 749 response = icl_conn_receive_pdu(ic, &available); 750 if (response == NULL) 751 continue; 752 753 if (response->ip_ahs_len > 0) { 754 ICL_WARN("received PDU with unsupported " 755 "AHS; opcode 0x%x; dropping connection", 756 response->ip_bhs->bhs_opcode); 757 icl_pdu_free(response); 758 icl_conn_fail(ic); 759 return; 760 } 761 762 (ic->ic_receive)(response); 763 } 764 } 765 766 static void 767 icl_receive_thread(void *arg) 768 { 769 struct icl_conn *ic; 770 size_t available; 771 struct socket *so; 772 773 ic = arg; 774 so = ic->ic_socket; 775 776 ICL_CONN_LOCK(ic); 777 ic->ic_receive_running = true; 778 ICL_CONN_UNLOCK(ic); 779 780 for (;;) { 781 if (ic->ic_disconnecting) { 782 //ICL_DEBUG("terminating"); 783 break; 784 } 785 786 /* 787 * Set the low watermark, to be checked by 788 * soreadable() in icl_soupcall_receive() 789 * to avoid unneccessary wakeups until there 790 * is enough data received to read the PDU. 791 */ 792 SOCKBUF_LOCK(&so->so_rcv); 793 available = sbavail(&so->so_rcv); 794 if (available < ic->ic_receive_len) { 795 so->so_rcv.sb_lowat = ic->ic_receive_len; 796 cv_wait(&ic->ic_receive_cv, &so->so_rcv.sb_mtx); 797 } else 798 so->so_rcv.sb_lowat = so->so_rcv.sb_hiwat + 1; 799 SOCKBUF_UNLOCK(&so->so_rcv); 800 801 icl_conn_receive_pdus(ic, available); 802 } 803 804 ICL_CONN_LOCK(ic); 805 ic->ic_receive_running = false; 806 cv_signal(&ic->ic_send_cv); 807 ICL_CONN_UNLOCK(ic); 808 kthread_exit(); 809 } 810 811 static int 812 icl_soupcall_receive(struct socket *so, void *arg, int waitflag) 813 { 814 struct icl_conn *ic; 815 816 if (!soreadable(so)) 817 return (SU_OK); 818 819 ic = arg; 820 cv_signal(&ic->ic_receive_cv); 821 return (SU_OK); 822 } 823 824 static int 825 icl_pdu_finalize(struct icl_pdu *request) 826 { 827 size_t padding, pdu_len; 828 uint32_t digest, zero = 0; 829 int ok; 830 struct icl_conn *ic; 831 832 ic = request->ip_conn; 833 834 icl_pdu_set_data_segment_length(request, request->ip_data_len); 835 836 pdu_len = icl_pdu_size(request); 837 838 if (ic->ic_header_crc32c) { 839 digest = icl_mbuf_to_crc32c(request->ip_bhs_mbuf); 840 ok = m_append(request->ip_bhs_mbuf, sizeof(digest), 841 (void *)&digest); 842 if (ok != 1) { 843 ICL_WARN("failed to append header digest"); 844 return (1); 845 } 846 } 847 848 if (request->ip_data_len != 0) { 849 padding = icl_pdu_padding(request); 850 if (padding > 0) { 851 ok = m_append(request->ip_data_mbuf, padding, 852 (void *)&zero); 853 if (ok != 1) { 854 ICL_WARN("failed to append padding"); 855 return (1); 856 } 857 } 858 859 if (ic->ic_data_crc32c) { 860 digest = icl_mbuf_to_crc32c(request->ip_data_mbuf); 861 862 ok = m_append(request->ip_data_mbuf, sizeof(digest), 863 (void *)&digest); 864 if (ok != 1) { 865 ICL_WARN("failed to append data digest"); 866 return (1); 867 } 868 } 869 870 m_cat(request->ip_bhs_mbuf, request->ip_data_mbuf); 871 request->ip_data_mbuf = NULL; 872 } 873 874 request->ip_bhs_mbuf->m_pkthdr.len = pdu_len; 875 876 return (0); 877 } 878 879 static void 880 icl_conn_send_pdus(struct icl_conn *ic, struct icl_pdu_stailq *queue) 881 { 882 struct icl_pdu *request, *request2; 883 struct socket *so; 884 size_t available, size, size2; 885 int coalesced, error; 886 887 ICL_CONN_LOCK_ASSERT_NOT(ic); 888 889 so = ic->ic_socket; 890 891 SOCKBUF_LOCK(&so->so_snd); 892 /* 893 * Check how much space do we have for transmit. We can't just 894 * call sosend() and retry when we get EWOULDBLOCK or EMSGSIZE, 895 * as it always frees the mbuf chain passed to it, even in case 896 * of error. 897 */ 898 available = sbspace(&so->so_snd); 899 900 /* 901 * Notify the socket upcall that we don't need wakeups 902 * for the time being. 903 */ 904 so->so_snd.sb_lowat = so->so_snd.sb_hiwat + 1; 905 SOCKBUF_UNLOCK(&so->so_snd); 906 907 while (!STAILQ_EMPTY(queue)) { 908 request = STAILQ_FIRST(queue); 909 size = icl_pdu_size(request); 910 if (available < size) { 911 912 /* 913 * Set the low watermark, to be checked by 914 * sowriteable() in icl_soupcall_send() 915 * to avoid unneccessary wakeups until there 916 * is enough space for the PDU to fit. 917 */ 918 SOCKBUF_LOCK(&so->so_snd); 919 available = sbspace(&so->so_snd); 920 if (available < size) { 921 #if 1 922 ICL_DEBUG("no space to send; " 923 "have %zd, need %zd", 924 available, size); 925 #endif 926 so->so_snd.sb_lowat = size; 927 SOCKBUF_UNLOCK(&so->so_snd); 928 return; 929 } 930 SOCKBUF_UNLOCK(&so->so_snd); 931 } 932 STAILQ_REMOVE_HEAD(queue, ip_next); 933 error = icl_pdu_finalize(request); 934 if (error != 0) { 935 ICL_DEBUG("failed to finalize PDU; " 936 "dropping connection"); 937 icl_conn_fail(ic); 938 icl_pdu_free(request); 939 return; 940 } 941 if (coalesce) { 942 coalesced = 1; 943 for (;;) { 944 request2 = STAILQ_FIRST(queue); 945 if (request2 == NULL) 946 break; 947 size2 = icl_pdu_size(request2); 948 if (available < size + size2) 949 break; 950 STAILQ_REMOVE_HEAD(queue, ip_next); 951 error = icl_pdu_finalize(request2); 952 if (error != 0) { 953 ICL_DEBUG("failed to finalize PDU; " 954 "dropping connection"); 955 icl_conn_fail(ic); 956 icl_pdu_free(request); 957 icl_pdu_free(request2); 958 return; 959 } 960 m_cat(request->ip_bhs_mbuf, request2->ip_bhs_mbuf); 961 request2->ip_bhs_mbuf = NULL; 962 request->ip_bhs_mbuf->m_pkthdr.len += size2; 963 size += size2; 964 STAILQ_REMOVE_AFTER(queue, request, ip_next); 965 icl_pdu_free(request2); 966 coalesced++; 967 } 968 #if 0 969 if (coalesced > 1) { 970 ICL_DEBUG("coalesced %d PDUs into %zd bytes", 971 coalesced, size); 972 } 973 #endif 974 } 975 available -= size; 976 error = sosend(so, NULL, NULL, request->ip_bhs_mbuf, 977 NULL, MSG_DONTWAIT, curthread); 978 request->ip_bhs_mbuf = NULL; /* Sosend consumes the mbuf. */ 979 if (error != 0) { 980 ICL_DEBUG("failed to send PDU, error %d; " 981 "dropping connection", error); 982 icl_conn_fail(ic); 983 icl_pdu_free(request); 984 return; 985 } 986 icl_pdu_free(request); 987 } 988 } 989 990 static void 991 icl_send_thread(void *arg) 992 { 993 struct icl_conn *ic; 994 struct icl_pdu_stailq queue; 995 996 ic = arg; 997 998 STAILQ_INIT(&queue); 999 1000 ICL_CONN_LOCK(ic); 1001 ic->ic_send_running = true; 1002 1003 for (;;) { 1004 for (;;) { 1005 /* 1006 * If the local queue is empty, populate it from 1007 * the main one. This way the icl_conn_send_pdus() 1008 * can go through all the queued PDUs without holding 1009 * any locks. 1010 */ 1011 if (STAILQ_EMPTY(&queue)) 1012 STAILQ_SWAP(&ic->ic_to_send, &queue, icl_pdu); 1013 1014 ic->ic_check_send_space = false; 1015 ICL_CONN_UNLOCK(ic); 1016 icl_conn_send_pdus(ic, &queue); 1017 ICL_CONN_LOCK(ic); 1018 1019 /* 1020 * The icl_soupcall_send() was called since the last 1021 * call to sbspace(); go around; 1022 */ 1023 if (ic->ic_check_send_space) 1024 continue; 1025 1026 /* 1027 * Local queue is empty, but we still have PDUs 1028 * in the main one; go around. 1029 */ 1030 if (STAILQ_EMPTY(&queue) && 1031 !STAILQ_EMPTY(&ic->ic_to_send)) 1032 continue; 1033 1034 /* 1035 * There might be some stuff in the local queue, 1036 * which didn't get sent due to not having enough send 1037 * space. Wait for socket upcall. 1038 */ 1039 break; 1040 } 1041 1042 if (ic->ic_disconnecting) { 1043 //ICL_DEBUG("terminating"); 1044 break; 1045 } 1046 1047 cv_wait(&ic->ic_send_cv, ic->ic_lock); 1048 } 1049 1050 /* 1051 * We're exiting; move PDUs back to the main queue, so they can 1052 * get freed properly. At this point ordering doesn't matter. 1053 */ 1054 STAILQ_CONCAT(&ic->ic_to_send, &queue); 1055 1056 ic->ic_send_running = false; 1057 cv_signal(&ic->ic_send_cv); 1058 ICL_CONN_UNLOCK(ic); 1059 kthread_exit(); 1060 } 1061 1062 static int 1063 icl_soupcall_send(struct socket *so, void *arg, int waitflag) 1064 { 1065 struct icl_conn *ic; 1066 1067 if (!sowriteable(so)) 1068 return (SU_OK); 1069 1070 ic = arg; 1071 1072 ICL_CONN_LOCK(ic); 1073 ic->ic_check_send_space = true; 1074 ICL_CONN_UNLOCK(ic); 1075 1076 cv_signal(&ic->ic_send_cv); 1077 1078 return (SU_OK); 1079 } 1080 1081 static int 1082 icl_pdu_append_data(struct icl_pdu *request, const void *addr, size_t len, 1083 int flags) 1084 { 1085 struct mbuf *mb, *newmb; 1086 size_t copylen, off = 0; 1087 1088 KASSERT(len > 0, ("len == 0")); 1089 1090 newmb = m_getm2(NULL, len, flags, MT_DATA, M_PKTHDR); 1091 if (newmb == NULL) { 1092 ICL_WARN("failed to allocate mbuf for %zd bytes", len); 1093 return (ENOMEM); 1094 } 1095 1096 for (mb = newmb; mb != NULL; mb = mb->m_next) { 1097 copylen = min(M_TRAILINGSPACE(mb), len - off); 1098 memcpy(mtod(mb, char *), (const char *)addr + off, copylen); 1099 mb->m_len = copylen; 1100 off += copylen; 1101 } 1102 KASSERT(off == len, ("%s: off != len", __func__)); 1103 1104 if (request->ip_data_mbuf == NULL) { 1105 request->ip_data_mbuf = newmb; 1106 request->ip_data_len = len; 1107 } else { 1108 m_cat(request->ip_data_mbuf, newmb); 1109 request->ip_data_len += len; 1110 } 1111 1112 return (0); 1113 } 1114 1115 int 1116 icl_soft_conn_pdu_append_data(struct icl_conn *ic, struct icl_pdu *request, 1117 const void *addr, size_t len, int flags) 1118 { 1119 1120 return (icl_pdu_append_data(request, addr, len, flags)); 1121 } 1122 1123 static void 1124 icl_pdu_get_data(struct icl_pdu *ip, size_t off, void *addr, size_t len) 1125 { 1126 1127 m_copydata(ip->ip_data_mbuf, off, len, addr); 1128 } 1129 1130 void 1131 icl_soft_conn_pdu_get_data(struct icl_conn *ic, struct icl_pdu *ip, 1132 size_t off, void *addr, size_t len) 1133 { 1134 1135 return (icl_pdu_get_data(ip, off, addr, len)); 1136 } 1137 1138 static void 1139 icl_pdu_queue(struct icl_pdu *ip) 1140 { 1141 struct icl_conn *ic; 1142 1143 ic = ip->ip_conn; 1144 1145 ICL_CONN_LOCK_ASSERT(ic); 1146 1147 if (ic->ic_disconnecting || ic->ic_socket == NULL) { 1148 ICL_DEBUG("icl_pdu_queue on closed connection"); 1149 icl_pdu_free(ip); 1150 return; 1151 } 1152 1153 if (!STAILQ_EMPTY(&ic->ic_to_send)) { 1154 STAILQ_INSERT_TAIL(&ic->ic_to_send, ip, ip_next); 1155 /* 1156 * If the queue is not empty, someone else had already 1157 * signaled the send thread; no need to do that again, 1158 * just return. 1159 */ 1160 return; 1161 } 1162 1163 STAILQ_INSERT_TAIL(&ic->ic_to_send, ip, ip_next); 1164 cv_signal(&ic->ic_send_cv); 1165 } 1166 1167 void 1168 icl_soft_conn_pdu_queue(struct icl_conn *ic, struct icl_pdu *ip) 1169 { 1170 1171 icl_pdu_queue(ip); 1172 } 1173 1174 static struct icl_conn * 1175 icl_soft_new_conn(const char *name, struct mtx *lock) 1176 { 1177 struct icl_conn *ic; 1178 1179 refcount_acquire(&icl_ncons); 1180 1181 ic = (struct icl_conn *)kobj_create(&icl_soft_class, M_ICL_SOFT, M_WAITOK | M_ZERO); 1182 1183 STAILQ_INIT(&ic->ic_to_send); 1184 ic->ic_lock = lock; 1185 cv_init(&ic->ic_send_cv, "icl_tx"); 1186 cv_init(&ic->ic_receive_cv, "icl_rx"); 1187 #ifdef DIAGNOSTIC 1188 refcount_init(&ic->ic_outstanding_pdus, 0); 1189 #endif 1190 ic->ic_max_data_segment_length = ICL_MAX_DATA_SEGMENT_LENGTH; 1191 ic->ic_name = name; 1192 ic->ic_offload = "None"; 1193 1194 return (ic); 1195 } 1196 1197 void 1198 icl_soft_conn_free(struct icl_conn *ic) 1199 { 1200 1201 cv_destroy(&ic->ic_send_cv); 1202 cv_destroy(&ic->ic_receive_cv); 1203 kobj_delete((struct kobj *)ic, M_ICL_SOFT); 1204 refcount_release(&icl_ncons); 1205 } 1206 1207 static int 1208 icl_conn_start(struct icl_conn *ic) 1209 { 1210 size_t minspace; 1211 struct sockopt opt; 1212 int error, one = 1; 1213 1214 ICL_CONN_LOCK(ic); 1215 1216 /* 1217 * XXX: Ugly hack. 1218 */ 1219 if (ic->ic_socket == NULL) { 1220 ICL_CONN_UNLOCK(ic); 1221 return (EINVAL); 1222 } 1223 1224 ic->ic_receive_state = ICL_CONN_STATE_BHS; 1225 ic->ic_receive_len = sizeof(struct iscsi_bhs); 1226 ic->ic_disconnecting = false; 1227 1228 ICL_CONN_UNLOCK(ic); 1229 1230 /* 1231 * For sendspace, this is required because the current code cannot 1232 * send a PDU in pieces; thus, the minimum buffer size is equal 1233 * to the maximum PDU size. "+4" is to account for possible padding. 1234 * 1235 * What we should actually do here is to use autoscaling, but set 1236 * some minimal buffer size to "minspace". I don't know a way to do 1237 * that, though. 1238 */ 1239 minspace = sizeof(struct iscsi_bhs) + ic->ic_max_data_segment_length + 1240 ISCSI_HEADER_DIGEST_SIZE + ISCSI_DATA_DIGEST_SIZE + 4; 1241 if (sendspace < minspace) { 1242 ICL_WARN("kern.icl.sendspace too low; must be at least %zd", 1243 minspace); 1244 sendspace = minspace; 1245 } 1246 if (recvspace < minspace) { 1247 ICL_WARN("kern.icl.recvspace too low; must be at least %zd", 1248 minspace); 1249 recvspace = minspace; 1250 } 1251 1252 error = soreserve(ic->ic_socket, sendspace, recvspace); 1253 if (error != 0) { 1254 ICL_WARN("soreserve failed with error %d", error); 1255 icl_soft_conn_close(ic); 1256 return (error); 1257 } 1258 ic->ic_socket->so_snd.sb_flags |= SB_AUTOSIZE; 1259 ic->ic_socket->so_rcv.sb_flags |= SB_AUTOSIZE; 1260 1261 /* 1262 * Disable Nagle. 1263 */ 1264 bzero(&opt, sizeof(opt)); 1265 opt.sopt_dir = SOPT_SET; 1266 opt.sopt_level = IPPROTO_TCP; 1267 opt.sopt_name = TCP_NODELAY; 1268 opt.sopt_val = &one; 1269 opt.sopt_valsize = sizeof(one); 1270 error = sosetopt(ic->ic_socket, &opt); 1271 if (error != 0) { 1272 ICL_WARN("disabling TCP_NODELAY failed with error %d", error); 1273 icl_soft_conn_close(ic); 1274 return (error); 1275 } 1276 1277 /* 1278 * Start threads. 1279 */ 1280 error = kthread_add(icl_send_thread, ic, NULL, NULL, 0, 0, "%stx", 1281 ic->ic_name); 1282 if (error != 0) { 1283 ICL_WARN("kthread_add(9) failed with error %d", error); 1284 icl_soft_conn_close(ic); 1285 return (error); 1286 } 1287 1288 error = kthread_add(icl_receive_thread, ic, NULL, NULL, 0, 0, "%srx", 1289 ic->ic_name); 1290 if (error != 0) { 1291 ICL_WARN("kthread_add(9) failed with error %d", error); 1292 icl_soft_conn_close(ic); 1293 return (error); 1294 } 1295 1296 /* 1297 * Register socket upcall, to get notified about incoming PDUs 1298 * and free space to send outgoing ones. 1299 */ 1300 SOCKBUF_LOCK(&ic->ic_socket->so_snd); 1301 soupcall_set(ic->ic_socket, SO_SND, icl_soupcall_send, ic); 1302 SOCKBUF_UNLOCK(&ic->ic_socket->so_snd); 1303 SOCKBUF_LOCK(&ic->ic_socket->so_rcv); 1304 soupcall_set(ic->ic_socket, SO_RCV, icl_soupcall_receive, ic); 1305 SOCKBUF_UNLOCK(&ic->ic_socket->so_rcv); 1306 1307 return (0); 1308 } 1309 1310 int 1311 icl_soft_conn_handoff(struct icl_conn *ic, int fd) 1312 { 1313 struct file *fp; 1314 struct socket *so; 1315 cap_rights_t rights; 1316 int error; 1317 1318 ICL_CONN_LOCK_ASSERT_NOT(ic); 1319 1320 /* 1321 * Steal the socket from userland. 1322 */ 1323 error = fget(curthread, fd, 1324 cap_rights_init(&rights, CAP_SOCK_CLIENT), &fp); 1325 if (error != 0) 1326 return (error); 1327 if (fp->f_type != DTYPE_SOCKET) { 1328 fdrop(fp, curthread); 1329 return (EINVAL); 1330 } 1331 so = fp->f_data; 1332 if (so->so_type != SOCK_STREAM) { 1333 fdrop(fp, curthread); 1334 return (EINVAL); 1335 } 1336 1337 ICL_CONN_LOCK(ic); 1338 1339 if (ic->ic_socket != NULL) { 1340 ICL_CONN_UNLOCK(ic); 1341 fdrop(fp, curthread); 1342 return (EBUSY); 1343 } 1344 1345 ic->ic_socket = fp->f_data; 1346 fp->f_ops = &badfileops; 1347 fp->f_data = NULL; 1348 fdrop(fp, curthread); 1349 ICL_CONN_UNLOCK(ic); 1350 1351 error = icl_conn_start(ic); 1352 1353 return (error); 1354 } 1355 1356 void 1357 icl_soft_conn_close(struct icl_conn *ic) 1358 { 1359 struct icl_pdu *pdu; 1360 1361 ICL_CONN_LOCK_ASSERT_NOT(ic); 1362 1363 ICL_CONN_LOCK(ic); 1364 if (ic->ic_socket == NULL) { 1365 ICL_CONN_UNLOCK(ic); 1366 return; 1367 } 1368 1369 /* 1370 * Deregister socket upcalls. 1371 */ 1372 ICL_CONN_UNLOCK(ic); 1373 SOCKBUF_LOCK(&ic->ic_socket->so_snd); 1374 if (ic->ic_socket->so_snd.sb_upcall != NULL) 1375 soupcall_clear(ic->ic_socket, SO_SND); 1376 SOCKBUF_UNLOCK(&ic->ic_socket->so_snd); 1377 SOCKBUF_LOCK(&ic->ic_socket->so_rcv); 1378 if (ic->ic_socket->so_rcv.sb_upcall != NULL) 1379 soupcall_clear(ic->ic_socket, SO_RCV); 1380 SOCKBUF_UNLOCK(&ic->ic_socket->so_rcv); 1381 ICL_CONN_LOCK(ic); 1382 1383 ic->ic_disconnecting = true; 1384 1385 /* 1386 * Wake up the threads, so they can properly terminate. 1387 */ 1388 while (ic->ic_receive_running || ic->ic_send_running) { 1389 //ICL_DEBUG("waiting for send/receive threads to terminate"); 1390 cv_signal(&ic->ic_receive_cv); 1391 cv_signal(&ic->ic_send_cv); 1392 cv_wait(&ic->ic_send_cv, ic->ic_lock); 1393 } 1394 //ICL_DEBUG("send/receive threads terminated"); 1395 1396 ICL_CONN_UNLOCK(ic); 1397 soclose(ic->ic_socket); 1398 ICL_CONN_LOCK(ic); 1399 ic->ic_socket = NULL; 1400 1401 if (ic->ic_receive_pdu != NULL) { 1402 //ICL_DEBUG("freeing partially received PDU"); 1403 icl_pdu_free(ic->ic_receive_pdu); 1404 ic->ic_receive_pdu = NULL; 1405 } 1406 1407 /* 1408 * Remove any outstanding PDUs from the send queue. 1409 */ 1410 while (!STAILQ_EMPTY(&ic->ic_to_send)) { 1411 pdu = STAILQ_FIRST(&ic->ic_to_send); 1412 STAILQ_REMOVE_HEAD(&ic->ic_to_send, ip_next); 1413 icl_pdu_free(pdu); 1414 } 1415 1416 KASSERT(STAILQ_EMPTY(&ic->ic_to_send), 1417 ("destroying session with non-empty send queue")); 1418 #ifdef DIAGNOSTIC 1419 KASSERT(ic->ic_outstanding_pdus == 0, 1420 ("destroying session with %d outstanding PDUs", 1421 ic->ic_outstanding_pdus)); 1422 #endif 1423 ICL_CONN_UNLOCK(ic); 1424 } 1425 1426 int 1427 icl_soft_conn_task_setup(struct icl_conn *ic, struct ccb_scsiio *csio, 1428 uint32_t *task_tagp, void **prvp) 1429 { 1430 1431 return (0); 1432 } 1433 1434 void 1435 icl_soft_conn_task_done(struct icl_conn *ic, void *prv) 1436 { 1437 } 1438 1439 int 1440 icl_soft_conn_transfer_setup(struct icl_conn *ic, union ctl_io *io, 1441 uint32_t *transfer_tag, void **prvp) 1442 { 1443 1444 return (0); 1445 } 1446 1447 void 1448 icl_soft_conn_transfer_done(struct icl_conn *ic, void *prv) 1449 { 1450 } 1451 1452 static int 1453 icl_soft_limits(size_t *limitp) 1454 { 1455 1456 *limitp = 128 * 1024; 1457 1458 return (0); 1459 } 1460 1461 #ifdef ICL_KERNEL_PROXY 1462 int 1463 icl_conn_handoff_sock(struct icl_conn *ic, struct socket *so) 1464 { 1465 int error; 1466 1467 ICL_CONN_LOCK_ASSERT_NOT(ic); 1468 1469 if (so->so_type != SOCK_STREAM) 1470 return (EINVAL); 1471 1472 ICL_CONN_LOCK(ic); 1473 if (ic->ic_socket != NULL) { 1474 ICL_CONN_UNLOCK(ic); 1475 return (EBUSY); 1476 } 1477 ic->ic_socket = so; 1478 ICL_CONN_UNLOCK(ic); 1479 1480 error = icl_conn_start(ic); 1481 1482 return (error); 1483 } 1484 #endif /* ICL_KERNEL_PROXY */ 1485 1486 static int 1487 icl_soft_load(void) 1488 { 1489 int error; 1490 1491 icl_pdu_zone = uma_zcreate("icl_pdu", 1492 sizeof(struct icl_pdu), NULL, NULL, NULL, NULL, 1493 UMA_ALIGN_PTR, 0); 1494 refcount_init(&icl_ncons, 0); 1495 1496 /* 1497 * The reason we call this "none" is that to the user, 1498 * it's known as "offload driver"; "offload driver: soft" 1499 * doesn't make much sense. 1500 */ 1501 error = icl_register("none", 0, icl_soft_limits, icl_soft_new_conn); 1502 KASSERT(error == 0, ("failed to register")); 1503 1504 return (error); 1505 } 1506 1507 static int 1508 icl_soft_unload(void) 1509 { 1510 1511 if (icl_ncons != 0) 1512 return (EBUSY); 1513 1514 icl_unregister("none"); 1515 1516 uma_zdestroy(icl_pdu_zone); 1517 1518 return (0); 1519 } 1520 1521 static int 1522 icl_soft_modevent(module_t mod, int what, void *arg) 1523 { 1524 1525 switch (what) { 1526 case MOD_LOAD: 1527 return (icl_soft_load()); 1528 case MOD_UNLOAD: 1529 return (icl_soft_unload()); 1530 default: 1531 return (EINVAL); 1532 } 1533 } 1534 1535 moduledata_t icl_soft_data = { 1536 "icl_soft", 1537 icl_soft_modevent, 1538 0 1539 }; 1540 1541 DECLARE_MODULE(icl_soft, icl_soft_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE); 1542 MODULE_DEPEND(icl_soft, icl, 1, 1, 1); 1543 MODULE_VERSION(icl_soft, 1); 1544