1 /*- 2 * Copyright (c) 2012 Chelsio Communications, Inc. 3 * All rights reserved. 4 * 5 * Chelsio T5xx iSCSI driver 6 * 7 * Written by: Sreenivasa Honnur <shonnur@chelsio.com> 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include "opt_inet.h" 35 #include "opt_inet6.h" 36 37 #include <sys/types.h> 38 #include <sys/param.h> 39 #include <sys/kernel.h> 40 #include <sys/ktr.h> 41 #include <sys/module.h> 42 #include <sys/systm.h> 43 44 #ifdef TCP_OFFLOAD 45 #include <sys/errno.h> 46 #include <sys/gsb_crc32.h> 47 #include <sys/kthread.h> 48 #include <sys/smp.h> 49 #include <sys/socket.h> 50 #include <sys/socketvar.h> 51 #include <sys/mbuf.h> 52 #include <sys/lock.h> 53 #include <sys/mutex.h> 54 #include <sys/condvar.h> 55 56 #include <netinet/in.h> 57 #include <netinet/in_pcb.h> 58 #include <netinet/toecore.h> 59 #include <netinet/tcp_var.h> 60 #include <netinet/tcp_fsm.h> 61 62 #include <cam/scsi/scsi_all.h> 63 #include <cam/scsi/scsi_da.h> 64 #include <cam/ctl/ctl_io.h> 65 #include <cam/ctl/ctl.h> 66 #include <cam/ctl/ctl_backend.h> 67 #include <cam/ctl/ctl_error.h> 68 #include <cam/ctl/ctl_frontend.h> 69 #include <cam/ctl/ctl_debug.h> 70 #include <cam/ctl/ctl_ha.h> 71 #include <cam/ctl/ctl_ioctl.h> 72 73 #include <dev/iscsi/icl.h> 74 #include <dev/iscsi/iscsi_proto.h> 75 #include <dev/iscsi/iscsi_ioctl.h> 76 #include <dev/iscsi/iscsi.h> 77 #include <cam/ctl/ctl_frontend_iscsi.h> 78 79 #include <cam/cam.h> 80 #include <cam/cam_ccb.h> 81 #include <cam/cam_xpt.h> 82 #include <cam/cam_debug.h> 83 #include <cam/cam_sim.h> 84 #include <cam/cam_xpt_sim.h> 85 #include <cam/cam_xpt_periph.h> 86 #include <cam/cam_periph.h> 87 #include <cam/cam_compat.h> 88 #include <cam/scsi/scsi_message.h> 89 90 #include "common/common.h" 91 #include "common/t4_msg.h" 92 #include "common/t4_regs.h" /* for PCIE_MEM_ACCESS */ 93 #include "tom/t4_tom.h" 94 #include "cxgbei.h" 95 96 static int worker_thread_count; 97 static struct cxgbei_worker_thread_softc *cwt_softc; 98 static struct proc *cxgbei_proc; 99 100 static void 101 read_pdu_limits(struct adapter *sc, uint32_t *max_tx_data_len, 102 uint32_t *max_rx_data_len, struct ppod_region *pr) 103 { 104 uint32_t tx_len, rx_len, r, v; 105 106 rx_len = t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE); 107 tx_len = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 108 109 r = t4_read_reg(sc, A_TP_PARA_REG2); 110 rx_len = min(rx_len, G_MAXRXDATA(r)); 111 tx_len = min(tx_len, G_MAXRXDATA(r)); 112 113 r = t4_read_reg(sc, A_TP_PARA_REG7); 114 v = min(G_PMMAXXFERLEN0(r), G_PMMAXXFERLEN1(r)); 115 rx_len = min(rx_len, v); 116 tx_len = min(tx_len, v); 117 118 /* 119 * AHS is not supported by the kernel so we'll not account for 120 * it either in our PDU len -> data segment len conversions. 121 */ 122 rx_len -= ISCSI_BHS_SIZE + ISCSI_HEADER_DIGEST_SIZE + 123 ISCSI_DATA_DIGEST_SIZE; 124 tx_len -= ISCSI_BHS_SIZE + ISCSI_HEADER_DIGEST_SIZE + 125 ISCSI_DATA_DIGEST_SIZE; 126 127 /* 128 * DDP can place only 4 pages for a single PDU. A single 129 * request might use larger pages than the smallest page size, 130 * but that cannot be guaranteed. Assume the smallest DDP 131 * page size for this limit. 132 */ 133 rx_len = min(rx_len, 4 * (1U << pr->pr_page_shift[0])); 134 135 if (chip_id(sc) == CHELSIO_T5) { 136 tx_len = min(tx_len, 15360); 137 138 rx_len = rounddown2(rx_len, 512); 139 tx_len = rounddown2(tx_len, 512); 140 } 141 142 *max_tx_data_len = tx_len; 143 *max_rx_data_len = rx_len; 144 } 145 146 /* 147 * Initialize the software state of the iSCSI ULP driver. 148 * 149 * ENXIO means firmware didn't set up something that it was supposed to. 150 */ 151 static int 152 cxgbei_init(struct adapter *sc, struct cxgbei_data *ci) 153 { 154 struct sysctl_oid *oid; 155 struct sysctl_oid_list *children; 156 struct ppod_region *pr; 157 uint32_t r; 158 int rc; 159 160 MPASS(sc->vres.iscsi.size > 0); 161 MPASS(ci != NULL); 162 163 pr = &ci->pr; 164 r = t4_read_reg(sc, A_ULP_RX_ISCSI_PSZ); 165 rc = t4_init_ppod_region(pr, &sc->vres.iscsi, r, "iSCSI page pods"); 166 if (rc != 0) { 167 device_printf(sc->dev, 168 "%s: failed to initialize the iSCSI page pod region: %u.\n", 169 __func__, rc); 170 return (rc); 171 } 172 173 r = t4_read_reg(sc, A_ULP_RX_ISCSI_TAGMASK); 174 r &= V_ISCSITAGMASK(M_ISCSITAGMASK); 175 if (r != pr->pr_tag_mask) { 176 /* 177 * Recent firmwares are supposed to set up the iSCSI tagmask 178 * but we'll do it ourselves it the computed value doesn't match 179 * what's in the register. 180 */ 181 device_printf(sc->dev, 182 "tagmask 0x%08x does not match computed mask 0x%08x.\n", r, 183 pr->pr_tag_mask); 184 t4_set_reg_field(sc, A_ULP_RX_ISCSI_TAGMASK, 185 V_ISCSITAGMASK(M_ISCSITAGMASK), pr->pr_tag_mask); 186 } 187 188 read_pdu_limits(sc, &ci->max_tx_data_len, &ci->max_rx_data_len, pr); 189 190 sysctl_ctx_init(&ci->ctx); 191 oid = device_get_sysctl_tree(sc->dev); /* dev.t5nex.X */ 192 children = SYSCTL_CHILDREN(oid); 193 194 oid = SYSCTL_ADD_NODE(&ci->ctx, children, OID_AUTO, "iscsi", 195 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "iSCSI ULP settings"); 196 children = SYSCTL_CHILDREN(oid); 197 198 ci->ddp_threshold = 2048; 199 SYSCTL_ADD_UINT(&ci->ctx, children, OID_AUTO, "ddp_threshold", 200 CTLFLAG_RW, &ci->ddp_threshold, 0, "Rx zero copy threshold"); 201 202 SYSCTL_ADD_UINT(&ci->ctx, children, OID_AUTO, "max_rx_data_len", 203 CTLFLAG_RD, &ci->max_rx_data_len, 0, 204 "Maximum receive data segment length"); 205 SYSCTL_ADD_UINT(&ci->ctx, children, OID_AUTO, "max_tx_data_len", 206 CTLFLAG_RD, &ci->max_tx_data_len, 0, 207 "Maximum transmit data segment length"); 208 209 return (0); 210 } 211 212 static int 213 do_rx_iscsi_hdr(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 214 { 215 struct adapter *sc = iq->adapter; 216 struct cpl_iscsi_hdr *cpl = mtod(m, struct cpl_iscsi_hdr *); 217 u_int tid = GET_TID(cpl); 218 struct toepcb *toep = lookup_tid(sc, tid); 219 struct icl_pdu *ip; 220 struct icl_cxgbei_pdu *icp; 221 uint16_t len_ddp = be16toh(cpl->pdu_len_ddp); 222 uint16_t len = be16toh(cpl->len); 223 224 M_ASSERTPKTHDR(m); 225 MPASS(m->m_pkthdr.len == len + sizeof(*cpl)); 226 227 ip = icl_cxgbei_new_pdu(M_NOWAIT); 228 if (ip == NULL) 229 CXGBE_UNIMPLEMENTED("PDU allocation failure"); 230 m_copydata(m, sizeof(*cpl), ISCSI_BHS_SIZE, (caddr_t)ip->ip_bhs); 231 ip->ip_data_len = G_ISCSI_PDU_LEN(len_ddp) - len; 232 icp = ip_to_icp(ip); 233 icp->icp_seq = ntohl(cpl->seq); 234 icp->icp_flags = ICPF_RX_HDR; 235 236 /* This is the start of a new PDU. There should be no old state. */ 237 MPASS(toep->ulpcb2 == NULL); 238 toep->ulpcb2 = icp; 239 240 #if 0 241 CTR5(KTR_CXGBE, "%s: tid %u, cpl->len %u, pdu_len_ddp 0x%04x, icp %p", 242 __func__, tid, len, len_ddp, icp); 243 #endif 244 245 m_freem(m); 246 return (0); 247 } 248 249 static int 250 do_rx_iscsi_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 251 { 252 struct adapter *sc = iq->adapter; 253 struct cpl_iscsi_data *cpl = mtod(m, struct cpl_iscsi_data *); 254 u_int tid = GET_TID(cpl); 255 struct toepcb *toep = lookup_tid(sc, tid); 256 struct icl_cxgbei_pdu *icp = toep->ulpcb2; 257 struct icl_pdu *ip; 258 259 M_ASSERTPKTHDR(m); 260 MPASS(m->m_pkthdr.len == be16toh(cpl->len) + sizeof(*cpl)); 261 262 if (icp == NULL) { 263 /* 264 * T6 completion enabled, start of a new pdu. Header 265 * will come in completion CPL. 266 */ 267 ip = icl_cxgbei_new_pdu(M_NOWAIT); 268 if (ip == NULL) 269 CXGBE_UNIMPLEMENTED("PDU allocation failure"); 270 icp = ip_to_icp(ip); 271 } else { 272 /* T5 mode, header is already received. */ 273 MPASS(icp->icp_flags == ICPF_RX_HDR); 274 MPASS(icp->ip.ip_data_mbuf == NULL); 275 MPASS(icp->ip.ip_data_len == m->m_pkthdr.len - sizeof(*cpl)); 276 } 277 278 /* Trim the cpl header from mbuf. */ 279 m_adj(m, sizeof(*cpl)); 280 281 icp->icp_flags |= ICPF_RX_FLBUF; 282 icp->ip.ip_data_mbuf = m; 283 toep->ofld_rxq->rx_iscsi_fl_pdus++; 284 toep->ofld_rxq->rx_iscsi_fl_octets += m->m_pkthdr.len; 285 286 /* 287 * For T6, save the icp for further processing in the 288 * completion handler. 289 */ 290 if (icp->icp_flags == ICPF_RX_FLBUF) { 291 MPASS(toep->ulpcb2 == NULL); 292 toep->ulpcb2 = icp; 293 } 294 295 #if 0 296 CTR4(KTR_CXGBE, "%s: tid %u, cpl->len %u, icp %p", __func__, tid, 297 be16toh(cpl->len), icp); 298 #endif 299 300 return (0); 301 } 302 303 static int 304 mbuf_crc32c_helper(void *arg, void *data, u_int len) 305 { 306 uint32_t *digestp = arg; 307 308 *digestp = calculate_crc32c(*digestp, data, len); 309 return (0); 310 } 311 312 static bool 313 parse_pdus(struct toepcb *toep, struct icl_cxgbei_conn *icc, struct sockbuf *sb) 314 { 315 struct iscsi_bhs bhs; 316 struct mbuf *m; 317 struct icl_pdu *ip; 318 u_int ahs_len, data_len, header_len, pdu_len, total_len; 319 uint32_t calc_digest, wire_digest; 320 321 total_len = sbused(sb); 322 CTR3(KTR_CXGBE, "%s: tid %u, %u bytes in so_rcv", __func__, toep->tid, 323 total_len); 324 325 m = sbcut_locked(sb, total_len); 326 KASSERT(m_length(m, NULL) == total_len, 327 ("sbcut returned less data (%u vs %u)", total_len, 328 m_length(m, NULL))); 329 330 header_len = sizeof(struct iscsi_bhs); 331 if (icc->ic.ic_header_crc32c) 332 header_len += ISCSI_HEADER_DIGEST_SIZE; 333 for (;;) { 334 if (total_len < sizeof(struct iscsi_bhs)) { 335 ICL_WARN("truncated pre-offload PDU with len %u", 336 total_len); 337 m_freem(m); 338 return (false); 339 } 340 m_copydata(m, 0, sizeof(struct iscsi_bhs), (caddr_t)&bhs); 341 342 ahs_len = bhs.bhs_total_ahs_len * 4; 343 data_len = bhs.bhs_data_segment_len[0] << 16 | 344 bhs.bhs_data_segment_len[1] << 8 | 345 bhs.bhs_data_segment_len[0]; 346 pdu_len = header_len + ahs_len + roundup2(data_len, 4); 347 if (icc->ic.ic_data_crc32c && data_len != 0) 348 pdu_len += ISCSI_DATA_DIGEST_SIZE; 349 350 if (total_len < pdu_len) { 351 ICL_WARN("truncated pre-offload PDU len %u vs %u", 352 total_len, pdu_len); 353 m_freem(m); 354 return (false); 355 } 356 357 if (ahs_len != 0) { 358 ICL_WARN("received pre-offload PDU with AHS"); 359 m_freem(m); 360 return (false); 361 } 362 363 if (icc->ic.ic_header_crc32c) { 364 m_copydata(m, sizeof(struct iscsi_bhs), 365 sizeof(wire_digest), (caddr_t)&wire_digest); 366 367 calc_digest = calculate_crc32c(0xffffffff, 368 (caddr_t)&bhs, sizeof(bhs)); 369 calc_digest ^= 0xffffffff; 370 if (calc_digest != wire_digest) { 371 ICL_WARN("received pre-offload PDU 0x%02x " 372 "with invalid header digest (0x%x vs 0x%x)", 373 bhs.bhs_opcode, wire_digest, calc_digest); 374 toep->ofld_rxq->rx_iscsi_header_digest_errors++; 375 m_free(m); 376 return (false); 377 } 378 } 379 380 m_adj(m, header_len); 381 382 if (icc->ic.ic_data_crc32c && data_len != 0) { 383 m_copydata(m, data_len, sizeof(wire_digest), 384 (caddr_t)&wire_digest); 385 386 calc_digest = 0xffffffff; 387 m_apply(m, 0, roundup2(data_len, 4), mbuf_crc32c_helper, 388 &calc_digest); 389 calc_digest ^= 0xffffffff; 390 if (calc_digest != wire_digest) { 391 ICL_WARN("received pre-offload PDU 0x%02x " 392 "with invalid data digest (0x%x vs 0x%x)", 393 bhs.bhs_opcode, wire_digest, calc_digest); 394 toep->ofld_rxq->rx_iscsi_data_digest_errors++; 395 m_free(m); 396 return (false); 397 } 398 } 399 400 ip = icl_cxgbei_new_pdu(M_NOWAIT); 401 if (ip == NULL) 402 CXGBE_UNIMPLEMENTED("PDU allocation failure"); 403 icl_cxgbei_new_pdu_set_conn(ip, &icc->ic); 404 *ip->ip_bhs = bhs; 405 ip->ip_data_len = data_len; 406 if (data_len != 0) 407 ip->ip_data_mbuf = m; 408 409 STAILQ_INSERT_TAIL(&icc->rcvd_pdus, ip, ip_next); 410 411 total_len -= pdu_len; 412 if (total_len == 0) { 413 if (data_len == 0) 414 m_freem(m); 415 return (true); 416 } 417 418 if (data_len != 0) { 419 m = m_split(m, roundup2(data_len, 4), M_NOWAIT); 420 if (m == NULL) { 421 ICL_WARN("failed to split mbuf chain for " 422 "pre-offload PDU"); 423 424 /* Don't free the mbuf chain as 'ip' owns it. */ 425 return (false); 426 } 427 if (icc->ic.ic_data_crc32c) 428 m_adj(m, ISCSI_DATA_DIGEST_SIZE); 429 } 430 } 431 } 432 433 static int 434 do_rx_iscsi_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 435 { 436 struct adapter *sc = iq->adapter; 437 const struct cpl_rx_data_ddp *cpl = (const void *)(rss + 1); 438 u_int tid = GET_TID(cpl); 439 struct toepcb *toep = lookup_tid(sc, tid); 440 struct inpcb *inp = toep->inp; 441 struct socket *so; 442 struct sockbuf *sb; 443 struct tcpcb *tp; 444 struct icl_cxgbei_conn *icc; 445 struct icl_conn *ic; 446 struct icl_cxgbei_pdu *icp = toep->ulpcb2; 447 struct icl_pdu *ip; 448 u_int pdu_len, val; 449 struct epoch_tracker et; 450 451 MPASS(m == NULL); 452 453 /* Must already be assembling a PDU. */ 454 MPASS(icp != NULL); 455 MPASS(icp->icp_flags & ICPF_RX_HDR); /* Data is optional. */ 456 MPASS((icp->icp_flags & ICPF_RX_STATUS) == 0); 457 458 pdu_len = be16toh(cpl->len); /* includes everything. */ 459 val = be32toh(cpl->ddpvld); 460 461 #if 0 462 CTR5(KTR_CXGBE, 463 "%s: tid %u, cpl->len %u, ddpvld 0x%08x, icp_flags 0x%08x", 464 __func__, tid, pdu_len, val, icp->icp_flags); 465 #endif 466 467 icp->icp_flags |= ICPF_RX_STATUS; 468 ip = &icp->ip; 469 if (val & F_DDP_PADDING_ERR) { 470 ICL_WARN("received PDU 0x%02x with invalid padding", 471 ip->ip_bhs->bhs_opcode); 472 toep->ofld_rxq->rx_iscsi_padding_errors++; 473 } 474 if (val & F_DDP_HDRCRC_ERR) { 475 ICL_WARN("received PDU 0x%02x with invalid header digest", 476 ip->ip_bhs->bhs_opcode); 477 toep->ofld_rxq->rx_iscsi_header_digest_errors++; 478 } 479 if (val & F_DDP_DATACRC_ERR) { 480 ICL_WARN("received PDU 0x%02x with invalid data digest", 481 ip->ip_bhs->bhs_opcode); 482 toep->ofld_rxq->rx_iscsi_data_digest_errors++; 483 } 484 if (val & F_DDP_PDU && ip->ip_data_mbuf == NULL) { 485 MPASS((icp->icp_flags & ICPF_RX_FLBUF) == 0); 486 MPASS(ip->ip_data_len > 0); 487 icp->icp_flags |= ICPF_RX_DDP; 488 toep->ofld_rxq->rx_iscsi_ddp_pdus++; 489 toep->ofld_rxq->rx_iscsi_ddp_octets += ip->ip_data_len; 490 } 491 492 INP_WLOCK(inp); 493 if (__predict_false(inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT))) { 494 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x", 495 __func__, tid, pdu_len, inp->inp_flags); 496 INP_WUNLOCK(inp); 497 icl_cxgbei_conn_pdu_free(NULL, ip); 498 toep->ulpcb2 = NULL; 499 return (0); 500 } 501 502 /* 503 * T6+ does not report data PDUs received via DDP without F 504 * set. This can result in gaps in the TCP sequence space. 505 */ 506 tp = intotcpcb(inp); 507 MPASS(chip_id(sc) >= CHELSIO_T6 || icp->icp_seq == tp->rcv_nxt); 508 tp->rcv_nxt = icp->icp_seq + pdu_len; 509 tp->t_rcvtime = ticks; 510 511 /* 512 * Don't update the window size or return credits since RX 513 * flow control is disabled. 514 */ 515 516 so = inp->inp_socket; 517 sb = &so->so_rcv; 518 SOCKBUF_LOCK(sb); 519 520 icc = toep->ulpcb; 521 if (__predict_false(icc == NULL || sb->sb_state & SBS_CANTRCVMORE)) { 522 CTR5(KTR_CXGBE, 523 "%s: tid %u, excess rx (%d bytes), icc %p, sb_state 0x%x", 524 __func__, tid, pdu_len, icc, sb->sb_state); 525 SOCKBUF_UNLOCK(sb); 526 INP_WUNLOCK(inp); 527 528 CURVNET_SET(so->so_vnet); 529 NET_EPOCH_ENTER(et); 530 INP_WLOCK(inp); 531 tp = tcp_drop(tp, ECONNRESET); 532 if (tp) 533 INP_WUNLOCK(inp); 534 NET_EPOCH_EXIT(et); 535 CURVNET_RESTORE(); 536 537 icl_cxgbei_conn_pdu_free(NULL, ip); 538 toep->ulpcb2 = NULL; 539 return (0); 540 } 541 MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE); 542 ic = &icc->ic; 543 if ((val & (F_DDP_PADDING_ERR | F_DDP_HDRCRC_ERR | 544 F_DDP_DATACRC_ERR)) != 0) { 545 SOCKBUF_UNLOCK(sb); 546 INP_WUNLOCK(inp); 547 548 icl_cxgbei_conn_pdu_free(NULL, ip); 549 toep->ulpcb2 = NULL; 550 ic->ic_error(ic); 551 return (0); 552 } 553 554 if (__predict_false(sbused(sb)) != 0) { 555 /* 556 * PDUs were received before the tid transitioned to ULP mode. 557 * Convert them to icl_cxgbei_pdus and send them to ICL before 558 * the PDU in icp/ip. 559 */ 560 if (!parse_pdus(toep, icc, sb)) { 561 SOCKBUF_UNLOCK(sb); 562 INP_WUNLOCK(inp); 563 564 icl_cxgbei_conn_pdu_free(NULL, ip); 565 toep->ulpcb2 = NULL; 566 ic->ic_error(ic); 567 return (0); 568 } 569 } 570 icl_cxgbei_new_pdu_set_conn(ip, ic); 571 572 STAILQ_INSERT_TAIL(&icc->rcvd_pdus, ip, ip_next); 573 if ((icc->rx_flags & RXF_ACTIVE) == 0) { 574 struct cxgbei_worker_thread_softc *cwt = &cwt_softc[icc->cwt]; 575 576 mtx_lock(&cwt->cwt_lock); 577 icc->rx_flags |= RXF_ACTIVE; 578 TAILQ_INSERT_TAIL(&cwt->rx_head, icc, rx_link); 579 if (cwt->cwt_state == CWT_SLEEPING) { 580 cwt->cwt_state = CWT_RUNNING; 581 cv_signal(&cwt->cwt_cv); 582 } 583 mtx_unlock(&cwt->cwt_lock); 584 } 585 SOCKBUF_UNLOCK(sb); 586 INP_WUNLOCK(inp); 587 588 toep->ulpcb2 = NULL; 589 590 return (0); 591 } 592 593 static int 594 do_rx_iscsi_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 595 { 596 struct epoch_tracker et; 597 struct adapter *sc = iq->adapter; 598 struct cpl_rx_iscsi_cmp *cpl = mtod(m, struct cpl_rx_iscsi_cmp *); 599 u_int tid = GET_TID(cpl); 600 struct toepcb *toep = lookup_tid(sc, tid); 601 struct icl_cxgbei_pdu *icp = toep->ulpcb2; 602 struct icl_pdu *ip; 603 struct cxgbei_cmp *cmp; 604 struct inpcb *inp = toep->inp; 605 #ifdef INVARIANTS 606 uint16_t len = be16toh(cpl->len); 607 u_int data_digest_len; 608 #endif 609 struct socket *so; 610 struct sockbuf *sb; 611 struct tcpcb *tp; 612 struct icl_cxgbei_conn *icc; 613 struct icl_conn *ic; 614 struct iscsi_bhs_data_out *bhsdo; 615 u_int val = be32toh(cpl->ddpvld); 616 u_int npdus, pdu_len; 617 uint32_t prev_seg_len; 618 619 M_ASSERTPKTHDR(m); 620 MPASS(m->m_pkthdr.len == len + sizeof(*cpl)); 621 622 if ((val & F_DDP_PDU) == 0) { 623 MPASS(icp != NULL); 624 MPASS((icp->icp_flags & ICPF_RX_STATUS) == 0); 625 ip = &icp->ip; 626 } 627 628 if (icp == NULL) { 629 /* T6 completion enabled, start of a new PDU. */ 630 ip = icl_cxgbei_new_pdu(M_NOWAIT); 631 if (ip == NULL) 632 CXGBE_UNIMPLEMENTED("PDU allocation failure"); 633 icp = ip_to_icp(ip); 634 } 635 pdu_len = G_ISCSI_PDU_LEN(be16toh(cpl->pdu_len_ddp)); 636 637 #if 0 638 CTR5(KTR_CXGBE, 639 "%s: tid %u, cpl->len %u, ddpvld 0x%08x, icp %p", 640 __func__, tid, pdu_len, val, icp); 641 #endif 642 643 /* Copy header */ 644 m_copydata(m, sizeof(*cpl), ISCSI_BHS_SIZE, (caddr_t)ip->ip_bhs); 645 bhsdo = (struct iscsi_bhs_data_out *)ip->ip_bhs; 646 ip->ip_data_len = bhsdo->bhsdo_data_segment_len[0] << 16 | 647 bhsdo->bhsdo_data_segment_len[1] << 8 | 648 bhsdo->bhsdo_data_segment_len[2]; 649 icp->icp_seq = ntohl(cpl->seq); 650 icp->icp_flags |= ICPF_RX_HDR; 651 icp->icp_flags |= ICPF_RX_STATUS; 652 653 if (val & F_DDP_PADDING_ERR) { 654 ICL_WARN("received PDU 0x%02x with invalid padding", 655 ip->ip_bhs->bhs_opcode); 656 toep->ofld_rxq->rx_iscsi_padding_errors++; 657 } 658 if (val & F_DDP_HDRCRC_ERR) { 659 ICL_WARN("received PDU 0x%02x with invalid header digest", 660 ip->ip_bhs->bhs_opcode); 661 toep->ofld_rxq->rx_iscsi_header_digest_errors++; 662 } 663 if (val & F_DDP_DATACRC_ERR) { 664 ICL_WARN("received PDU 0x%02x with invalid data digest", 665 ip->ip_bhs->bhs_opcode); 666 toep->ofld_rxq->rx_iscsi_data_digest_errors++; 667 } 668 669 INP_WLOCK(inp); 670 if (__predict_false(inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT))) { 671 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x", 672 __func__, tid, pdu_len, inp->inp_flags); 673 INP_WUNLOCK(inp); 674 icl_cxgbei_conn_pdu_free(NULL, ip); 675 toep->ulpcb2 = NULL; 676 m_freem(m); 677 return (0); 678 } 679 680 tp = intotcpcb(inp); 681 682 /* 683 * If icc is NULL, the connection is being closed in 684 * icl_cxgbei_conn_close(), just drop this data. 685 */ 686 icc = toep->ulpcb; 687 if (__predict_false(icc == NULL)) { 688 CTR4(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes), icc %p", 689 __func__, tid, pdu_len, icc); 690 691 /* 692 * Update rcv_nxt so the sequence number of the FIN 693 * doesn't appear wrong. 694 */ 695 tp->rcv_nxt = icp->icp_seq + pdu_len; 696 tp->t_rcvtime = ticks; 697 INP_WUNLOCK(inp); 698 699 icl_cxgbei_conn_pdu_free(NULL, ip); 700 toep->ulpcb2 = NULL; 701 m_freem(m); 702 return (0); 703 } 704 705 MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE); 706 ic = &icc->ic; 707 if ((val & (F_DDP_PADDING_ERR | F_DDP_HDRCRC_ERR | 708 F_DDP_DATACRC_ERR)) != 0) { 709 INP_WUNLOCK(inp); 710 711 icl_cxgbei_conn_pdu_free(NULL, ip); 712 toep->ulpcb2 = NULL; 713 m_freem(m); 714 ic->ic_error(ic); 715 return (0); 716 } 717 718 #ifdef INVARIANTS 719 data_digest_len = (icc->ulp_submode & ULP_CRC_DATA) ? 720 ISCSI_DATA_DIGEST_SIZE : 0; 721 MPASS(roundup2(ip->ip_data_len, 4) == pdu_len - len - data_digest_len); 722 #endif 723 724 if (val & F_DDP_PDU && ip->ip_data_mbuf == NULL) { 725 MPASS((icp->icp_flags & ICPF_RX_FLBUF) == 0); 726 MPASS(ip->ip_data_len > 0); 727 icp->icp_flags |= ICPF_RX_DDP; 728 bhsdo = (struct iscsi_bhs_data_out *)ip->ip_bhs; 729 730 switch (ip->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) { 731 case ISCSI_BHS_OPCODE_SCSI_DATA_IN: 732 cmp = cxgbei_find_cmp(icc, 733 be32toh(bhsdo->bhsdo_initiator_task_tag)); 734 break; 735 case ISCSI_BHS_OPCODE_SCSI_DATA_OUT: 736 cmp = cxgbei_find_cmp(icc, 737 be32toh(bhsdo->bhsdo_target_transfer_tag)); 738 break; 739 default: 740 __assert_unreachable(); 741 } 742 MPASS(cmp != NULL); 743 744 /* 745 * The difference between the end of the last burst 746 * and the offset of the last PDU in this burst is 747 * the additional data received via DDP. 748 */ 749 prev_seg_len = be32toh(bhsdo->bhsdo_buffer_offset) - 750 cmp->next_buffer_offset; 751 752 if (prev_seg_len != 0) { 753 uint32_t orig_datasn; 754 755 /* 756 * Return a "large" PDU representing the burst 757 * of PDUs. Adjust the offset and length of 758 * this PDU to represent the entire burst. 759 */ 760 ip->ip_data_len += prev_seg_len; 761 bhsdo->bhsdo_data_segment_len[2] = ip->ip_data_len; 762 bhsdo->bhsdo_data_segment_len[1] = ip->ip_data_len >> 8; 763 bhsdo->bhsdo_data_segment_len[0] = ip->ip_data_len >> 16; 764 bhsdo->bhsdo_buffer_offset = 765 htobe32(cmp->next_buffer_offset); 766 767 orig_datasn = htobe32(bhsdo->bhsdo_datasn); 768 npdus = orig_datasn - cmp->last_datasn; 769 bhsdo->bhsdo_datasn = htobe32(cmp->last_datasn + 1); 770 cmp->last_datasn = orig_datasn; 771 ip->ip_additional_pdus = npdus - 1; 772 } else { 773 MPASS(htobe32(bhsdo->bhsdo_datasn) == 774 cmp->last_datasn + 1); 775 npdus = 1; 776 cmp->last_datasn = htobe32(bhsdo->bhsdo_datasn); 777 } 778 779 cmp->next_buffer_offset += ip->ip_data_len; 780 toep->ofld_rxq->rx_iscsi_ddp_pdus += npdus; 781 toep->ofld_rxq->rx_iscsi_ddp_octets += ip->ip_data_len; 782 } else { 783 MPASS(icp->icp_flags & (ICPF_RX_FLBUF)); 784 MPASS(ip->ip_data_len == ip->ip_data_mbuf->m_pkthdr.len); 785 } 786 787 tp->rcv_nxt = icp->icp_seq + pdu_len; 788 tp->t_rcvtime = ticks; 789 790 /* 791 * Don't update the window size or return credits since RX 792 * flow control is disabled. 793 */ 794 795 so = inp->inp_socket; 796 sb = &so->so_rcv; 797 SOCKBUF_LOCK(sb); 798 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) { 799 CTR5(KTR_CXGBE, 800 "%s: tid %u, excess rx (%d bytes), icc %p, sb_state 0x%x", 801 __func__, tid, pdu_len, icc, sb->sb_state); 802 SOCKBUF_UNLOCK(sb); 803 INP_WUNLOCK(inp); 804 805 CURVNET_SET(so->so_vnet); 806 NET_EPOCH_ENTER(et); 807 INP_WLOCK(inp); 808 tp = tcp_drop(tp, ECONNRESET); 809 if (tp != NULL) 810 INP_WUNLOCK(inp); 811 NET_EPOCH_EXIT(et); 812 CURVNET_RESTORE(); 813 814 icl_cxgbei_conn_pdu_free(NULL, ip); 815 toep->ulpcb2 = NULL; 816 m_freem(m); 817 return (0); 818 } 819 820 if (__predict_false(sbused(sb)) != 0) { 821 /* 822 * PDUs were received before the tid transitioned to ULP mode. 823 * Convert them to icl_cxgbei_pdus and send them to ICL before 824 * the PDU in icp/ip. 825 */ 826 if (!parse_pdus(toep, icc, sb)) { 827 SOCKBUF_UNLOCK(sb); 828 INP_WUNLOCK(inp); 829 830 icl_cxgbei_conn_pdu_free(NULL, ip); 831 toep->ulpcb2 = NULL; 832 ic->ic_error(ic); 833 return (0); 834 } 835 } 836 icl_cxgbei_new_pdu_set_conn(ip, ic); 837 838 /* Enqueue the PDU to the received pdus queue. */ 839 STAILQ_INSERT_TAIL(&icc->rcvd_pdus, ip, ip_next); 840 if ((icc->rx_flags & RXF_ACTIVE) == 0) { 841 struct cxgbei_worker_thread_softc *cwt = &cwt_softc[icc->cwt]; 842 843 mtx_lock(&cwt->cwt_lock); 844 icc->rx_flags |= RXF_ACTIVE; 845 TAILQ_INSERT_TAIL(&cwt->rx_head, icc, rx_link); 846 if (cwt->cwt_state == CWT_SLEEPING) { 847 cwt->cwt_state = CWT_RUNNING; 848 cv_signal(&cwt->cwt_cv); 849 } 850 mtx_unlock(&cwt->cwt_lock); 851 } 852 SOCKBUF_UNLOCK(sb); 853 INP_WUNLOCK(inp); 854 855 toep->ulpcb2 = NULL; 856 m_freem(m); 857 858 return (0); 859 } 860 861 static int 862 cxgbei_activate(struct adapter *sc) 863 { 864 struct cxgbei_data *ci; 865 int rc; 866 867 ASSERT_SYNCHRONIZED_OP(sc); 868 869 if (uld_active(sc, ULD_ISCSI)) { 870 KASSERT(0, ("%s: iSCSI offload already enabled on adapter %p", 871 __func__, sc)); 872 return (0); 873 } 874 875 if (sc->iscsicaps == 0 || sc->vres.iscsi.size == 0) { 876 device_printf(sc->dev, 877 "not iSCSI offload capable, or capability disabled.\n"); 878 return (ENOSYS); 879 } 880 881 /* per-adapter softc for iSCSI */ 882 ci = malloc(sizeof(*ci), M_CXGBE, M_ZERO | M_WAITOK); 883 if (ci == NULL) 884 return (ENOMEM); 885 886 rc = cxgbei_init(sc, ci); 887 if (rc != 0) { 888 free(ci, M_CXGBE); 889 return (rc); 890 } 891 892 sc->iscsi_ulp_softc = ci; 893 894 return (0); 895 } 896 897 static int 898 cxgbei_deactivate(struct adapter *sc) 899 { 900 struct cxgbei_data *ci = sc->iscsi_ulp_softc; 901 902 ASSERT_SYNCHRONIZED_OP(sc); 903 904 if (ci != NULL) { 905 sysctl_ctx_free(&ci->ctx); 906 t4_free_ppod_region(&ci->pr); 907 free(ci, M_CXGBE); 908 sc->iscsi_ulp_softc = NULL; 909 } 910 911 return (0); 912 } 913 914 static void 915 cxgbei_activate_all(struct adapter *sc, void *arg __unused) 916 { 917 918 if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4isact") != 0) 919 return; 920 921 /* Activate iSCSI if any port on this adapter has IFCAP_TOE enabled. */ 922 if (sc->offload_map && !uld_active(sc, ULD_ISCSI)) 923 (void) t4_activate_uld(sc, ULD_ISCSI); 924 925 end_synchronized_op(sc, 0); 926 } 927 928 static void 929 cxgbei_deactivate_all(struct adapter *sc, void *arg __unused) 930 { 931 932 if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4isdea") != 0) 933 return; 934 935 if (uld_active(sc, ULD_ISCSI)) 936 (void) t4_deactivate_uld(sc, ULD_ISCSI); 937 938 end_synchronized_op(sc, 0); 939 } 940 941 static struct uld_info cxgbei_uld_info = { 942 .uld_id = ULD_ISCSI, 943 .activate = cxgbei_activate, 944 .deactivate = cxgbei_deactivate, 945 }; 946 947 static void 948 cwt_main(void *arg) 949 { 950 struct cxgbei_worker_thread_softc *cwt = arg; 951 struct icl_cxgbei_conn *icc = NULL; 952 struct icl_conn *ic; 953 struct icl_pdu *ip; 954 struct sockbuf *sb; 955 STAILQ_HEAD(, icl_pdu) rx_pdus = STAILQ_HEAD_INITIALIZER(rx_pdus); 956 957 MPASS(cwt != NULL); 958 959 mtx_lock(&cwt->cwt_lock); 960 MPASS(cwt->cwt_state == 0); 961 cwt->cwt_state = CWT_RUNNING; 962 cv_signal(&cwt->cwt_cv); 963 964 while (__predict_true(cwt->cwt_state != CWT_STOP)) { 965 cwt->cwt_state = CWT_RUNNING; 966 while ((icc = TAILQ_FIRST(&cwt->rx_head)) != NULL) { 967 TAILQ_REMOVE(&cwt->rx_head, icc, rx_link); 968 mtx_unlock(&cwt->cwt_lock); 969 970 ic = &icc->ic; 971 sb = &ic->ic_socket->so_rcv; 972 973 SOCKBUF_LOCK(sb); 974 MPASS(icc->rx_flags & RXF_ACTIVE); 975 if (__predict_true(!(sb->sb_state & SBS_CANTRCVMORE))) { 976 MPASS(STAILQ_EMPTY(&rx_pdus)); 977 STAILQ_SWAP(&icc->rcvd_pdus, &rx_pdus, icl_pdu); 978 SOCKBUF_UNLOCK(sb); 979 980 /* Hand over PDUs to ICL. */ 981 while ((ip = STAILQ_FIRST(&rx_pdus)) != NULL) { 982 STAILQ_REMOVE_HEAD(&rx_pdus, ip_next); 983 ic->ic_receive(ip); 984 } 985 986 SOCKBUF_LOCK(sb); 987 MPASS(STAILQ_EMPTY(&rx_pdus)); 988 } 989 MPASS(icc->rx_flags & RXF_ACTIVE); 990 if (STAILQ_EMPTY(&icc->rcvd_pdus) || 991 __predict_false(sb->sb_state & SBS_CANTRCVMORE)) { 992 icc->rx_flags &= ~RXF_ACTIVE; 993 } else { 994 /* 995 * More PDUs were received while we were busy 996 * handing over the previous batch to ICL. 997 * Re-add this connection to the end of the 998 * queue. 999 */ 1000 mtx_lock(&cwt->cwt_lock); 1001 TAILQ_INSERT_TAIL(&cwt->rx_head, icc, 1002 rx_link); 1003 mtx_unlock(&cwt->cwt_lock); 1004 } 1005 SOCKBUF_UNLOCK(sb); 1006 1007 mtx_lock(&cwt->cwt_lock); 1008 } 1009 1010 /* Inner loop doesn't check for CWT_STOP, do that first. */ 1011 if (__predict_false(cwt->cwt_state == CWT_STOP)) 1012 break; 1013 cwt->cwt_state = CWT_SLEEPING; 1014 cv_wait(&cwt->cwt_cv, &cwt->cwt_lock); 1015 } 1016 1017 MPASS(TAILQ_FIRST(&cwt->rx_head) == NULL); 1018 mtx_assert(&cwt->cwt_lock, MA_OWNED); 1019 cwt->cwt_state = CWT_STOPPED; 1020 cv_signal(&cwt->cwt_cv); 1021 mtx_unlock(&cwt->cwt_lock); 1022 kthread_exit(); 1023 } 1024 1025 static int 1026 start_worker_threads(void) 1027 { 1028 int i, rc; 1029 struct cxgbei_worker_thread_softc *cwt; 1030 1031 worker_thread_count = min(mp_ncpus, 32); 1032 cwt_softc = malloc(worker_thread_count * sizeof(*cwt), M_CXGBE, 1033 M_WAITOK | M_ZERO); 1034 1035 MPASS(cxgbei_proc == NULL); 1036 for (i = 0, cwt = &cwt_softc[0]; i < worker_thread_count; i++, cwt++) { 1037 mtx_init(&cwt->cwt_lock, "cwt lock", NULL, MTX_DEF); 1038 cv_init(&cwt->cwt_cv, "cwt cv"); 1039 TAILQ_INIT(&cwt->rx_head); 1040 rc = kproc_kthread_add(cwt_main, cwt, &cxgbei_proc, NULL, 0, 0, 1041 "cxgbei", "%d", i); 1042 if (rc != 0) { 1043 printf("cxgbei: failed to start thread #%d/%d (%d)\n", 1044 i + 1, worker_thread_count, rc); 1045 mtx_destroy(&cwt->cwt_lock); 1046 cv_destroy(&cwt->cwt_cv); 1047 bzero(cwt, sizeof(*cwt)); 1048 if (i == 0) { 1049 free(cwt_softc, M_CXGBE); 1050 worker_thread_count = 0; 1051 1052 return (rc); 1053 } 1054 1055 /* Not fatal, carry on with fewer threads. */ 1056 worker_thread_count = i; 1057 rc = 0; 1058 break; 1059 } 1060 1061 /* Wait for thread to start before moving on to the next one. */ 1062 mtx_lock(&cwt->cwt_lock); 1063 while (cwt->cwt_state == 0) 1064 cv_wait(&cwt->cwt_cv, &cwt->cwt_lock); 1065 mtx_unlock(&cwt->cwt_lock); 1066 } 1067 1068 MPASS(cwt_softc != NULL); 1069 MPASS(worker_thread_count > 0); 1070 return (0); 1071 } 1072 1073 static void 1074 stop_worker_threads(void) 1075 { 1076 int i; 1077 struct cxgbei_worker_thread_softc *cwt = &cwt_softc[0]; 1078 1079 MPASS(worker_thread_count >= 0); 1080 1081 for (i = 0, cwt = &cwt_softc[0]; i < worker_thread_count; i++, cwt++) { 1082 mtx_lock(&cwt->cwt_lock); 1083 MPASS(cwt->cwt_state == CWT_RUNNING || 1084 cwt->cwt_state == CWT_SLEEPING); 1085 cwt->cwt_state = CWT_STOP; 1086 cv_signal(&cwt->cwt_cv); 1087 do { 1088 cv_wait(&cwt->cwt_cv, &cwt->cwt_lock); 1089 } while (cwt->cwt_state != CWT_STOPPED); 1090 mtx_unlock(&cwt->cwt_lock); 1091 mtx_destroy(&cwt->cwt_lock); 1092 cv_destroy(&cwt->cwt_cv); 1093 } 1094 free(cwt_softc, M_CXGBE); 1095 } 1096 1097 /* Select a worker thread for a connection. */ 1098 u_int 1099 cxgbei_select_worker_thread(struct icl_cxgbei_conn *icc) 1100 { 1101 struct adapter *sc = icc->sc; 1102 struct toepcb *toep = icc->toep; 1103 u_int i, n; 1104 1105 n = worker_thread_count / sc->sge.nofldrxq; 1106 if (n > 0) 1107 i = toep->vi->pi->port_id * n + arc4random() % n; 1108 else 1109 i = arc4random() % worker_thread_count; 1110 1111 CTR3(KTR_CXGBE, "%s: tid %u, cwt %u", __func__, toep->tid, i); 1112 1113 return (i); 1114 } 1115 1116 static int 1117 cxgbei_mod_load(void) 1118 { 1119 int rc; 1120 1121 t4_register_cpl_handler(CPL_ISCSI_HDR, do_rx_iscsi_hdr); 1122 t4_register_cpl_handler(CPL_ISCSI_DATA, do_rx_iscsi_data); 1123 t4_register_cpl_handler(CPL_RX_ISCSI_DDP, do_rx_iscsi_ddp); 1124 t4_register_cpl_handler(CPL_RX_ISCSI_CMP, do_rx_iscsi_cmp); 1125 1126 rc = start_worker_threads(); 1127 if (rc != 0) 1128 return (rc); 1129 1130 rc = t4_register_uld(&cxgbei_uld_info); 1131 if (rc != 0) { 1132 stop_worker_threads(); 1133 return (rc); 1134 } 1135 1136 t4_iterate(cxgbei_activate_all, NULL); 1137 1138 return (rc); 1139 } 1140 1141 static int 1142 cxgbei_mod_unload(void) 1143 { 1144 1145 t4_iterate(cxgbei_deactivate_all, NULL); 1146 1147 if (t4_unregister_uld(&cxgbei_uld_info) == EBUSY) 1148 return (EBUSY); 1149 1150 stop_worker_threads(); 1151 1152 t4_register_cpl_handler(CPL_ISCSI_HDR, NULL); 1153 t4_register_cpl_handler(CPL_ISCSI_DATA, NULL); 1154 t4_register_cpl_handler(CPL_RX_ISCSI_DDP, NULL); 1155 t4_register_cpl_handler(CPL_RX_ISCSI_CMP, NULL); 1156 1157 return (0); 1158 } 1159 #endif 1160 1161 static int 1162 cxgbei_modevent(module_t mod, int cmd, void *arg) 1163 { 1164 int rc = 0; 1165 1166 #ifdef TCP_OFFLOAD 1167 switch (cmd) { 1168 case MOD_LOAD: 1169 rc = cxgbei_mod_load(); 1170 if (rc == 0) 1171 rc = icl_cxgbei_mod_load(); 1172 break; 1173 1174 case MOD_UNLOAD: 1175 rc = icl_cxgbei_mod_unload(); 1176 if (rc == 0) 1177 rc = cxgbei_mod_unload(); 1178 break; 1179 1180 default: 1181 rc = EINVAL; 1182 } 1183 #else 1184 printf("cxgbei: compiled without TCP_OFFLOAD support.\n"); 1185 rc = EOPNOTSUPP; 1186 #endif 1187 1188 return (rc); 1189 } 1190 1191 static moduledata_t cxgbei_mod = { 1192 "cxgbei", 1193 cxgbei_modevent, 1194 NULL, 1195 }; 1196 1197 MODULE_VERSION(cxgbei, 1); 1198 DECLARE_MODULE(cxgbei, cxgbei_mod, SI_SUB_EXEC, SI_ORDER_ANY); 1199 MODULE_DEPEND(cxgbei, t4_tom, 1, 1, 1); 1200 MODULE_DEPEND(cxgbei, cxgbe, 1, 1, 1); 1201 MODULE_DEPEND(cxgbei, icl, 1, 1, 1); 1202