1 /* 2 * Copyright (c) 2009-2013, 2016 Chelsio, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include "opt_inet.h" 36 37 #ifdef TCP_OFFLOAD 38 #include <sys/types.h> 39 #include <sys/malloc.h> 40 #include <sys/socket.h> 41 #include <sys/socketvar.h> 42 #include <sys/sockio.h> 43 #include <sys/taskqueue.h> 44 #include <netinet/in.h> 45 #include <net/route.h> 46 47 #include <netinet/in_systm.h> 48 #include <netinet/in_pcb.h> 49 #include <netinet/ip.h> 50 #include <netinet/in_fib.h> 51 #include <netinet/ip_var.h> 52 #include <netinet/tcp_var.h> 53 #include <netinet/tcp.h> 54 #include <netinet/tcpip.h> 55 56 #include <netinet/toecore.h> 57 58 struct sge_iq; 59 struct rss_header; 60 struct cpl_set_tcb_rpl; 61 #include <linux/types.h> 62 #include "offload.h" 63 #include "tom/t4_tom.h" 64 65 #define TOEPCB(so) ((struct toepcb *)(so_sototcpcb((so))->t_toe)) 66 67 #include "iw_cxgbe.h" 68 #include <linux/module.h> 69 #include <linux/workqueue.h> 70 #include <linux/notifier.h> 71 #include <linux/inetdevice.h> 72 #include <linux/if_vlan.h> 73 #include <net/netevent.h> 74 75 static spinlock_t req_lock; 76 static TAILQ_HEAD(c4iw_ep_list, c4iw_ep_common) req_list; 77 static struct work_struct c4iw_task; 78 static struct workqueue_struct *c4iw_taskq; 79 static LIST_HEAD(err_cqe_list); 80 static spinlock_t err_cqe_lock; 81 82 static void process_req(struct work_struct *ctx); 83 static void start_ep_timer(struct c4iw_ep *ep); 84 static int stop_ep_timer(struct c4iw_ep *ep); 85 static int set_tcpinfo(struct c4iw_ep *ep); 86 static void process_timeout(struct c4iw_ep *ep); 87 static void process_err_cqes(void); 88 static enum c4iw_ep_state state_read(struct c4iw_ep_common *epc); 89 static void __state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state tostate); 90 static void state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state tostate); 91 static void *alloc_ep(int size, gfp_t flags); 92 static int find_route(__be32 local_ip, __be32 peer_ip, __be16 local_port, 93 __be16 peer_port, u8 tos, struct nhop4_extended *pnh4); 94 static void close_socket(struct socket *so); 95 static int send_mpa_req(struct c4iw_ep *ep); 96 static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen); 97 static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen); 98 static void close_complete_upcall(struct c4iw_ep *ep, int status); 99 static int send_abort(struct c4iw_ep *ep); 100 static void peer_close_upcall(struct c4iw_ep *ep); 101 static void peer_abort_upcall(struct c4iw_ep *ep); 102 static void connect_reply_upcall(struct c4iw_ep *ep, int status); 103 static int connect_request_upcall(struct c4iw_ep *ep); 104 static void established_upcall(struct c4iw_ep *ep); 105 static int process_mpa_reply(struct c4iw_ep *ep); 106 static int process_mpa_request(struct c4iw_ep *ep); 107 static void process_peer_close(struct c4iw_ep *ep); 108 static void process_conn_error(struct c4iw_ep *ep); 109 static void process_close_complete(struct c4iw_ep *ep); 110 static void ep_timeout(unsigned long arg); 111 static void init_iwarp_socket(struct socket *so, void *arg); 112 static void uninit_iwarp_socket(struct socket *so); 113 static void process_data(struct c4iw_ep *ep); 114 static void process_connected(struct c4iw_ep *ep); 115 static int c4iw_so_upcall(struct socket *so, void *arg, int waitflag); 116 static void process_socket_event(struct c4iw_ep *ep); 117 static void release_ep_resources(struct c4iw_ep *ep); 118 static int process_terminate(struct c4iw_ep *ep); 119 static int terminate(struct sge_iq *iq, const struct rss_header *rss, 120 struct mbuf *m); 121 static int add_ep_to_req_list(struct c4iw_ep *ep, int ep_events); 122 #define START_EP_TIMER(ep) \ 123 do { \ 124 CTR3(KTR_IW_CXGBE, "start_ep_timer (%s:%d) ep %p", \ 125 __func__, __LINE__, (ep)); \ 126 start_ep_timer(ep); \ 127 } while (0) 128 129 #define STOP_EP_TIMER(ep) \ 130 ({ \ 131 CTR3(KTR_IW_CXGBE, "stop_ep_timer (%s:%d) ep %p", \ 132 __func__, __LINE__, (ep)); \ 133 stop_ep_timer(ep); \ 134 }) 135 136 #ifdef KTR 137 static char *states[] = { 138 "idle", 139 "listen", 140 "connecting", 141 "mpa_wait_req", 142 "mpa_req_sent", 143 "mpa_req_rcvd", 144 "mpa_rep_sent", 145 "fpdu_mode", 146 "aborting", 147 "closing", 148 "moribund", 149 "dead", 150 NULL, 151 }; 152 #endif 153 154 155 static void deref_cm_id(struct c4iw_ep_common *epc) 156 { 157 epc->cm_id->rem_ref(epc->cm_id); 158 epc->cm_id = NULL; 159 set_bit(CM_ID_DEREFED, &epc->history); 160 } 161 162 static void ref_cm_id(struct c4iw_ep_common *epc) 163 { 164 set_bit(CM_ID_REFED, &epc->history); 165 epc->cm_id->add_ref(epc->cm_id); 166 } 167 168 static void deref_qp(struct c4iw_ep *ep) 169 { 170 c4iw_qp_rem_ref(&ep->com.qp->ibqp); 171 clear_bit(QP_REFERENCED, &ep->com.flags); 172 set_bit(QP_DEREFED, &ep->com.history); 173 } 174 175 static void ref_qp(struct c4iw_ep *ep) 176 { 177 set_bit(QP_REFERENCED, &ep->com.flags); 178 set_bit(QP_REFED, &ep->com.history); 179 c4iw_qp_add_ref(&ep->com.qp->ibqp); 180 } 181 182 static void process_timeout(struct c4iw_ep *ep) 183 { 184 struct c4iw_qp_attributes attrs; 185 int abort = 1; 186 187 mutex_lock(&ep->com.mutex); 188 CTR4(KTR_IW_CXGBE, "%s ep :%p, tid:%u, state %d", __func__, 189 ep, ep->hwtid, ep->com.state); 190 set_bit(TIMEDOUT, &ep->com.history); 191 switch (ep->com.state) { 192 case MPA_REQ_SENT: 193 connect_reply_upcall(ep, -ETIMEDOUT); 194 break; 195 case MPA_REQ_WAIT: 196 case MPA_REQ_RCVD: 197 case MPA_REP_SENT: 198 case FPDU_MODE: 199 break; 200 case CLOSING: 201 case MORIBUND: 202 if (ep->com.cm_id && ep->com.qp) { 203 attrs.next_state = C4IW_QP_STATE_ERROR; 204 c4iw_modify_qp(ep->com.dev, ep->com.qp, 205 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 206 } 207 close_complete_upcall(ep, -ETIMEDOUT); 208 break; 209 case ABORTING: 210 case DEAD: 211 /* 212 * These states are expected if the ep timed out at the same 213 * time as another thread was calling stop_ep_timer(). 214 * So we silently do nothing for these states. 215 */ 216 abort = 0; 217 break; 218 default: 219 CTR4(KTR_IW_CXGBE, "%s unexpected state ep %p tid %u state %u\n" 220 , __func__, ep, ep->hwtid, ep->com.state); 221 abort = 0; 222 } 223 mutex_unlock(&ep->com.mutex); 224 if (abort) 225 c4iw_ep_disconnect(ep, 1, GFP_KERNEL); 226 c4iw_put_ep(&ep->com); 227 return; 228 } 229 230 struct cqe_list_entry { 231 struct list_head entry; 232 struct c4iw_dev *rhp; 233 struct t4_cqe err_cqe; 234 }; 235 236 static void 237 process_err_cqes(void) 238 { 239 unsigned long flag; 240 struct cqe_list_entry *cle; 241 242 spin_lock_irqsave(&err_cqe_lock, flag); 243 while (!list_empty(&err_cqe_list)) { 244 struct list_head *tmp; 245 tmp = err_cqe_list.next; 246 list_del(tmp); 247 tmp->next = tmp->prev = NULL; 248 spin_unlock_irqrestore(&err_cqe_lock, flag); 249 cle = list_entry(tmp, struct cqe_list_entry, entry); 250 c4iw_ev_dispatch(cle->rhp, &cle->err_cqe); 251 free(cle, M_CXGBE); 252 spin_lock_irqsave(&err_cqe_lock, flag); 253 } 254 spin_unlock_irqrestore(&err_cqe_lock, flag); 255 256 return; 257 } 258 259 static void 260 process_req(struct work_struct *ctx) 261 { 262 struct c4iw_ep_common *epc; 263 unsigned long flag; 264 int ep_events; 265 266 process_err_cqes(); 267 spin_lock_irqsave(&req_lock, flag); 268 while (!TAILQ_EMPTY(&req_list)) { 269 epc = TAILQ_FIRST(&req_list); 270 TAILQ_REMOVE(&req_list, epc, entry); 271 epc->entry.tqe_prev = NULL; 272 ep_events = epc->ep_events; 273 epc->ep_events = 0; 274 spin_unlock_irqrestore(&req_lock, flag); 275 CTR4(KTR_IW_CXGBE, "%s: so %p, ep %p, events 0x%x", __func__, 276 epc->so, epc, ep_events); 277 if (ep_events & C4IW_EVENT_TERM) 278 process_terminate((struct c4iw_ep *)epc); 279 if (ep_events & C4IW_EVENT_TIMEOUT) 280 process_timeout((struct c4iw_ep *)epc); 281 if (ep_events & C4IW_EVENT_SOCKET) 282 process_socket_event((struct c4iw_ep *)epc); 283 c4iw_put_ep(epc); 284 process_err_cqes(); 285 spin_lock_irqsave(&req_lock, flag); 286 } 287 spin_unlock_irqrestore(&req_lock, flag); 288 } 289 290 /* 291 * XXX: doesn't belong here in the iWARP driver. 292 * XXX: assumes that the connection was offloaded by cxgbe/t4_tom if TF_TOE is 293 * set. Is this a valid assumption for active open? 294 */ 295 static int 296 set_tcpinfo(struct c4iw_ep *ep) 297 { 298 struct socket *so = ep->com.so; 299 struct inpcb *inp = sotoinpcb(so); 300 struct tcpcb *tp; 301 struct toepcb *toep; 302 int rc = 0; 303 304 INP_WLOCK(inp); 305 tp = intotcpcb(inp); 306 if ((tp->t_flags & TF_TOE) == 0) { 307 rc = EINVAL; 308 log(LOG_ERR, "%s: connection not offloaded (so %p, ep %p)\n", 309 __func__, so, ep); 310 goto done; 311 } 312 toep = TOEPCB(so); 313 314 ep->hwtid = toep->tid; 315 ep->snd_seq = tp->snd_nxt; 316 ep->rcv_seq = tp->rcv_nxt; 317 ep->emss = max(tp->t_maxseg, 128); 318 done: 319 INP_WUNLOCK(inp); 320 return (rc); 321 322 } 323 324 static int 325 find_route(__be32 local_ip, __be32 peer_ip, __be16 local_port, 326 __be16 peer_port, u8 tos, struct nhop4_extended *pnh4) 327 { 328 struct in_addr addr; 329 int err; 330 331 CTR5(KTR_IW_CXGBE, "%s:frtB %x, %x, %d, %d", __func__, local_ip, 332 peer_ip, ntohs(local_port), ntohs(peer_port)); 333 334 addr.s_addr = peer_ip; 335 err = fib4_lookup_nh_ext(RT_DEFAULT_FIB, addr, NHR_REF, 0, pnh4); 336 337 CTR2(KTR_IW_CXGBE, "%s:frtE %d", __func__, err); 338 return err; 339 } 340 341 static void 342 close_socket(struct socket *so) 343 { 344 345 uninit_iwarp_socket(so); 346 sodisconnect(so); 347 } 348 349 static void 350 process_peer_close(struct c4iw_ep *ep) 351 { 352 struct c4iw_qp_attributes attrs; 353 int disconnect = 1; 354 int release = 0; 355 356 CTR4(KTR_IW_CXGBE, "%s:ppcB ep %p so %p state %s", __func__, ep, 357 ep->com.so, states[ep->com.state]); 358 359 mutex_lock(&ep->com.mutex); 360 switch (ep->com.state) { 361 362 case MPA_REQ_WAIT: 363 CTR2(KTR_IW_CXGBE, "%s:ppc1 %p MPA_REQ_WAIT CLOSING", 364 __func__, ep); 365 __state_set(&ep->com, CLOSING); 366 break; 367 368 case MPA_REQ_SENT: 369 CTR2(KTR_IW_CXGBE, "%s:ppc2 %p MPA_REQ_SENT CLOSING", 370 __func__, ep); 371 __state_set(&ep->com, DEAD); 372 connect_reply_upcall(ep, -ECONNABORTED); 373 374 disconnect = 0; 375 STOP_EP_TIMER(ep); 376 close_socket(ep->com.so); 377 deref_cm_id(&ep->com); 378 release = 1; 379 break; 380 381 case MPA_REQ_RCVD: 382 383 /* 384 * We're gonna mark this puppy DEAD, but keep 385 * the reference on it until the ULP accepts or 386 * rejects the CR. 387 */ 388 CTR2(KTR_IW_CXGBE, "%s:ppc3 %p MPA_REQ_RCVD CLOSING", 389 __func__, ep); 390 __state_set(&ep->com, CLOSING); 391 c4iw_get_ep(&ep->com); 392 break; 393 394 case MPA_REP_SENT: 395 CTR2(KTR_IW_CXGBE, "%s:ppc4 %p MPA_REP_SENT CLOSING", 396 __func__, ep); 397 __state_set(&ep->com, CLOSING); 398 break; 399 400 case FPDU_MODE: 401 CTR2(KTR_IW_CXGBE, "%s:ppc5 %p FPDU_MODE CLOSING", 402 __func__, ep); 403 START_EP_TIMER(ep); 404 __state_set(&ep->com, CLOSING); 405 attrs.next_state = C4IW_QP_STATE_CLOSING; 406 c4iw_modify_qp(ep->com.dev, ep->com.qp, 407 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 408 peer_close_upcall(ep); 409 break; 410 411 case ABORTING: 412 CTR2(KTR_IW_CXGBE, "%s:ppc6 %p ABORTING (disconn)", 413 __func__, ep); 414 disconnect = 0; 415 break; 416 417 case CLOSING: 418 CTR2(KTR_IW_CXGBE, "%s:ppc7 %p CLOSING MORIBUND", 419 __func__, ep); 420 __state_set(&ep->com, MORIBUND); 421 disconnect = 0; 422 break; 423 424 case MORIBUND: 425 CTR2(KTR_IW_CXGBE, "%s:ppc8 %p MORIBUND DEAD", __func__, 426 ep); 427 STOP_EP_TIMER(ep); 428 if (ep->com.cm_id && ep->com.qp) { 429 attrs.next_state = C4IW_QP_STATE_IDLE; 430 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 431 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 432 } 433 close_socket(ep->com.so); 434 close_complete_upcall(ep, 0); 435 __state_set(&ep->com, DEAD); 436 release = 1; 437 disconnect = 0; 438 break; 439 440 case DEAD: 441 CTR2(KTR_IW_CXGBE, "%s:ppc9 %p DEAD (disconn)", 442 __func__, ep); 443 disconnect = 0; 444 break; 445 446 default: 447 panic("%s: ep %p state %d", __func__, ep, 448 ep->com.state); 449 break; 450 } 451 452 mutex_unlock(&ep->com.mutex); 453 454 if (disconnect) { 455 456 CTR2(KTR_IW_CXGBE, "%s:ppca %p", __func__, ep); 457 c4iw_ep_disconnect(ep, 0, M_NOWAIT); 458 } 459 if (release) { 460 461 CTR2(KTR_IW_CXGBE, "%s:ppcb %p", __func__, ep); 462 c4iw_put_ep(&ep->com); 463 } 464 CTR2(KTR_IW_CXGBE, "%s:ppcE %p", __func__, ep); 465 return; 466 } 467 468 static void 469 process_conn_error(struct c4iw_ep *ep) 470 { 471 struct c4iw_qp_attributes attrs; 472 int ret; 473 int state; 474 475 mutex_lock(&ep->com.mutex); 476 state = ep->com.state; 477 CTR5(KTR_IW_CXGBE, "%s:pceB ep %p so %p so->so_error %u state %s", 478 __func__, ep, ep->com.so, ep->com.so->so_error, 479 states[ep->com.state]); 480 481 switch (state) { 482 483 case MPA_REQ_WAIT: 484 STOP_EP_TIMER(ep); 485 break; 486 487 case MPA_REQ_SENT: 488 STOP_EP_TIMER(ep); 489 connect_reply_upcall(ep, -ECONNRESET); 490 break; 491 492 case MPA_REP_SENT: 493 ep->com.rpl_err = ECONNRESET; 494 CTR1(KTR_IW_CXGBE, "waking up ep %p", ep); 495 break; 496 497 case MPA_REQ_RCVD: 498 499 /* 500 * We're gonna mark this puppy DEAD, but keep 501 * the reference on it until the ULP accepts or 502 * rejects the CR. 503 */ 504 c4iw_get_ep(&ep->com); 505 break; 506 507 case MORIBUND: 508 case CLOSING: 509 STOP_EP_TIMER(ep); 510 /*FALLTHROUGH*/ 511 case FPDU_MODE: 512 513 if (ep->com.cm_id && ep->com.qp) { 514 515 attrs.next_state = C4IW_QP_STATE_ERROR; 516 ret = c4iw_modify_qp(ep->com.qp->rhp, 517 ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, 518 &attrs, 1); 519 if (ret) 520 log(LOG_ERR, 521 "%s - qp <- error failed!\n", 522 __func__); 523 } 524 peer_abort_upcall(ep); 525 break; 526 527 case ABORTING: 528 break; 529 530 case DEAD: 531 CTR2(KTR_IW_CXGBE, "%s so_error %d IN DEAD STATE!!!!", 532 __func__, ep->com.so->so_error); 533 mutex_unlock(&ep->com.mutex); 534 return; 535 536 default: 537 panic("%s: ep %p state %d", __func__, ep, state); 538 break; 539 } 540 541 if (state != ABORTING) { 542 close_socket(ep->com.so); 543 __state_set(&ep->com, DEAD); 544 c4iw_put_ep(&ep->com); 545 } 546 mutex_unlock(&ep->com.mutex); 547 CTR2(KTR_IW_CXGBE, "%s:pceE %p", __func__, ep); 548 return; 549 } 550 551 static void 552 process_close_complete(struct c4iw_ep *ep) 553 { 554 struct c4iw_qp_attributes attrs; 555 int release = 0; 556 557 CTR4(KTR_IW_CXGBE, "%s:pccB ep %p so %p state %s", __func__, ep, 558 ep->com.so, states[ep->com.state]); 559 560 /* The cm_id may be null if we failed to connect */ 561 mutex_lock(&ep->com.mutex); 562 set_bit(CLOSE_CON_RPL, &ep->com.history); 563 564 switch (ep->com.state) { 565 566 case CLOSING: 567 CTR2(KTR_IW_CXGBE, "%s:pcc1 %p CLOSING MORIBUND", 568 __func__, ep); 569 __state_set(&ep->com, MORIBUND); 570 break; 571 572 case MORIBUND: 573 CTR2(KTR_IW_CXGBE, "%s:pcc1 %p MORIBUND DEAD", __func__, 574 ep); 575 STOP_EP_TIMER(ep); 576 577 if ((ep->com.cm_id) && (ep->com.qp)) { 578 579 CTR2(KTR_IW_CXGBE, "%s:pcc2 %p QP_STATE_IDLE", 580 __func__, ep); 581 attrs.next_state = C4IW_QP_STATE_IDLE; 582 c4iw_modify_qp(ep->com.dev, 583 ep->com.qp, 584 C4IW_QP_ATTR_NEXT_STATE, 585 &attrs, 1); 586 } 587 588 close_socket(ep->com.so); 589 close_complete_upcall(ep, 0); 590 __state_set(&ep->com, DEAD); 591 release = 1; 592 break; 593 594 case ABORTING: 595 CTR2(KTR_IW_CXGBE, "%s:pcc5 %p ABORTING", __func__, ep); 596 break; 597 598 case DEAD: 599 CTR2(KTR_IW_CXGBE, "%s:pcc6 %p DEAD", __func__, ep); 600 break; 601 default: 602 CTR2(KTR_IW_CXGBE, "%s:pcc7 %p unknown ep state", 603 __func__, ep); 604 panic("%s:pcc6 %p unknown ep state", __func__, ep); 605 break; 606 } 607 mutex_unlock(&ep->com.mutex); 608 609 if (release) { 610 611 CTR2(KTR_IW_CXGBE, "%s:pcc8 %p", __func__, ep); 612 c4iw_put_ep(&ep->com); 613 } 614 CTR2(KTR_IW_CXGBE, "%s:pccE %p", __func__, ep); 615 return; 616 } 617 618 static void 619 init_iwarp_socket(struct socket *so, void *arg) 620 { 621 int rc; 622 struct sockopt sopt; 623 int on = 1; 624 625 SOCKBUF_LOCK(&so->so_rcv); 626 soupcall_set(so, SO_RCV, c4iw_so_upcall, arg); 627 so->so_state |= SS_NBIO; 628 SOCKBUF_UNLOCK(&so->so_rcv); 629 sopt.sopt_dir = SOPT_SET; 630 sopt.sopt_level = IPPROTO_TCP; 631 sopt.sopt_name = TCP_NODELAY; 632 sopt.sopt_val = (caddr_t)&on; 633 sopt.sopt_valsize = sizeof on; 634 sopt.sopt_td = NULL; 635 rc = sosetopt(so, &sopt); 636 if (rc) { 637 log(LOG_ERR, "%s: can't set TCP_NODELAY on so %p (%d)\n", 638 __func__, so, rc); 639 } 640 } 641 642 static void 643 uninit_iwarp_socket(struct socket *so) 644 { 645 646 SOCKBUF_LOCK(&so->so_rcv); 647 soupcall_clear(so, SO_RCV); 648 SOCKBUF_UNLOCK(&so->so_rcv); 649 } 650 651 static void 652 process_data(struct c4iw_ep *ep) 653 { 654 struct sockaddr_in *local, *remote; 655 int disconnect = 0; 656 657 CTR5(KTR_IW_CXGBE, "%s: so %p, ep %p, state %s, sbused %d", __func__, 658 ep->com.so, ep, states[ep->com.state], sbused(&ep->com.so->so_rcv)); 659 660 switch (state_read(&ep->com)) { 661 case MPA_REQ_SENT: 662 disconnect = process_mpa_reply(ep); 663 break; 664 case MPA_REQ_WAIT: 665 in_getsockaddr(ep->com.so, (struct sockaddr **)&local); 666 in_getpeeraddr(ep->com.so, (struct sockaddr **)&remote); 667 ep->com.local_addr = *local; 668 ep->com.remote_addr = *remote; 669 free(local, M_SONAME); 670 free(remote, M_SONAME); 671 disconnect = process_mpa_request(ep); 672 break; 673 default: 674 if (sbused(&ep->com.so->so_rcv)) 675 log(LOG_ERR, "%s: Unexpected streaming data. ep %p, " 676 "state %d, so %p, so_state 0x%x, sbused %u\n", 677 __func__, ep, state_read(&ep->com), ep->com.so, 678 ep->com.so->so_state, sbused(&ep->com.so->so_rcv)); 679 break; 680 } 681 if (disconnect) 682 c4iw_ep_disconnect(ep, disconnect == 2, GFP_KERNEL); 683 684 } 685 686 static void 687 process_connected(struct c4iw_ep *ep) 688 { 689 struct socket *so = ep->com.so; 690 691 if ((so->so_state & SS_ISCONNECTED) && !so->so_error) { 692 if (send_mpa_req(ep)) 693 goto err; 694 } else { 695 connect_reply_upcall(ep, -so->so_error); 696 goto err; 697 } 698 return; 699 err: 700 close_socket(so); 701 state_set(&ep->com, DEAD); 702 c4iw_put_ep(&ep->com); 703 return; 704 } 705 706 void 707 process_newconn(struct iw_cm_id *parent_cm_id, struct socket *child_so) 708 { 709 struct c4iw_ep *child_ep; 710 struct sockaddr_in *local; 711 struct sockaddr_in *remote; 712 struct c4iw_ep *parent_ep = parent_cm_id->provider_data; 713 int ret = 0; 714 715 MPASS(child_so != NULL); 716 717 child_ep = alloc_ep(sizeof(*child_ep), GFP_KERNEL); 718 719 CTR5(KTR_IW_CXGBE, 720 "%s: parent so %p, parent ep %p, child so %p, child ep %p", 721 __func__, parent_ep->com.so, parent_ep, child_so, child_ep); 722 723 in_getsockaddr(child_so, (struct sockaddr **)&local); 724 in_getpeeraddr(child_so, (struct sockaddr **)&remote); 725 726 child_ep->com.local_addr = *local; 727 child_ep->com.remote_addr = *remote; 728 child_ep->com.dev = parent_ep->com.dev; 729 child_ep->com.so = child_so; 730 child_ep->com.cm_id = NULL; 731 child_ep->com.thread = parent_ep->com.thread; 732 child_ep->parent_ep = parent_ep; 733 734 free(local, M_SONAME); 735 free(remote, M_SONAME); 736 737 init_iwarp_socket(child_so, &child_ep->com); 738 c4iw_get_ep(&parent_ep->com); 739 init_timer(&child_ep->timer); 740 state_set(&child_ep->com, MPA_REQ_WAIT); 741 START_EP_TIMER(child_ep); 742 743 /* maybe the request has already been queued up on the socket... */ 744 ret = process_mpa_request(child_ep); 745 if (ret == 2) 746 /* ABORT */ 747 c4iw_ep_disconnect(child_ep, 1, GFP_KERNEL); 748 else if (ret == 1) 749 /* CLOSE */ 750 c4iw_ep_disconnect(child_ep, 0, GFP_KERNEL); 751 752 return; 753 } 754 755 static int 756 add_ep_to_req_list(struct c4iw_ep *ep, int new_ep_event) 757 { 758 unsigned long flag; 759 760 spin_lock_irqsave(&req_lock, flag); 761 if (ep && ep->com.so) { 762 ep->com.ep_events |= new_ep_event; 763 if (!ep->com.entry.tqe_prev) { 764 c4iw_get_ep(&ep->com); 765 TAILQ_INSERT_TAIL(&req_list, &ep->com, entry); 766 queue_work(c4iw_taskq, &c4iw_task); 767 } 768 } 769 spin_unlock_irqrestore(&req_lock, flag); 770 771 return (0); 772 } 773 774 static int 775 c4iw_so_upcall(struct socket *so, void *arg, int waitflag) 776 { 777 struct c4iw_ep *ep = arg; 778 779 CTR6(KTR_IW_CXGBE, 780 "%s: so %p, so_state 0x%x, ep %p, ep_state %s, tqe_prev %p", 781 __func__, so, so->so_state, ep, states[ep->com.state], 782 ep->com.entry.tqe_prev); 783 784 MPASS(ep->com.so == so); 785 add_ep_to_req_list(ep, C4IW_EVENT_SOCKET); 786 787 return (SU_OK); 788 } 789 790 791 static int 792 terminate(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 793 { 794 struct adapter *sc = iq->adapter; 795 const struct cpl_rdma_terminate *cpl = mtod(m, const void *); 796 unsigned int tid = GET_TID(cpl); 797 struct toepcb *toep = lookup_tid(sc, tid); 798 struct socket *so; 799 struct c4iw_ep *ep; 800 801 INP_WLOCK(toep->inp); 802 so = inp_inpcbtosocket(toep->inp); 803 ep = so->so_rcv.sb_upcallarg; 804 INP_WUNLOCK(toep->inp); 805 806 CTR3(KTR_IW_CXGBE, "%s: so %p, ep %p", __func__, so, ep); 807 add_ep_to_req_list(ep, C4IW_EVENT_TERM); 808 809 return 0; 810 } 811 812 static void 813 process_socket_event(struct c4iw_ep *ep) 814 { 815 int state = state_read(&ep->com); 816 struct socket *so = ep->com.so; 817 818 CTR6(KTR_IW_CXGBE, "process_socket_event: so %p, so_state 0x%x, " 819 "so_err %d, sb_state 0x%x, ep %p, ep_state %s", so, so->so_state, 820 so->so_error, so->so_rcv.sb_state, ep, states[state]); 821 822 if (state == CONNECTING) { 823 process_connected(ep); 824 return; 825 } 826 827 if (state == LISTEN) { 828 /* socket listening events are handled at IWCM */ 829 CTR3(KTR_IW_CXGBE, "%s Invalid ep state:%u, ep:%p", __func__, 830 ep->com.state, ep); 831 BUG(); 832 return; 833 } 834 835 /* connection error */ 836 if (so->so_error) { 837 process_conn_error(ep); 838 return; 839 } 840 841 /* peer close */ 842 if ((so->so_rcv.sb_state & SBS_CANTRCVMORE) && state <= CLOSING) { 843 process_peer_close(ep); 844 /* 845 * check whether socket disconnect event is pending before 846 * returning. Fallthrough if yes. 847 */ 848 if (!(so->so_state & SS_ISDISCONNECTED)) 849 return; 850 } 851 852 /* close complete */ 853 if (so->so_state & SS_ISDISCONNECTED) { 854 process_close_complete(ep); 855 return; 856 } 857 858 /* rx data */ 859 process_data(ep); 860 } 861 862 SYSCTL_NODE(_hw, OID_AUTO, iw_cxgbe, CTLFLAG_RD, 0, "iw_cxgbe driver parameters"); 863 864 static int dack_mode = 0; 865 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, dack_mode, CTLFLAG_RWTUN, &dack_mode, 0, 866 "Delayed ack mode (default = 0)"); 867 868 int c4iw_max_read_depth = 8; 869 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, c4iw_max_read_depth, CTLFLAG_RWTUN, &c4iw_max_read_depth, 0, 870 "Per-connection max ORD/IRD (default = 8)"); 871 872 static int enable_tcp_timestamps; 873 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, enable_tcp_timestamps, CTLFLAG_RWTUN, &enable_tcp_timestamps, 0, 874 "Enable tcp timestamps (default = 0)"); 875 876 static int enable_tcp_sack; 877 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, enable_tcp_sack, CTLFLAG_RWTUN, &enable_tcp_sack, 0, 878 "Enable tcp SACK (default = 0)"); 879 880 static int enable_tcp_window_scaling = 1; 881 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, enable_tcp_window_scaling, CTLFLAG_RWTUN, &enable_tcp_window_scaling, 0, 882 "Enable tcp window scaling (default = 1)"); 883 884 int c4iw_debug = 0; 885 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, c4iw_debug, CTLFLAG_RWTUN, &c4iw_debug, 0, 886 "Enable debug logging (default = 0)"); 887 888 static int peer2peer = 1; 889 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, peer2peer, CTLFLAG_RWTUN, &peer2peer, 0, 890 "Support peer2peer ULPs (default = 1)"); 891 892 static int p2p_type = FW_RI_INIT_P2PTYPE_READ_REQ; 893 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, p2p_type, CTLFLAG_RWTUN, &p2p_type, 0, 894 "RDMAP opcode to use for the RTR message: 1 = RDMA_READ 0 = RDMA_WRITE (default 1)"); 895 896 static int ep_timeout_secs = 60; 897 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, ep_timeout_secs, CTLFLAG_RWTUN, &ep_timeout_secs, 0, 898 "CM Endpoint operation timeout in seconds (default = 60)"); 899 900 static int mpa_rev = 1; 901 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, mpa_rev, CTLFLAG_RWTUN, &mpa_rev, 0, 902 "MPA Revision, 0 supports amso1100, 1 is RFC5044 spec compliant, 2 is IETF MPA Peer Connect Draft compliant (default = 1)"); 903 904 static int markers_enabled; 905 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, markers_enabled, CTLFLAG_RWTUN, &markers_enabled, 0, 906 "Enable MPA MARKERS (default(0) = disabled)"); 907 908 static int crc_enabled = 1; 909 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, crc_enabled, CTLFLAG_RWTUN, &crc_enabled, 0, 910 "Enable MPA CRC (default(1) = enabled)"); 911 912 static int rcv_win = 256 * 1024; 913 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, rcv_win, CTLFLAG_RWTUN, &rcv_win, 0, 914 "TCP receive window in bytes (default = 256KB)"); 915 916 static int snd_win = 128 * 1024; 917 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, snd_win, CTLFLAG_RWTUN, &snd_win, 0, 918 "TCP send window in bytes (default = 128KB)"); 919 920 static void 921 start_ep_timer(struct c4iw_ep *ep) 922 { 923 924 if (timer_pending(&ep->timer)) { 925 CTR2(KTR_IW_CXGBE, "%s: ep %p, already started", __func__, ep); 926 printk(KERN_ERR "%s timer already started! ep %p\n", __func__, 927 ep); 928 return; 929 } 930 clear_bit(TIMEOUT, &ep->com.flags); 931 c4iw_get_ep(&ep->com); 932 ep->timer.expires = jiffies + ep_timeout_secs * HZ; 933 ep->timer.data = (unsigned long)ep; 934 ep->timer.function = ep_timeout; 935 add_timer(&ep->timer); 936 } 937 938 static int 939 stop_ep_timer(struct c4iw_ep *ep) 940 { 941 942 del_timer_sync(&ep->timer); 943 if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) { 944 c4iw_put_ep(&ep->com); 945 return 0; 946 } 947 return 1; 948 } 949 950 static enum 951 c4iw_ep_state state_read(struct c4iw_ep_common *epc) 952 { 953 enum c4iw_ep_state state; 954 955 mutex_lock(&epc->mutex); 956 state = epc->state; 957 mutex_unlock(&epc->mutex); 958 959 return (state); 960 } 961 962 static void 963 __state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new) 964 { 965 966 epc->state = new; 967 } 968 969 static void 970 state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new) 971 { 972 973 mutex_lock(&epc->mutex); 974 __state_set(epc, new); 975 mutex_unlock(&epc->mutex); 976 } 977 978 static void * 979 alloc_ep(int size, gfp_t gfp) 980 { 981 struct c4iw_ep_common *epc; 982 983 epc = kzalloc(size, gfp); 984 if (epc == NULL) 985 return (NULL); 986 987 kref_init(&epc->kref); 988 mutex_init(&epc->mutex); 989 c4iw_init_wr_wait(&epc->wr_wait); 990 991 return (epc); 992 } 993 994 void _c4iw_free_ep(struct kref *kref) 995 { 996 struct c4iw_ep *ep; 997 struct c4iw_ep_common *epc; 998 999 ep = container_of(kref, struct c4iw_ep, com.kref); 1000 epc = &ep->com; 1001 KASSERT(!epc->entry.tqe_prev, ("%s epc %p still on req list", 1002 __func__, epc)); 1003 if (test_bit(QP_REFERENCED, &ep->com.flags)) 1004 deref_qp(ep); 1005 kfree(ep); 1006 } 1007 1008 static void release_ep_resources(struct c4iw_ep *ep) 1009 { 1010 CTR2(KTR_IW_CXGBE, "%s:rerB %p", __func__, ep); 1011 set_bit(RELEASE_RESOURCES, &ep->com.flags); 1012 c4iw_put_ep(&ep->com); 1013 CTR2(KTR_IW_CXGBE, "%s:rerE %p", __func__, ep); 1014 } 1015 1016 static int 1017 send_mpa_req(struct c4iw_ep *ep) 1018 { 1019 int mpalen; 1020 struct mpa_message *mpa; 1021 struct mpa_v2_conn_params mpa_v2_params; 1022 struct mbuf *m; 1023 char mpa_rev_to_use = mpa_rev; 1024 int err = 0; 1025 1026 if (ep->retry_with_mpa_v1) 1027 mpa_rev_to_use = 1; 1028 mpalen = sizeof(*mpa) + ep->plen; 1029 if (mpa_rev_to_use == 2) 1030 mpalen += sizeof(struct mpa_v2_conn_params); 1031 1032 mpa = malloc(mpalen, M_CXGBE, M_NOWAIT); 1033 if (mpa == NULL) { 1034 err = -ENOMEM; 1035 CTR3(KTR_IW_CXGBE, "%s:smr1 ep: %p , error: %d", 1036 __func__, ep, err); 1037 goto err; 1038 } 1039 1040 memset(mpa, 0, mpalen); 1041 memcpy(mpa->key, MPA_KEY_REQ, sizeof(mpa->key)); 1042 mpa->flags = (crc_enabled ? MPA_CRC : 0) | 1043 (markers_enabled ? MPA_MARKERS : 0) | 1044 (mpa_rev_to_use == 2 ? MPA_ENHANCED_RDMA_CONN : 0); 1045 mpa->private_data_size = htons(ep->plen); 1046 mpa->revision = mpa_rev_to_use; 1047 1048 if (mpa_rev_to_use == 1) { 1049 ep->tried_with_mpa_v1 = 1; 1050 ep->retry_with_mpa_v1 = 0; 1051 } 1052 1053 if (mpa_rev_to_use == 2) { 1054 mpa->private_data_size += 1055 htons(sizeof(struct mpa_v2_conn_params)); 1056 mpa_v2_params.ird = htons((u16)ep->ird); 1057 mpa_v2_params.ord = htons((u16)ep->ord); 1058 1059 if (peer2peer) { 1060 mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); 1061 1062 if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) { 1063 mpa_v2_params.ord |= 1064 htons(MPA_V2_RDMA_WRITE_RTR); 1065 } else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) { 1066 mpa_v2_params.ord |= 1067 htons(MPA_V2_RDMA_READ_RTR); 1068 } 1069 } 1070 memcpy(mpa->private_data, &mpa_v2_params, 1071 sizeof(struct mpa_v2_conn_params)); 1072 1073 if (ep->plen) { 1074 1075 memcpy(mpa->private_data + 1076 sizeof(struct mpa_v2_conn_params), 1077 ep->mpa_pkt + sizeof(*mpa), ep->plen); 1078 } 1079 } else { 1080 1081 if (ep->plen) 1082 memcpy(mpa->private_data, 1083 ep->mpa_pkt + sizeof(*mpa), ep->plen); 1084 CTR2(KTR_IW_CXGBE, "%s:smr7 %p", __func__, ep); 1085 } 1086 1087 m = m_getm(NULL, mpalen, M_NOWAIT, MT_DATA); 1088 if (m == NULL) { 1089 err = -ENOMEM; 1090 CTR3(KTR_IW_CXGBE, "%s:smr2 ep: %p , error: %d", 1091 __func__, ep, err); 1092 free(mpa, M_CXGBE); 1093 goto err; 1094 } 1095 m_copyback(m, 0, mpalen, (void *)mpa); 1096 free(mpa, M_CXGBE); 1097 1098 err = -sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, 1099 ep->com.thread); 1100 if (err) { 1101 CTR3(KTR_IW_CXGBE, "%s:smr3 ep: %p , error: %d", 1102 __func__, ep, err); 1103 goto err; 1104 } 1105 1106 START_EP_TIMER(ep); 1107 state_set(&ep->com, MPA_REQ_SENT); 1108 ep->mpa_attr.initiator = 1; 1109 CTR3(KTR_IW_CXGBE, "%s:smrE %p, error: %d", __func__, ep, err); 1110 return 0; 1111 err: 1112 connect_reply_upcall(ep, err); 1113 CTR3(KTR_IW_CXGBE, "%s:smrE %p, error: %d", __func__, ep, err); 1114 return err; 1115 } 1116 1117 static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen) 1118 { 1119 int mpalen ; 1120 struct mpa_message *mpa; 1121 struct mpa_v2_conn_params mpa_v2_params; 1122 struct mbuf *m; 1123 int err; 1124 1125 CTR4(KTR_IW_CXGBE, "%s:smrejB %p %u %d", __func__, ep, ep->hwtid, 1126 ep->plen); 1127 1128 mpalen = sizeof(*mpa) + plen; 1129 1130 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 1131 1132 mpalen += sizeof(struct mpa_v2_conn_params); 1133 CTR4(KTR_IW_CXGBE, "%s:smrej1 %p %u %d", __func__, ep, 1134 ep->mpa_attr.version, mpalen); 1135 } 1136 1137 mpa = malloc(mpalen, M_CXGBE, M_NOWAIT); 1138 if (mpa == NULL) 1139 return (-ENOMEM); 1140 1141 memset(mpa, 0, mpalen); 1142 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 1143 mpa->flags = MPA_REJECT; 1144 mpa->revision = mpa_rev; 1145 mpa->private_data_size = htons(plen); 1146 1147 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 1148 1149 mpa->flags |= MPA_ENHANCED_RDMA_CONN; 1150 mpa->private_data_size += 1151 htons(sizeof(struct mpa_v2_conn_params)); 1152 mpa_v2_params.ird = htons(((u16)ep->ird) | 1153 (peer2peer ? MPA_V2_PEER2PEER_MODEL : 1154 0)); 1155 mpa_v2_params.ord = htons(((u16)ep->ord) | (peer2peer ? 1156 (p2p_type == 1157 FW_RI_INIT_P2PTYPE_RDMA_WRITE ? 1158 MPA_V2_RDMA_WRITE_RTR : p2p_type == 1159 FW_RI_INIT_P2PTYPE_READ_REQ ? 1160 MPA_V2_RDMA_READ_RTR : 0) : 0)); 1161 memcpy(mpa->private_data, &mpa_v2_params, 1162 sizeof(struct mpa_v2_conn_params)); 1163 1164 if (ep->plen) 1165 memcpy(mpa->private_data + 1166 sizeof(struct mpa_v2_conn_params), pdata, plen); 1167 CTR5(KTR_IW_CXGBE, "%s:smrej3 %p %d %d %d", __func__, ep, 1168 mpa_v2_params.ird, mpa_v2_params.ord, ep->plen); 1169 } else 1170 if (plen) 1171 memcpy(mpa->private_data, pdata, plen); 1172 1173 m = m_getm(NULL, mpalen, M_NOWAIT, MT_DATA); 1174 if (m == NULL) { 1175 free(mpa, M_CXGBE); 1176 return (-ENOMEM); 1177 } 1178 m_copyback(m, 0, mpalen, (void *)mpa); 1179 free(mpa, M_CXGBE); 1180 1181 err = -sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, ep->com.thread); 1182 if (!err) 1183 ep->snd_seq += mpalen; 1184 CTR4(KTR_IW_CXGBE, "%s:smrejE %p %u %d", __func__, ep, ep->hwtid, err); 1185 return err; 1186 } 1187 1188 static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen) 1189 { 1190 int mpalen; 1191 struct mpa_message *mpa; 1192 struct mbuf *m; 1193 struct mpa_v2_conn_params mpa_v2_params; 1194 int err; 1195 1196 CTR2(KTR_IW_CXGBE, "%s:smrepB %p", __func__, ep); 1197 1198 mpalen = sizeof(*mpa) + plen; 1199 1200 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 1201 1202 CTR3(KTR_IW_CXGBE, "%s:smrep1 %p %d", __func__, ep, 1203 ep->mpa_attr.version); 1204 mpalen += sizeof(struct mpa_v2_conn_params); 1205 } 1206 1207 mpa = malloc(mpalen, M_CXGBE, M_NOWAIT); 1208 if (mpa == NULL) 1209 return (-ENOMEM); 1210 1211 memset(mpa, 0, sizeof(*mpa)); 1212 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 1213 mpa->flags = (ep->mpa_attr.crc_enabled ? MPA_CRC : 0) | 1214 (markers_enabled ? MPA_MARKERS : 0); 1215 mpa->revision = ep->mpa_attr.version; 1216 mpa->private_data_size = htons(plen); 1217 1218 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 1219 1220 mpa->flags |= MPA_ENHANCED_RDMA_CONN; 1221 mpa->private_data_size += 1222 htons(sizeof(struct mpa_v2_conn_params)); 1223 mpa_v2_params.ird = htons((u16)ep->ird); 1224 mpa_v2_params.ord = htons((u16)ep->ord); 1225 CTR5(KTR_IW_CXGBE, "%s:smrep3 %p %d %d %d", __func__, ep, 1226 ep->mpa_attr.version, mpa_v2_params.ird, mpa_v2_params.ord); 1227 1228 if (peer2peer && (ep->mpa_attr.p2p_type != 1229 FW_RI_INIT_P2PTYPE_DISABLED)) { 1230 1231 mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); 1232 1233 if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) { 1234 1235 mpa_v2_params.ord |= 1236 htons(MPA_V2_RDMA_WRITE_RTR); 1237 CTR5(KTR_IW_CXGBE, "%s:smrep4 %p %d %d %d", 1238 __func__, ep, p2p_type, mpa_v2_params.ird, 1239 mpa_v2_params.ord); 1240 } 1241 else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) { 1242 1243 mpa_v2_params.ord |= 1244 htons(MPA_V2_RDMA_READ_RTR); 1245 CTR5(KTR_IW_CXGBE, "%s:smrep5 %p %d %d %d", 1246 __func__, ep, p2p_type, mpa_v2_params.ird, 1247 mpa_v2_params.ord); 1248 } 1249 } 1250 1251 memcpy(mpa->private_data, &mpa_v2_params, 1252 sizeof(struct mpa_v2_conn_params)); 1253 1254 if (ep->plen) 1255 memcpy(mpa->private_data + 1256 sizeof(struct mpa_v2_conn_params), pdata, plen); 1257 } else 1258 if (plen) 1259 memcpy(mpa->private_data, pdata, plen); 1260 1261 m = m_getm(NULL, mpalen, M_NOWAIT, MT_DATA); 1262 if (m == NULL) { 1263 free(mpa, M_CXGBE); 1264 return (-ENOMEM); 1265 } 1266 m_copyback(m, 0, mpalen, (void *)mpa); 1267 free(mpa, M_CXGBE); 1268 1269 1270 state_set(&ep->com, MPA_REP_SENT); 1271 ep->snd_seq += mpalen; 1272 err = -sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, 1273 ep->com.thread); 1274 CTR3(KTR_IW_CXGBE, "%s:smrepE %p %d", __func__, ep, err); 1275 return err; 1276 } 1277 1278 1279 1280 static void close_complete_upcall(struct c4iw_ep *ep, int status) 1281 { 1282 struct iw_cm_event event; 1283 1284 CTR2(KTR_IW_CXGBE, "%s:ccuB %p", __func__, ep); 1285 memset(&event, 0, sizeof(event)); 1286 event.event = IW_CM_EVENT_CLOSE; 1287 event.status = status; 1288 1289 if (ep->com.cm_id) { 1290 1291 CTR2(KTR_IW_CXGBE, "%s:ccu1 %1", __func__, ep); 1292 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1293 deref_cm_id(&ep->com); 1294 set_bit(CLOSE_UPCALL, &ep->com.history); 1295 } 1296 CTR2(KTR_IW_CXGBE, "%s:ccuE %p", __func__, ep); 1297 } 1298 1299 static int 1300 send_abort(struct c4iw_ep *ep) 1301 { 1302 struct socket *so = ep->com.so; 1303 struct sockopt sopt; 1304 int rc; 1305 struct linger l; 1306 1307 CTR5(KTR_IW_CXGBE, "%s ep %p so %p state %s tid %d", __func__, ep, so, 1308 states[ep->com.state], ep->hwtid); 1309 1310 l.l_onoff = 1; 1311 l.l_linger = 0; 1312 1313 /* linger_time of 0 forces RST to be sent */ 1314 sopt.sopt_dir = SOPT_SET; 1315 sopt.sopt_level = SOL_SOCKET; 1316 sopt.sopt_name = SO_LINGER; 1317 sopt.sopt_val = (caddr_t)&l; 1318 sopt.sopt_valsize = sizeof l; 1319 sopt.sopt_td = NULL; 1320 rc = sosetopt(so, &sopt); 1321 if (rc != 0) { 1322 log(LOG_ERR, "%s: sosetopt(%p, linger = 0) failed with %d.\n", 1323 __func__, so, rc); 1324 } 1325 1326 uninit_iwarp_socket(so); 1327 sodisconnect(so); 1328 set_bit(ABORT_CONN, &ep->com.history); 1329 1330 /* 1331 * TBD: iw_cxgbe driver should receive ABORT reply for every ABORT 1332 * request it has sent. But the current TOE driver is not propagating 1333 * this ABORT reply event (via do_abort_rpl) to iw_cxgbe. So as a work- 1334 * around de-refer 'ep' (which was refered before sending ABORT request) 1335 * here instead of doing it in abort_rpl() handler of iw_cxgbe driver. 1336 */ 1337 c4iw_put_ep(&ep->com); 1338 1339 return (0); 1340 } 1341 1342 static void peer_close_upcall(struct c4iw_ep *ep) 1343 { 1344 struct iw_cm_event event; 1345 1346 CTR2(KTR_IW_CXGBE, "%s:pcuB %p", __func__, ep); 1347 memset(&event, 0, sizeof(event)); 1348 event.event = IW_CM_EVENT_DISCONNECT; 1349 1350 if (ep->com.cm_id) { 1351 1352 CTR2(KTR_IW_CXGBE, "%s:pcu1 %p", __func__, ep); 1353 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1354 set_bit(DISCONN_UPCALL, &ep->com.history); 1355 } 1356 CTR2(KTR_IW_CXGBE, "%s:pcuE %p", __func__, ep); 1357 } 1358 1359 static void peer_abort_upcall(struct c4iw_ep *ep) 1360 { 1361 struct iw_cm_event event; 1362 1363 CTR2(KTR_IW_CXGBE, "%s:pauB %p", __func__, ep); 1364 memset(&event, 0, sizeof(event)); 1365 event.event = IW_CM_EVENT_CLOSE; 1366 event.status = -ECONNRESET; 1367 1368 if (ep->com.cm_id) { 1369 1370 CTR2(KTR_IW_CXGBE, "%s:pau1 %p", __func__, ep); 1371 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1372 deref_cm_id(&ep->com); 1373 set_bit(ABORT_UPCALL, &ep->com.history); 1374 } 1375 CTR2(KTR_IW_CXGBE, "%s:pauE %p", __func__, ep); 1376 } 1377 1378 static void connect_reply_upcall(struct c4iw_ep *ep, int status) 1379 { 1380 struct iw_cm_event event; 1381 1382 CTR3(KTR_IW_CXGBE, "%s:cruB %p, status: %d", __func__, ep, status); 1383 memset(&event, 0, sizeof(event)); 1384 event.event = IW_CM_EVENT_CONNECT_REPLY; 1385 event.status = ((status == -ECONNABORTED) || (status == -EPIPE)) ? 1386 -ECONNRESET : status; 1387 event.local_addr = ep->com.local_addr; 1388 event.remote_addr = ep->com.remote_addr; 1389 1390 if ((status == 0) || (status == -ECONNREFUSED)) { 1391 1392 if (!ep->tried_with_mpa_v1) { 1393 1394 CTR2(KTR_IW_CXGBE, "%s:cru1 %p", __func__, ep); 1395 /* this means MPA_v2 is used */ 1396 event.private_data_len = ep->plen - 1397 sizeof(struct mpa_v2_conn_params); 1398 event.private_data = ep->mpa_pkt + 1399 sizeof(struct mpa_message) + 1400 sizeof(struct mpa_v2_conn_params); 1401 } else { 1402 1403 CTR2(KTR_IW_CXGBE, "%s:cru2 %p", __func__, ep); 1404 /* this means MPA_v1 is used */ 1405 event.private_data_len = ep->plen; 1406 event.private_data = ep->mpa_pkt + 1407 sizeof(struct mpa_message); 1408 } 1409 } 1410 1411 if (ep->com.cm_id) { 1412 1413 CTR2(KTR_IW_CXGBE, "%s:cru3 %p", __func__, ep); 1414 set_bit(CONN_RPL_UPCALL, &ep->com.history); 1415 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1416 } 1417 1418 if(status == -ECONNABORTED) { 1419 1420 CTR3(KTR_IW_CXGBE, "%s:cruE %p %d", __func__, ep, status); 1421 return; 1422 } 1423 1424 if (status < 0) { 1425 1426 CTR3(KTR_IW_CXGBE, "%s:cru4 %p %d", __func__, ep, status); 1427 deref_cm_id(&ep->com); 1428 } 1429 1430 CTR2(KTR_IW_CXGBE, "%s:cruE %p", __func__, ep); 1431 } 1432 1433 static int connect_request_upcall(struct c4iw_ep *ep) 1434 { 1435 struct iw_cm_event event; 1436 int ret; 1437 1438 CTR3(KTR_IW_CXGBE, "%s: ep %p, mpa_v1 %d", __func__, ep, 1439 ep->tried_with_mpa_v1); 1440 1441 memset(&event, 0, sizeof(event)); 1442 event.event = IW_CM_EVENT_CONNECT_REQUEST; 1443 event.local_addr = ep->com.local_addr; 1444 event.remote_addr = ep->com.remote_addr; 1445 event.provider_data = ep; 1446 event.so = ep->com.so; 1447 1448 if (!ep->tried_with_mpa_v1) { 1449 /* this means MPA_v2 is used */ 1450 event.ord = ep->ord; 1451 event.ird = ep->ird; 1452 event.private_data_len = ep->plen - 1453 sizeof(struct mpa_v2_conn_params); 1454 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message) + 1455 sizeof(struct mpa_v2_conn_params); 1456 } else { 1457 1458 /* this means MPA_v1 is used. Send max supported */ 1459 event.ord = c4iw_max_read_depth; 1460 event.ird = c4iw_max_read_depth; 1461 event.private_data_len = ep->plen; 1462 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message); 1463 } 1464 1465 c4iw_get_ep(&ep->com); 1466 ret = ep->parent_ep->com.cm_id->event_handler(ep->parent_ep->com.cm_id, 1467 &event); 1468 if(ret) 1469 c4iw_put_ep(&ep->com); 1470 1471 set_bit(CONNREQ_UPCALL, &ep->com.history); 1472 c4iw_put_ep(&ep->parent_ep->com); 1473 return ret; 1474 } 1475 1476 static void established_upcall(struct c4iw_ep *ep) 1477 { 1478 struct iw_cm_event event; 1479 1480 CTR2(KTR_IW_CXGBE, "%s:euB %p", __func__, ep); 1481 memset(&event, 0, sizeof(event)); 1482 event.event = IW_CM_EVENT_ESTABLISHED; 1483 event.ird = ep->ird; 1484 event.ord = ep->ord; 1485 1486 if (ep->com.cm_id) { 1487 1488 CTR2(KTR_IW_CXGBE, "%s:eu1 %p", __func__, ep); 1489 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1490 set_bit(ESTAB_UPCALL, &ep->com.history); 1491 } 1492 CTR2(KTR_IW_CXGBE, "%s:euE %p", __func__, ep); 1493 } 1494 1495 1496 /* 1497 * process_mpa_reply - process streaming mode MPA reply 1498 * 1499 * Returns: 1500 * 1501 * 0 upon success indicating a connect request was delivered to the ULP 1502 * or the mpa request is incomplete but valid so far. 1503 * 1504 * 1 if a failure requires the caller to close the connection. 1505 * 1506 * 2 if a failure requires the caller to abort the connection. 1507 */ 1508 static int process_mpa_reply(struct c4iw_ep *ep) 1509 { 1510 struct mpa_message *mpa; 1511 struct mpa_v2_conn_params *mpa_v2_params; 1512 u16 plen; 1513 u16 resp_ird, resp_ord; 1514 u8 rtr_mismatch = 0, insuff_ird = 0; 1515 struct c4iw_qp_attributes attrs; 1516 enum c4iw_qp_attr_mask mask; 1517 int err; 1518 struct mbuf *top, *m; 1519 int flags = MSG_DONTWAIT; 1520 struct uio uio; 1521 int disconnect = 0; 1522 1523 CTR2(KTR_IW_CXGBE, "%s:pmrB %p", __func__, ep); 1524 1525 /* 1526 * Stop mpa timer. If it expired, then 1527 * we ignore the MPA reply. process_timeout() 1528 * will abort the connection. 1529 */ 1530 if (STOP_EP_TIMER(ep)) 1531 return 0; 1532 1533 uio.uio_resid = 1000000; 1534 uio.uio_td = ep->com.thread; 1535 err = soreceive(ep->com.so, NULL, &uio, &top, NULL, &flags); 1536 1537 if (err) { 1538 1539 if (err == EWOULDBLOCK) { 1540 1541 CTR2(KTR_IW_CXGBE, "%s:pmr1 %p", __func__, ep); 1542 START_EP_TIMER(ep); 1543 return 0; 1544 } 1545 err = -err; 1546 CTR2(KTR_IW_CXGBE, "%s:pmr2 %p", __func__, ep); 1547 goto err; 1548 } 1549 1550 if (ep->com.so->so_rcv.sb_mb) { 1551 1552 CTR2(KTR_IW_CXGBE, "%s:pmr3 %p", __func__, ep); 1553 printf("%s data after soreceive called! so %p sb_mb %p top %p\n", 1554 __func__, ep->com.so, ep->com.so->so_rcv.sb_mb, top); 1555 } 1556 1557 m = top; 1558 1559 do { 1560 1561 CTR2(KTR_IW_CXGBE, "%s:pmr4 %p", __func__, ep); 1562 /* 1563 * If we get more than the supported amount of private data 1564 * then we must fail this connection. 1565 */ 1566 if (ep->mpa_pkt_len + m->m_len > sizeof(ep->mpa_pkt)) { 1567 1568 CTR3(KTR_IW_CXGBE, "%s:pmr5 %p %d", __func__, ep, 1569 ep->mpa_pkt_len + m->m_len); 1570 err = (-EINVAL); 1571 goto err_stop_timer; 1572 } 1573 1574 /* 1575 * copy the new data into our accumulation buffer. 1576 */ 1577 m_copydata(m, 0, m->m_len, &(ep->mpa_pkt[ep->mpa_pkt_len])); 1578 ep->mpa_pkt_len += m->m_len; 1579 if (!m->m_next) 1580 m = m->m_nextpkt; 1581 else 1582 m = m->m_next; 1583 } while (m); 1584 1585 m_freem(top); 1586 /* 1587 * if we don't even have the mpa message, then bail. 1588 */ 1589 if (ep->mpa_pkt_len < sizeof(*mpa)) { 1590 return 0; 1591 } 1592 mpa = (struct mpa_message *) ep->mpa_pkt; 1593 1594 /* Validate MPA header. */ 1595 if (mpa->revision > mpa_rev) { 1596 1597 CTR4(KTR_IW_CXGBE, "%s:pmr6 %p %d %d", __func__, ep, 1598 mpa->revision, mpa_rev); 1599 printk(KERN_ERR MOD "%s MPA version mismatch. Local = %d, " 1600 " Received = %d\n", __func__, mpa_rev, mpa->revision); 1601 err = -EPROTO; 1602 goto err_stop_timer; 1603 } 1604 1605 if (memcmp(mpa->key, MPA_KEY_REP, sizeof(mpa->key))) { 1606 1607 CTR2(KTR_IW_CXGBE, "%s:pmr7 %p", __func__, ep); 1608 err = -EPROTO; 1609 goto err_stop_timer; 1610 } 1611 1612 plen = ntohs(mpa->private_data_size); 1613 1614 /* 1615 * Fail if there's too much private data. 1616 */ 1617 if (plen > MPA_MAX_PRIVATE_DATA) { 1618 1619 CTR2(KTR_IW_CXGBE, "%s:pmr8 %p", __func__, ep); 1620 err = -EPROTO; 1621 goto err_stop_timer; 1622 } 1623 1624 /* 1625 * If plen does not account for pkt size 1626 */ 1627 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { 1628 1629 CTR2(KTR_IW_CXGBE, "%s:pmr9 %p", __func__, ep); 1630 STOP_EP_TIMER(ep); 1631 err = -EPROTO; 1632 goto err_stop_timer; 1633 } 1634 1635 ep->plen = (u8) plen; 1636 1637 /* 1638 * If we don't have all the pdata yet, then bail. 1639 * We'll continue process when more data arrives. 1640 */ 1641 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) { 1642 1643 CTR2(KTR_IW_CXGBE, "%s:pmra %p", __func__, ep); 1644 return 0; 1645 } 1646 1647 if (mpa->flags & MPA_REJECT) { 1648 1649 CTR2(KTR_IW_CXGBE, "%s:pmrb %p", __func__, ep); 1650 err = -ECONNREFUSED; 1651 goto err_stop_timer; 1652 } 1653 1654 /* 1655 * If we get here we have accumulated the entire mpa 1656 * start reply message including private data. And 1657 * the MPA header is valid. 1658 */ 1659 state_set(&ep->com, FPDU_MODE); 1660 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 1661 ep->mpa_attr.recv_marker_enabled = markers_enabled; 1662 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 1663 ep->mpa_attr.version = mpa->revision; 1664 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1665 1666 if (mpa->revision == 2) { 1667 1668 CTR2(KTR_IW_CXGBE, "%s:pmrc %p", __func__, ep); 1669 ep->mpa_attr.enhanced_rdma_conn = 1670 mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; 1671 1672 if (ep->mpa_attr.enhanced_rdma_conn) { 1673 1674 CTR2(KTR_IW_CXGBE, "%s:pmrd %p", __func__, ep); 1675 mpa_v2_params = (struct mpa_v2_conn_params *) 1676 (ep->mpa_pkt + sizeof(*mpa)); 1677 resp_ird = ntohs(mpa_v2_params->ird) & 1678 MPA_V2_IRD_ORD_MASK; 1679 resp_ord = ntohs(mpa_v2_params->ord) & 1680 MPA_V2_IRD_ORD_MASK; 1681 1682 /* 1683 * This is a double-check. Ideally, below checks are 1684 * not required since ird/ord stuff has been taken 1685 * care of in c4iw_accept_cr 1686 */ 1687 if ((ep->ird < resp_ord) || (ep->ord > resp_ird)) { 1688 1689 CTR2(KTR_IW_CXGBE, "%s:pmre %p", __func__, ep); 1690 err = -ENOMEM; 1691 ep->ird = resp_ord; 1692 ep->ord = resp_ird; 1693 insuff_ird = 1; 1694 } 1695 1696 if (ntohs(mpa_v2_params->ird) & 1697 MPA_V2_PEER2PEER_MODEL) { 1698 1699 CTR2(KTR_IW_CXGBE, "%s:pmrf %p", __func__, ep); 1700 if (ntohs(mpa_v2_params->ord) & 1701 MPA_V2_RDMA_WRITE_RTR) { 1702 1703 CTR2(KTR_IW_CXGBE, "%s:pmrg %p", __func__, ep); 1704 ep->mpa_attr.p2p_type = 1705 FW_RI_INIT_P2PTYPE_RDMA_WRITE; 1706 } 1707 else if (ntohs(mpa_v2_params->ord) & 1708 MPA_V2_RDMA_READ_RTR) { 1709 1710 CTR2(KTR_IW_CXGBE, "%s:pmrh %p", __func__, ep); 1711 ep->mpa_attr.p2p_type = 1712 FW_RI_INIT_P2PTYPE_READ_REQ; 1713 } 1714 } 1715 } 1716 } else { 1717 1718 CTR2(KTR_IW_CXGBE, "%s:pmri %p", __func__, ep); 1719 1720 if (mpa->revision == 1) { 1721 1722 CTR2(KTR_IW_CXGBE, "%s:pmrj %p", __func__, ep); 1723 1724 if (peer2peer) { 1725 1726 CTR2(KTR_IW_CXGBE, "%s:pmrk %p", __func__, ep); 1727 ep->mpa_attr.p2p_type = p2p_type; 1728 } 1729 } 1730 } 1731 1732 if (set_tcpinfo(ep)) { 1733 1734 CTR2(KTR_IW_CXGBE, "%s:pmrl %p", __func__, ep); 1735 printf("%s set_tcpinfo error\n", __func__); 1736 err = -ECONNRESET; 1737 goto err; 1738 } 1739 1740 CTR6(KTR_IW_CXGBE, "%s - crc_enabled = %d, recv_marker_enabled = %d, " 1741 "xmit_marker_enabled = %d, version = %d p2p_type = %d", __func__, 1742 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled, 1743 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version, 1744 ep->mpa_attr.p2p_type); 1745 1746 /* 1747 * If responder's RTR does not match with that of initiator, assign 1748 * FW_RI_INIT_P2PTYPE_DISABLED in mpa attributes so that RTR is not 1749 * generated when moving QP to RTS state. 1750 * A TERM message will be sent after QP has moved to RTS state 1751 */ 1752 if ((ep->mpa_attr.version == 2) && peer2peer && 1753 (ep->mpa_attr.p2p_type != p2p_type)) { 1754 1755 CTR2(KTR_IW_CXGBE, "%s:pmrm %p", __func__, ep); 1756 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1757 rtr_mismatch = 1; 1758 } 1759 1760 1761 //ep->ofld_txq = TOEPCB(ep->com.so)->ofld_txq; 1762 attrs.mpa_attr = ep->mpa_attr; 1763 attrs.max_ird = ep->ird; 1764 attrs.max_ord = ep->ord; 1765 attrs.llp_stream_handle = ep; 1766 attrs.next_state = C4IW_QP_STATE_RTS; 1767 1768 mask = C4IW_QP_ATTR_NEXT_STATE | 1769 C4IW_QP_ATTR_LLP_STREAM_HANDLE | C4IW_QP_ATTR_MPA_ATTR | 1770 C4IW_QP_ATTR_MAX_IRD | C4IW_QP_ATTR_MAX_ORD; 1771 1772 /* bind QP and TID with INIT_WR */ 1773 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, mask, &attrs, 1); 1774 1775 if (err) { 1776 1777 CTR2(KTR_IW_CXGBE, "%s:pmrn %p", __func__, ep); 1778 goto err; 1779 } 1780 1781 /* 1782 * If responder's RTR requirement did not match with what initiator 1783 * supports, generate TERM message 1784 */ 1785 if (rtr_mismatch) { 1786 1787 CTR2(KTR_IW_CXGBE, "%s:pmro %p", __func__, ep); 1788 printk(KERN_ERR "%s: RTR mismatch, sending TERM\n", __func__); 1789 attrs.layer_etype = LAYER_MPA | DDP_LLP; 1790 attrs.ecode = MPA_NOMATCH_RTR; 1791 attrs.next_state = C4IW_QP_STATE_TERMINATE; 1792 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1793 C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); 1794 err = -ENOMEM; 1795 disconnect = 1; 1796 goto out; 1797 } 1798 1799 /* 1800 * Generate TERM if initiator IRD is not sufficient for responder 1801 * provided ORD. Currently, we do the same behaviour even when 1802 * responder provided IRD is also not sufficient as regards to 1803 * initiator ORD. 1804 */ 1805 if (insuff_ird) { 1806 1807 CTR2(KTR_IW_CXGBE, "%s:pmrp %p", __func__, ep); 1808 printk(KERN_ERR "%s: Insufficient IRD, sending TERM\n", 1809 __func__); 1810 attrs.layer_etype = LAYER_MPA | DDP_LLP; 1811 attrs.ecode = MPA_INSUFF_IRD; 1812 attrs.next_state = C4IW_QP_STATE_TERMINATE; 1813 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1814 C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); 1815 err = -ENOMEM; 1816 disconnect = 1; 1817 goto out; 1818 } 1819 goto out; 1820 err_stop_timer: 1821 STOP_EP_TIMER(ep); 1822 err: 1823 disconnect = 2; 1824 out: 1825 connect_reply_upcall(ep, err); 1826 CTR2(KTR_IW_CXGBE, "%s:pmrE %p", __func__, ep); 1827 return disconnect; 1828 } 1829 1830 /* 1831 * process_mpa_request - process streaming mode MPA request 1832 * 1833 * Returns: 1834 * 1835 * 0 upon success indicating a connect request was delivered to the ULP 1836 * or the mpa request is incomplete but valid so far. 1837 * 1838 * 1 if a failure requires the caller to close the connection. 1839 * 1840 * 2 if a failure requires the caller to abort the connection. 1841 */ 1842 static int 1843 process_mpa_request(struct c4iw_ep *ep) 1844 { 1845 struct mpa_message *mpa; 1846 u16 plen; 1847 int flags = MSG_DONTWAIT; 1848 int rc; 1849 struct iovec iov; 1850 struct uio uio; 1851 enum c4iw_ep_state state = state_read(&ep->com); 1852 1853 CTR3(KTR_IW_CXGBE, "%s: ep %p, state %s", __func__, ep, states[state]); 1854 1855 if (state != MPA_REQ_WAIT) 1856 return 0; 1857 1858 iov.iov_base = &ep->mpa_pkt[ep->mpa_pkt_len]; 1859 iov.iov_len = sizeof(ep->mpa_pkt) - ep->mpa_pkt_len; 1860 uio.uio_iov = &iov; 1861 uio.uio_iovcnt = 1; 1862 uio.uio_offset = 0; 1863 uio.uio_resid = sizeof(ep->mpa_pkt) - ep->mpa_pkt_len; 1864 uio.uio_segflg = UIO_SYSSPACE; 1865 uio.uio_rw = UIO_READ; 1866 uio.uio_td = NULL; /* uio.uio_td = ep->com.thread; */ 1867 1868 rc = soreceive(ep->com.so, NULL, &uio, NULL, NULL, &flags); 1869 if (rc == EAGAIN) 1870 return 0; 1871 else if (rc) 1872 goto err_stop_timer; 1873 1874 KASSERT(uio.uio_offset > 0, ("%s: sorecieve on so %p read no data", 1875 __func__, ep->com.so)); 1876 ep->mpa_pkt_len += uio.uio_offset; 1877 1878 /* 1879 * If we get more than the supported amount of private data then we must 1880 * fail this connection. XXX: check so_rcv->sb_cc, or peek with another 1881 * soreceive, or increase the size of mpa_pkt by 1 and abort if the last 1882 * byte is filled by the soreceive above. 1883 */ 1884 1885 /* Don't even have the MPA message. Wait for more data to arrive. */ 1886 if (ep->mpa_pkt_len < sizeof(*mpa)) 1887 return 0; 1888 mpa = (struct mpa_message *) ep->mpa_pkt; 1889 1890 /* 1891 * Validate MPA Header. 1892 */ 1893 if (mpa->revision > mpa_rev) { 1894 log(LOG_ERR, "%s: MPA version mismatch. Local = %d," 1895 " Received = %d\n", __func__, mpa_rev, mpa->revision); 1896 goto err_stop_timer; 1897 } 1898 1899 if (memcmp(mpa->key, MPA_KEY_REQ, sizeof(mpa->key))) 1900 goto err_stop_timer; 1901 1902 /* 1903 * Fail if there's too much private data. 1904 */ 1905 plen = ntohs(mpa->private_data_size); 1906 if (plen > MPA_MAX_PRIVATE_DATA) 1907 goto err_stop_timer; 1908 1909 /* 1910 * If plen does not account for pkt size 1911 */ 1912 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) 1913 goto err_stop_timer; 1914 1915 ep->plen = (u8) plen; 1916 1917 /* 1918 * If we don't have all the pdata yet, then bail. 1919 */ 1920 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) 1921 return 0; 1922 1923 /* 1924 * If we get here we have accumulated the entire mpa 1925 * start reply message including private data. 1926 */ 1927 ep->mpa_attr.initiator = 0; 1928 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 1929 ep->mpa_attr.recv_marker_enabled = markers_enabled; 1930 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 1931 ep->mpa_attr.version = mpa->revision; 1932 if (mpa->revision == 1) 1933 ep->tried_with_mpa_v1 = 1; 1934 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1935 1936 if (mpa->revision == 2) { 1937 ep->mpa_attr.enhanced_rdma_conn = 1938 mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; 1939 if (ep->mpa_attr.enhanced_rdma_conn) { 1940 struct mpa_v2_conn_params *mpa_v2_params; 1941 u16 ird, ord; 1942 1943 mpa_v2_params = (void *)&ep->mpa_pkt[sizeof(*mpa)]; 1944 ird = ntohs(mpa_v2_params->ird); 1945 ord = ntohs(mpa_v2_params->ord); 1946 1947 ep->ird = ird & MPA_V2_IRD_ORD_MASK; 1948 ep->ord = ord & MPA_V2_IRD_ORD_MASK; 1949 if (ird & MPA_V2_PEER2PEER_MODEL && peer2peer) { 1950 if (ord & MPA_V2_RDMA_WRITE_RTR) { 1951 ep->mpa_attr.p2p_type = 1952 FW_RI_INIT_P2PTYPE_RDMA_WRITE; 1953 } else if (ord & MPA_V2_RDMA_READ_RTR) { 1954 ep->mpa_attr.p2p_type = 1955 FW_RI_INIT_P2PTYPE_READ_REQ; 1956 } 1957 } 1958 } 1959 } else if (mpa->revision == 1 && peer2peer) 1960 ep->mpa_attr.p2p_type = p2p_type; 1961 1962 if (set_tcpinfo(ep)) 1963 goto err_stop_timer; 1964 1965 CTR5(KTR_IW_CXGBE, "%s: crc_enabled = %d, recv_marker_enabled = %d, " 1966 "xmit_marker_enabled = %d, version = %d", __func__, 1967 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled, 1968 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version); 1969 1970 state_set(&ep->com, MPA_REQ_RCVD); 1971 STOP_EP_TIMER(ep); 1972 1973 /* drive upcall */ 1974 mutex_lock(&ep->parent_ep->com.mutex); 1975 if (ep->parent_ep->com.state != DEAD) { 1976 if (connect_request_upcall(ep)) 1977 goto err_unlock_parent; 1978 } else 1979 goto err_unlock_parent; 1980 mutex_unlock(&ep->parent_ep->com.mutex); 1981 return 0; 1982 1983 err_unlock_parent: 1984 mutex_unlock(&ep->parent_ep->com.mutex); 1985 goto err_out; 1986 err_stop_timer: 1987 STOP_EP_TIMER(ep); 1988 err_out: 1989 return 2; 1990 } 1991 1992 /* 1993 * Upcall from the adapter indicating data has been transmitted. 1994 * For us its just the single MPA request or reply. We can now free 1995 * the skb holding the mpa message. 1996 */ 1997 int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) 1998 { 1999 int err; 2000 struct c4iw_ep *ep = to_ep(cm_id); 2001 CTR2(KTR_IW_CXGBE, "%s:crcB %p", __func__, ep); 2002 int abort = 0; 2003 2004 if ((state_read(&ep->com) == DEAD) || 2005 (state_read(&ep->com) != MPA_REQ_RCVD)) { 2006 2007 CTR2(KTR_IW_CXGBE, "%s:crc1 %p", __func__, ep); 2008 c4iw_put_ep(&ep->com); 2009 return -ECONNRESET; 2010 } 2011 set_bit(ULP_REJECT, &ep->com.history); 2012 2013 if (mpa_rev == 0) { 2014 2015 CTR2(KTR_IW_CXGBE, "%s:crc2 %p", __func__, ep); 2016 abort = 1; 2017 } 2018 else { 2019 2020 CTR2(KTR_IW_CXGBE, "%s:crc3 %p", __func__, ep); 2021 abort = send_mpa_reject(ep, pdata, pdata_len); 2022 } 2023 stop_ep_timer(ep); 2024 err = c4iw_ep_disconnect(ep, abort != 0, GFP_KERNEL); 2025 c4iw_put_ep(&ep->com); 2026 CTR3(KTR_IW_CXGBE, "%s:crc4 %p, err: %d", __func__, ep, err); 2027 return 0; 2028 } 2029 2030 int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 2031 { 2032 int err; 2033 struct c4iw_qp_attributes attrs; 2034 enum c4iw_qp_attr_mask mask; 2035 struct c4iw_ep *ep = to_ep(cm_id); 2036 struct c4iw_dev *h = to_c4iw_dev(cm_id->device); 2037 struct c4iw_qp *qp = get_qhp(h, conn_param->qpn); 2038 int abort = 0; 2039 2040 CTR2(KTR_IW_CXGBE, "%s:cacB %p", __func__, ep); 2041 2042 if (state_read(&ep->com) == DEAD) { 2043 2044 CTR2(KTR_IW_CXGBE, "%s:cac1 %p", __func__, ep); 2045 err = -ECONNRESET; 2046 goto err_out; 2047 } 2048 2049 BUG_ON(state_read(&ep->com) != MPA_REQ_RCVD); 2050 BUG_ON(!qp); 2051 2052 set_bit(ULP_ACCEPT, &ep->com.history); 2053 2054 if ((conn_param->ord > c4iw_max_read_depth) || 2055 (conn_param->ird > c4iw_max_read_depth)) { 2056 2057 CTR2(KTR_IW_CXGBE, "%s:cac2 %p", __func__, ep); 2058 err = -EINVAL; 2059 goto err_abort; 2060 } 2061 2062 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 2063 2064 CTR2(KTR_IW_CXGBE, "%s:cac3 %p", __func__, ep); 2065 2066 if (conn_param->ord > ep->ird) { 2067 2068 CTR2(KTR_IW_CXGBE, "%s:cac4 %p", __func__, ep); 2069 ep->ird = conn_param->ird; 2070 ep->ord = conn_param->ord; 2071 send_mpa_reject(ep, conn_param->private_data, 2072 conn_param->private_data_len); 2073 err = -ENOMEM; 2074 goto err_abort; 2075 } 2076 2077 if (conn_param->ird > ep->ord) { 2078 2079 CTR2(KTR_IW_CXGBE, "%s:cac5 %p", __func__, ep); 2080 2081 if (!ep->ord) { 2082 2083 CTR2(KTR_IW_CXGBE, "%s:cac6 %p", __func__, ep); 2084 conn_param->ird = 1; 2085 } 2086 else { 2087 CTR2(KTR_IW_CXGBE, "%s:cac7 %p", __func__, ep); 2088 err = -ENOMEM; 2089 goto err_abort; 2090 } 2091 } 2092 2093 } 2094 ep->ird = conn_param->ird; 2095 ep->ord = conn_param->ord; 2096 2097 if (ep->mpa_attr.version != 2) { 2098 2099 CTR2(KTR_IW_CXGBE, "%s:cac8 %p", __func__, ep); 2100 2101 if (peer2peer && ep->ird == 0) { 2102 2103 CTR2(KTR_IW_CXGBE, "%s:cac9 %p", __func__, ep); 2104 ep->ird = 1; 2105 } 2106 } 2107 2108 2109 ep->com.cm_id = cm_id; 2110 ref_cm_id(&ep->com); 2111 ep->com.qp = qp; 2112 ref_qp(ep); 2113 //ep->ofld_txq = TOEPCB(ep->com.so)->ofld_txq; 2114 2115 /* bind QP to EP and move to RTS */ 2116 attrs.mpa_attr = ep->mpa_attr; 2117 attrs.max_ird = ep->ird; 2118 attrs.max_ord = ep->ord; 2119 attrs.llp_stream_handle = ep; 2120 attrs.next_state = C4IW_QP_STATE_RTS; 2121 2122 /* bind QP and TID with INIT_WR */ 2123 mask = C4IW_QP_ATTR_NEXT_STATE | 2124 C4IW_QP_ATTR_LLP_STREAM_HANDLE | 2125 C4IW_QP_ATTR_MPA_ATTR | 2126 C4IW_QP_ATTR_MAX_IRD | 2127 C4IW_QP_ATTR_MAX_ORD; 2128 2129 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, mask, &attrs, 1); 2130 2131 if (err) { 2132 2133 CTR2(KTR_IW_CXGBE, "%s:caca %p", __func__, ep); 2134 goto err_defef_cm_id; 2135 } 2136 err = send_mpa_reply(ep, conn_param->private_data, 2137 conn_param->private_data_len); 2138 2139 if (err) { 2140 2141 CTR2(KTR_IW_CXGBE, "%s:caca %p", __func__, ep); 2142 goto err_defef_cm_id; 2143 } 2144 2145 state_set(&ep->com, FPDU_MODE); 2146 established_upcall(ep); 2147 c4iw_put_ep(&ep->com); 2148 CTR2(KTR_IW_CXGBE, "%s:cacE %p", __func__, ep); 2149 return 0; 2150 err_defef_cm_id: 2151 deref_cm_id(&ep->com); 2152 err_abort: 2153 abort = 1; 2154 err_out: 2155 if (abort) 2156 c4iw_ep_disconnect(ep, 1, GFP_KERNEL); 2157 c4iw_put_ep(&ep->com); 2158 CTR2(KTR_IW_CXGBE, "%s:cacE err %p", __func__, ep); 2159 return err; 2160 } 2161 2162 2163 2164 int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 2165 { 2166 int err = 0; 2167 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); 2168 struct c4iw_ep *ep = NULL; 2169 struct nhop4_extended nh4; 2170 2171 CTR2(KTR_IW_CXGBE, "%s:ccB %p", __func__, cm_id); 2172 2173 if ((conn_param->ord > c4iw_max_read_depth) || 2174 (conn_param->ird > c4iw_max_read_depth)) { 2175 2176 CTR2(KTR_IW_CXGBE, "%s:cc1 %p", __func__, cm_id); 2177 err = -EINVAL; 2178 goto out; 2179 } 2180 ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 2181 init_timer(&ep->timer); 2182 ep->plen = conn_param->private_data_len; 2183 2184 if (ep->plen) { 2185 2186 CTR2(KTR_IW_CXGBE, "%s:cc3 %p", __func__, ep); 2187 memcpy(ep->mpa_pkt + sizeof(struct mpa_message), 2188 conn_param->private_data, ep->plen); 2189 } 2190 ep->ird = conn_param->ird; 2191 ep->ord = conn_param->ord; 2192 2193 if (peer2peer && ep->ord == 0) { 2194 2195 CTR2(KTR_IW_CXGBE, "%s:cc4 %p", __func__, ep); 2196 ep->ord = 1; 2197 } 2198 2199 ep->com.dev = dev; 2200 ep->com.cm_id = cm_id; 2201 ref_cm_id(&ep->com); 2202 ep->com.qp = get_qhp(dev, conn_param->qpn); 2203 2204 if (!ep->com.qp) { 2205 2206 CTR2(KTR_IW_CXGBE, "%s:cc5 %p", __func__, ep); 2207 err = -EINVAL; 2208 goto fail2; 2209 } 2210 ref_qp(ep); 2211 ep->com.thread = curthread; 2212 ep->com.so = cm_id->so; 2213 2214 /* find a route */ 2215 err = find_route( 2216 cm_id->local_addr.sin_addr.s_addr, 2217 cm_id->remote_addr.sin_addr.s_addr, 2218 cm_id->local_addr.sin_port, 2219 cm_id->remote_addr.sin_port, 0, &nh4); 2220 2221 if (err) { 2222 2223 CTR2(KTR_IW_CXGBE, "%s:cc7 %p", __func__, ep); 2224 printk(KERN_ERR MOD "%s - cannot find route.\n", __func__); 2225 err = -EHOSTUNREACH; 2226 goto fail2; 2227 } 2228 2229 if (!(nh4.nh_ifp->if_capenable & IFCAP_TOE) || 2230 TOEDEV(nh4.nh_ifp) == NULL) { 2231 err = -ENOPROTOOPT; 2232 goto fail3; 2233 } 2234 fib4_free_nh_ext(RT_DEFAULT_FIB, &nh4); 2235 2236 state_set(&ep->com, CONNECTING); 2237 ep->tos = 0; 2238 ep->com.local_addr = cm_id->local_addr; 2239 ep->com.remote_addr = cm_id->remote_addr; 2240 err = -soconnect(ep->com.so, (struct sockaddr *)&ep->com.remote_addr, 2241 ep->com.thread); 2242 2243 if (!err) { 2244 init_iwarp_socket(cm_id->so, &ep->com); 2245 goto out; 2246 } else { 2247 goto fail2; 2248 } 2249 2250 fail3: 2251 fib4_free_nh_ext(RT_DEFAULT_FIB, &nh4); 2252 fail2: 2253 deref_cm_id(&ep->com); 2254 c4iw_put_ep(&ep->com); 2255 ep = NULL; /* CTR shouldn't display already-freed ep. */ 2256 out: 2257 CTR2(KTR_IW_CXGBE, "%s:ccE %p", __func__, ep); 2258 return err; 2259 } 2260 2261 /* 2262 * iwcm->create_listen_ep. Returns -errno on failure. 2263 */ 2264 int 2265 c4iw_create_listen_ep(struct iw_cm_id *cm_id, int backlog) 2266 { 2267 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); 2268 struct c4iw_listen_ep *ep; 2269 struct socket *so = cm_id->so; 2270 2271 ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 2272 ep->com.cm_id = cm_id; 2273 ref_cm_id(&ep->com); 2274 ep->com.dev = dev; 2275 ep->backlog = backlog; 2276 ep->com.local_addr = cm_id->local_addr; 2277 ep->com.thread = curthread; 2278 state_set(&ep->com, LISTEN); 2279 ep->com.so = so; 2280 2281 cm_id->provider_data = ep; 2282 return (0); 2283 } 2284 2285 void 2286 c4iw_destroy_listen_ep(struct iw_cm_id *cm_id) 2287 { 2288 struct c4iw_listen_ep *ep = to_listen_ep(cm_id); 2289 2290 CTR4(KTR_IW_CXGBE, "%s: cm_id %p, so %p, state %s", __func__, cm_id, 2291 cm_id->so, states[ep->com.state]); 2292 2293 state_set(&ep->com, DEAD); 2294 deref_cm_id(&ep->com); 2295 c4iw_put_ep(&ep->com); 2296 2297 return; 2298 } 2299 2300 int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp) 2301 { 2302 int ret = 0; 2303 int close = 0; 2304 int fatal = 0; 2305 struct c4iw_rdev *rdev; 2306 2307 mutex_lock(&ep->com.mutex); 2308 2309 CTR2(KTR_IW_CXGBE, "%s:cedB %p", __func__, ep); 2310 2311 rdev = &ep->com.dev->rdev; 2312 2313 if (c4iw_fatal_error(rdev)) { 2314 2315 CTR2(KTR_IW_CXGBE, "%s:ced1 %p", __func__, ep); 2316 fatal = 1; 2317 close_complete_upcall(ep, -ECONNRESET); 2318 ep->com.state = DEAD; 2319 } 2320 CTR3(KTR_IW_CXGBE, "%s:ced2 %p %s", __func__, ep, 2321 states[ep->com.state]); 2322 2323 switch (ep->com.state) { 2324 2325 case MPA_REQ_WAIT: 2326 case MPA_REQ_SENT: 2327 case MPA_REQ_RCVD: 2328 case MPA_REP_SENT: 2329 case FPDU_MODE: 2330 close = 1; 2331 if (abrupt) 2332 ep->com.state = ABORTING; 2333 else { 2334 ep->com.state = CLOSING; 2335 START_EP_TIMER(ep); 2336 } 2337 set_bit(CLOSE_SENT, &ep->com.flags); 2338 break; 2339 2340 case CLOSING: 2341 2342 if (!test_and_set_bit(CLOSE_SENT, &ep->com.flags)) { 2343 2344 close = 1; 2345 if (abrupt) { 2346 STOP_EP_TIMER(ep); 2347 ep->com.state = ABORTING; 2348 } else 2349 ep->com.state = MORIBUND; 2350 } 2351 break; 2352 2353 case MORIBUND: 2354 case ABORTING: 2355 case DEAD: 2356 CTR3(KTR_IW_CXGBE, 2357 "%s ignoring disconnect ep %p state %u", __func__, 2358 ep, ep->com.state); 2359 break; 2360 2361 default: 2362 BUG(); 2363 break; 2364 } 2365 2366 mutex_unlock(&ep->com.mutex); 2367 2368 if (close) { 2369 2370 CTR2(KTR_IW_CXGBE, "%s:ced3 %p", __func__, ep); 2371 2372 if (abrupt) { 2373 2374 CTR2(KTR_IW_CXGBE, "%s:ced4 %p", __func__, ep); 2375 set_bit(EP_DISC_ABORT, &ep->com.history); 2376 close_complete_upcall(ep, -ECONNRESET); 2377 ret = send_abort(ep); 2378 if (ret) 2379 fatal = 1; 2380 } else { 2381 2382 CTR2(KTR_IW_CXGBE, "%s:ced5 %p", __func__, ep); 2383 set_bit(EP_DISC_CLOSE, &ep->com.history); 2384 2385 if (!ep->parent_ep) 2386 __state_set(&ep->com, MORIBUND); 2387 sodisconnect(ep->com.so); 2388 } 2389 2390 } 2391 2392 if (fatal) { 2393 set_bit(EP_DISC_FAIL, &ep->com.history); 2394 if (!abrupt) { 2395 STOP_EP_TIMER(ep); 2396 close_complete_upcall(ep, -EIO); 2397 } 2398 if (ep->com.qp) { 2399 struct c4iw_qp_attributes attrs; 2400 2401 attrs.next_state = C4IW_QP_STATE_ERROR; 2402 ret = c4iw_modify_qp(ep->com.dev, ep->com.qp, 2403 C4IW_QP_ATTR_NEXT_STATE, 2404 &attrs, 1); 2405 if (ret) { 2406 CTR2(KTR_IW_CXGBE, "%s:ced7 %p", __func__, ep); 2407 printf("%s - qp <- error failed!\n", __func__); 2408 } 2409 } 2410 release_ep_resources(ep); 2411 ep->com.state = DEAD; 2412 CTR2(KTR_IW_CXGBE, "%s:ced6 %p", __func__, ep); 2413 } 2414 CTR2(KTR_IW_CXGBE, "%s:cedE %p", __func__, ep); 2415 return ret; 2416 } 2417 2418 #ifdef C4IW_EP_REDIRECT 2419 int c4iw_ep_redirect(void *ctx, struct dst_entry *old, struct dst_entry *new, 2420 struct l2t_entry *l2t) 2421 { 2422 struct c4iw_ep *ep = ctx; 2423 2424 if (ep->dst != old) 2425 return 0; 2426 2427 PDBG("%s ep %p redirect to dst %p l2t %p\n", __func__, ep, new, 2428 l2t); 2429 dst_hold(new); 2430 cxgb4_l2t_release(ep->l2t); 2431 ep->l2t = l2t; 2432 dst_release(old); 2433 ep->dst = new; 2434 return 1; 2435 } 2436 #endif 2437 2438 2439 2440 static void ep_timeout(unsigned long arg) 2441 { 2442 struct c4iw_ep *ep = (struct c4iw_ep *)arg; 2443 2444 if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) { 2445 2446 /* 2447 * Only insert if it is not already on the list. 2448 */ 2449 if (!(ep->com.ep_events & C4IW_EVENT_TIMEOUT)) { 2450 CTR2(KTR_IW_CXGBE, "%s:et1 %p", __func__, ep); 2451 add_ep_to_req_list(ep, C4IW_EVENT_TIMEOUT); 2452 } 2453 } 2454 } 2455 2456 static int fw6_wr_rpl(struct adapter *sc, const __be64 *rpl) 2457 { 2458 uint64_t val = be64toh(*rpl); 2459 int ret; 2460 struct c4iw_wr_wait *wr_waitp; 2461 2462 ret = (int)((val >> 8) & 0xff); 2463 wr_waitp = (struct c4iw_wr_wait *)rpl[1]; 2464 CTR3(KTR_IW_CXGBE, "%s wr_waitp %p ret %u", __func__, wr_waitp, ret); 2465 if (wr_waitp) 2466 c4iw_wake_up(wr_waitp, ret ? -ret : 0); 2467 2468 return (0); 2469 } 2470 2471 static int fw6_cqe_handler(struct adapter *sc, const __be64 *rpl) 2472 { 2473 struct cqe_list_entry *cle; 2474 unsigned long flag; 2475 2476 cle = malloc(sizeof(*cle), M_CXGBE, M_NOWAIT); 2477 cle->rhp = sc->iwarp_softc; 2478 cle->err_cqe = *(const struct t4_cqe *)(&rpl[0]); 2479 2480 spin_lock_irqsave(&err_cqe_lock, flag); 2481 list_add_tail(&cle->entry, &err_cqe_list); 2482 queue_work(c4iw_taskq, &c4iw_task); 2483 spin_unlock_irqrestore(&err_cqe_lock, flag); 2484 2485 return (0); 2486 } 2487 2488 static int 2489 process_terminate(struct c4iw_ep *ep) 2490 { 2491 struct c4iw_qp_attributes attrs; 2492 2493 CTR2(KTR_IW_CXGBE, "%s:tB %p %d", __func__, ep); 2494 2495 if (ep && ep->com.qp) { 2496 2497 printk(KERN_WARNING MOD "TERM received tid %u qpid %u\n", 2498 ep->hwtid, ep->com.qp->wq.sq.qid); 2499 attrs.next_state = C4IW_QP_STATE_TERMINATE; 2500 c4iw_modify_qp(ep->com.dev, ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 2501 1); 2502 } else 2503 printk(KERN_WARNING MOD "TERM received tid %u no ep/qp\n", 2504 ep->hwtid); 2505 CTR2(KTR_IW_CXGBE, "%s:tE %p %d", __func__, ep); 2506 2507 return 0; 2508 } 2509 2510 int __init c4iw_cm_init(void) 2511 { 2512 2513 t4_register_cpl_handler(CPL_RDMA_TERMINATE, terminate); 2514 t4_register_fw_msg_handler(FW6_TYPE_WR_RPL, fw6_wr_rpl); 2515 t4_register_fw_msg_handler(FW6_TYPE_CQE, fw6_cqe_handler); 2516 t4_register_an_handler(c4iw_ev_handler); 2517 2518 TAILQ_INIT(&req_list); 2519 spin_lock_init(&req_lock); 2520 INIT_LIST_HEAD(&err_cqe_list); 2521 spin_lock_init(&err_cqe_lock); 2522 2523 INIT_WORK(&c4iw_task, process_req); 2524 2525 c4iw_taskq = create_singlethread_workqueue("iw_cxgbe"); 2526 if (!c4iw_taskq) 2527 return -ENOMEM; 2528 2529 return 0; 2530 } 2531 2532 void __exit c4iw_cm_term(void) 2533 { 2534 WARN_ON(!TAILQ_EMPTY(&req_list)); 2535 WARN_ON(!list_empty(&err_cqe_list)); 2536 flush_workqueue(c4iw_taskq); 2537 destroy_workqueue(c4iw_taskq); 2538 2539 t4_register_cpl_handler(CPL_RDMA_TERMINATE, NULL); 2540 t4_register_fw_msg_handler(FW6_TYPE_WR_RPL, NULL); 2541 t4_register_fw_msg_handler(FW6_TYPE_CQE, NULL); 2542 t4_register_an_handler(NULL); 2543 } 2544 #endif 2545