1 /* 2 * Copyright (c) 2009-2013, 2016 Chelsio, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include "opt_inet.h" 36 37 #ifdef TCP_OFFLOAD 38 #include <sys/types.h> 39 #include <sys/malloc.h> 40 #include <sys/socket.h> 41 #include <sys/socketvar.h> 42 #include <sys/sockio.h> 43 #include <sys/taskqueue.h> 44 #include <netinet/in.h> 45 #include <net/route.h> 46 47 #include <netinet/in_systm.h> 48 #include <netinet/in_pcb.h> 49 #include <netinet/ip.h> 50 #include <netinet/in_fib.h> 51 #include <netinet/ip_var.h> 52 #include <netinet/tcp_var.h> 53 #include <netinet/tcp.h> 54 #include <netinet/tcpip.h> 55 56 #include <netinet/toecore.h> 57 58 struct sge_iq; 59 struct rss_header; 60 struct cpl_set_tcb_rpl; 61 #include <linux/types.h> 62 #include "offload.h" 63 #include "tom/t4_tom.h" 64 65 #define TOEPCB(so) ((struct toepcb *)(so_sototcpcb((so))->t_toe)) 66 67 #include "iw_cxgbe.h" 68 #include <linux/module.h> 69 #include <linux/workqueue.h> 70 #include <linux/notifier.h> 71 #include <linux/inetdevice.h> 72 #include <linux/if_vlan.h> 73 #include <net/netevent.h> 74 75 static spinlock_t req_lock; 76 static TAILQ_HEAD(c4iw_ep_list, c4iw_ep_common) req_list; 77 static struct work_struct c4iw_task; 78 static struct workqueue_struct *c4iw_taskq; 79 static LIST_HEAD(err_cqe_list); 80 static spinlock_t err_cqe_lock; 81 82 static void process_req(struct work_struct *ctx); 83 static void start_ep_timer(struct c4iw_ep *ep); 84 static int stop_ep_timer(struct c4iw_ep *ep); 85 static int set_tcpinfo(struct c4iw_ep *ep); 86 static void process_timeout(struct c4iw_ep *ep); 87 static void process_err_cqes(void); 88 static enum c4iw_ep_state state_read(struct c4iw_ep_common *epc); 89 static void __state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state tostate); 90 static void state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state tostate); 91 static void *alloc_ep(int size, gfp_t flags); 92 static int find_route(__be32 local_ip, __be32 peer_ip, __be16 local_port, 93 __be16 peer_port, u8 tos, struct nhop4_extended *pnh4); 94 static void close_socket(struct socket *so); 95 static int send_mpa_req(struct c4iw_ep *ep); 96 static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen); 97 static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen); 98 static void close_complete_upcall(struct c4iw_ep *ep, int status); 99 static int send_abort(struct c4iw_ep *ep); 100 static void peer_close_upcall(struct c4iw_ep *ep); 101 static void peer_abort_upcall(struct c4iw_ep *ep); 102 static void connect_reply_upcall(struct c4iw_ep *ep, int status); 103 static int connect_request_upcall(struct c4iw_ep *ep); 104 static void established_upcall(struct c4iw_ep *ep); 105 static int process_mpa_reply(struct c4iw_ep *ep); 106 static int process_mpa_request(struct c4iw_ep *ep); 107 static void process_peer_close(struct c4iw_ep *ep); 108 static void process_conn_error(struct c4iw_ep *ep); 109 static void process_close_complete(struct c4iw_ep *ep); 110 static void ep_timeout(unsigned long arg); 111 static void init_iwarp_socket(struct socket *so, void *arg); 112 static void uninit_iwarp_socket(struct socket *so); 113 static void process_data(struct c4iw_ep *ep); 114 static void process_connected(struct c4iw_ep *ep); 115 static int c4iw_so_upcall(struct socket *so, void *arg, int waitflag); 116 static void process_socket_event(struct c4iw_ep *ep); 117 static void release_ep_resources(struct c4iw_ep *ep); 118 static int process_terminate(struct c4iw_ep *ep); 119 static int terminate(struct sge_iq *iq, const struct rss_header *rss, 120 struct mbuf *m); 121 static int add_ep_to_req_list(struct c4iw_ep *ep, int ep_events); 122 #define START_EP_TIMER(ep) \ 123 do { \ 124 CTR3(KTR_IW_CXGBE, "start_ep_timer (%s:%d) ep %p", \ 125 __func__, __LINE__, (ep)); \ 126 start_ep_timer(ep); \ 127 } while (0) 128 129 #define STOP_EP_TIMER(ep) \ 130 ({ \ 131 CTR3(KTR_IW_CXGBE, "stop_ep_timer (%s:%d) ep %p", \ 132 __func__, __LINE__, (ep)); \ 133 stop_ep_timer(ep); \ 134 }) 135 136 #ifdef KTR 137 static char *states[] = { 138 "idle", 139 "listen", 140 "connecting", 141 "mpa_wait_req", 142 "mpa_req_sent", 143 "mpa_req_rcvd", 144 "mpa_rep_sent", 145 "fpdu_mode", 146 "aborting", 147 "closing", 148 "moribund", 149 "dead", 150 NULL, 151 }; 152 #endif 153 154 155 static void deref_cm_id(struct c4iw_ep_common *epc) 156 { 157 epc->cm_id->rem_ref(epc->cm_id); 158 epc->cm_id = NULL; 159 set_bit(CM_ID_DEREFED, &epc->history); 160 } 161 162 static void ref_cm_id(struct c4iw_ep_common *epc) 163 { 164 set_bit(CM_ID_REFED, &epc->history); 165 epc->cm_id->add_ref(epc->cm_id); 166 } 167 168 static void deref_qp(struct c4iw_ep *ep) 169 { 170 c4iw_qp_rem_ref(&ep->com.qp->ibqp); 171 clear_bit(QP_REFERENCED, &ep->com.flags); 172 set_bit(QP_DEREFED, &ep->com.history); 173 } 174 175 static void ref_qp(struct c4iw_ep *ep) 176 { 177 set_bit(QP_REFERENCED, &ep->com.flags); 178 set_bit(QP_REFED, &ep->com.history); 179 c4iw_qp_add_ref(&ep->com.qp->ibqp); 180 } 181 182 static void process_timeout(struct c4iw_ep *ep) 183 { 184 struct c4iw_qp_attributes attrs; 185 int abort = 1; 186 187 mutex_lock(&ep->com.mutex); 188 CTR4(KTR_IW_CXGBE, "%s ep :%p, tid:%u, state %d", __func__, 189 ep, ep->hwtid, ep->com.state); 190 set_bit(TIMEDOUT, &ep->com.history); 191 switch (ep->com.state) { 192 case MPA_REQ_SENT: 193 connect_reply_upcall(ep, -ETIMEDOUT); 194 break; 195 case MPA_REQ_WAIT: 196 case MPA_REQ_RCVD: 197 case MPA_REP_SENT: 198 case FPDU_MODE: 199 break; 200 case CLOSING: 201 case MORIBUND: 202 if (ep->com.cm_id && ep->com.qp) { 203 attrs.next_state = C4IW_QP_STATE_ERROR; 204 c4iw_modify_qp(ep->com.dev, ep->com.qp, 205 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 206 } 207 close_complete_upcall(ep, -ETIMEDOUT); 208 break; 209 case ABORTING: 210 case DEAD: 211 /* 212 * These states are expected if the ep timed out at the same 213 * time as another thread was calling stop_ep_timer(). 214 * So we silently do nothing for these states. 215 */ 216 abort = 0; 217 break; 218 default: 219 CTR4(KTR_IW_CXGBE, "%s unexpected state ep %p tid %u state %u\n" 220 , __func__, ep, ep->hwtid, ep->com.state); 221 abort = 0; 222 } 223 mutex_unlock(&ep->com.mutex); 224 if (abort) 225 c4iw_ep_disconnect(ep, 1, GFP_KERNEL); 226 c4iw_put_ep(&ep->com); 227 return; 228 } 229 230 struct cqe_list_entry { 231 struct list_head entry; 232 struct c4iw_dev *rhp; 233 struct t4_cqe err_cqe; 234 }; 235 236 static void 237 process_err_cqes(void) 238 { 239 unsigned long flag; 240 struct cqe_list_entry *cle; 241 242 spin_lock_irqsave(&err_cqe_lock, flag); 243 while (!list_empty(&err_cqe_list)) { 244 struct list_head *tmp; 245 tmp = err_cqe_list.next; 246 list_del(tmp); 247 tmp->next = tmp->prev = NULL; 248 spin_unlock_irqrestore(&err_cqe_lock, flag); 249 cle = list_entry(tmp, struct cqe_list_entry, entry); 250 c4iw_ev_dispatch(cle->rhp, &cle->err_cqe); 251 free(cle, M_CXGBE); 252 spin_lock_irqsave(&err_cqe_lock, flag); 253 } 254 spin_unlock_irqrestore(&err_cqe_lock, flag); 255 256 return; 257 } 258 259 static void 260 process_req(struct work_struct *ctx) 261 { 262 struct c4iw_ep_common *epc; 263 unsigned long flag; 264 int ep_events; 265 266 process_err_cqes(); 267 spin_lock_irqsave(&req_lock, flag); 268 while (!TAILQ_EMPTY(&req_list)) { 269 epc = TAILQ_FIRST(&req_list); 270 TAILQ_REMOVE(&req_list, epc, entry); 271 epc->entry.tqe_prev = NULL; 272 ep_events = epc->ep_events; 273 epc->ep_events = 0; 274 spin_unlock_irqrestore(&req_lock, flag); 275 CTR4(KTR_IW_CXGBE, "%s: so %p, ep %p, events 0x%x", __func__, 276 epc->so, epc, ep_events); 277 if (ep_events & C4IW_EVENT_TERM) 278 process_terminate((struct c4iw_ep *)epc); 279 if (ep_events & C4IW_EVENT_TIMEOUT) 280 process_timeout((struct c4iw_ep *)epc); 281 if (ep_events & C4IW_EVENT_SOCKET) 282 process_socket_event((struct c4iw_ep *)epc); 283 c4iw_put_ep(epc); 284 process_err_cqes(); 285 spin_lock_irqsave(&req_lock, flag); 286 } 287 spin_unlock_irqrestore(&req_lock, flag); 288 } 289 290 /* 291 * XXX: doesn't belong here in the iWARP driver. 292 * XXX: assumes that the connection was offloaded by cxgbe/t4_tom if TF_TOE is 293 * set. Is this a valid assumption for active open? 294 */ 295 static int 296 set_tcpinfo(struct c4iw_ep *ep) 297 { 298 struct socket *so = ep->com.so; 299 struct inpcb *inp = sotoinpcb(so); 300 struct tcpcb *tp; 301 struct toepcb *toep; 302 int rc = 0; 303 304 INP_WLOCK(inp); 305 tp = intotcpcb(inp); 306 if ((tp->t_flags & TF_TOE) == 0) { 307 rc = EINVAL; 308 log(LOG_ERR, "%s: connection not offloaded (so %p, ep %p)\n", 309 __func__, so, ep); 310 goto done; 311 } 312 toep = TOEPCB(so); 313 314 ep->hwtid = toep->tid; 315 ep->snd_seq = tp->snd_nxt; 316 ep->rcv_seq = tp->rcv_nxt; 317 ep->emss = max(tp->t_maxseg, 128); 318 done: 319 INP_WUNLOCK(inp); 320 return (rc); 321 322 } 323 324 static int 325 find_route(__be32 local_ip, __be32 peer_ip, __be16 local_port, 326 __be16 peer_port, u8 tos, struct nhop4_extended *pnh4) 327 { 328 struct in_addr addr; 329 int err; 330 331 CTR5(KTR_IW_CXGBE, "%s:frtB %x, %x, %d, %d", __func__, local_ip, 332 peer_ip, ntohs(local_port), ntohs(peer_port)); 333 334 addr.s_addr = peer_ip; 335 err = fib4_lookup_nh_ext(RT_DEFAULT_FIB, addr, NHR_REF, 0, pnh4); 336 337 CTR2(KTR_IW_CXGBE, "%s:frtE %d", __func__, err); 338 return err; 339 } 340 341 static void 342 close_socket(struct socket *so) 343 { 344 345 uninit_iwarp_socket(so); 346 sodisconnect(so); 347 } 348 349 static void 350 process_peer_close(struct c4iw_ep *ep) 351 { 352 struct c4iw_qp_attributes attrs; 353 int disconnect = 1; 354 int release = 0; 355 356 CTR4(KTR_IW_CXGBE, "%s:ppcB ep %p so %p state %s", __func__, ep, 357 ep->com.so, states[ep->com.state]); 358 359 mutex_lock(&ep->com.mutex); 360 switch (ep->com.state) { 361 362 case MPA_REQ_WAIT: 363 CTR2(KTR_IW_CXGBE, "%s:ppc1 %p MPA_REQ_WAIT CLOSING", 364 __func__, ep); 365 __state_set(&ep->com, CLOSING); 366 break; 367 368 case MPA_REQ_SENT: 369 CTR2(KTR_IW_CXGBE, "%s:ppc2 %p MPA_REQ_SENT CLOSING", 370 __func__, ep); 371 __state_set(&ep->com, DEAD); 372 connect_reply_upcall(ep, -ECONNABORTED); 373 374 disconnect = 0; 375 STOP_EP_TIMER(ep); 376 close_socket(ep->com.so); 377 deref_cm_id(&ep->com); 378 release = 1; 379 break; 380 381 case MPA_REQ_RCVD: 382 383 /* 384 * We're gonna mark this puppy DEAD, but keep 385 * the reference on it until the ULP accepts or 386 * rejects the CR. 387 */ 388 CTR2(KTR_IW_CXGBE, "%s:ppc3 %p MPA_REQ_RCVD CLOSING", 389 __func__, ep); 390 __state_set(&ep->com, CLOSING); 391 c4iw_get_ep(&ep->com); 392 break; 393 394 case MPA_REP_SENT: 395 CTR2(KTR_IW_CXGBE, "%s:ppc4 %p MPA_REP_SENT CLOSING", 396 __func__, ep); 397 __state_set(&ep->com, CLOSING); 398 break; 399 400 case FPDU_MODE: 401 CTR2(KTR_IW_CXGBE, "%s:ppc5 %p FPDU_MODE CLOSING", 402 __func__, ep); 403 START_EP_TIMER(ep); 404 __state_set(&ep->com, CLOSING); 405 attrs.next_state = C4IW_QP_STATE_CLOSING; 406 c4iw_modify_qp(ep->com.dev, ep->com.qp, 407 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 408 peer_close_upcall(ep); 409 break; 410 411 case ABORTING: 412 CTR2(KTR_IW_CXGBE, "%s:ppc6 %p ABORTING (disconn)", 413 __func__, ep); 414 disconnect = 0; 415 break; 416 417 case CLOSING: 418 CTR2(KTR_IW_CXGBE, "%s:ppc7 %p CLOSING MORIBUND", 419 __func__, ep); 420 __state_set(&ep->com, MORIBUND); 421 disconnect = 0; 422 break; 423 424 case MORIBUND: 425 CTR2(KTR_IW_CXGBE, "%s:ppc8 %p MORIBUND DEAD", __func__, 426 ep); 427 STOP_EP_TIMER(ep); 428 if (ep->com.cm_id && ep->com.qp) { 429 attrs.next_state = C4IW_QP_STATE_IDLE; 430 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 431 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 432 } 433 close_socket(ep->com.so); 434 close_complete_upcall(ep, 0); 435 __state_set(&ep->com, DEAD); 436 release = 1; 437 disconnect = 0; 438 break; 439 440 case DEAD: 441 CTR2(KTR_IW_CXGBE, "%s:ppc9 %p DEAD (disconn)", 442 __func__, ep); 443 disconnect = 0; 444 break; 445 446 default: 447 panic("%s: ep %p state %d", __func__, ep, 448 ep->com.state); 449 break; 450 } 451 452 mutex_unlock(&ep->com.mutex); 453 454 if (disconnect) { 455 456 CTR2(KTR_IW_CXGBE, "%s:ppca %p", __func__, ep); 457 c4iw_ep_disconnect(ep, 0, M_NOWAIT); 458 } 459 if (release) { 460 461 CTR2(KTR_IW_CXGBE, "%s:ppcb %p", __func__, ep); 462 c4iw_put_ep(&ep->com); 463 } 464 CTR2(KTR_IW_CXGBE, "%s:ppcE %p", __func__, ep); 465 return; 466 } 467 468 static void 469 process_conn_error(struct c4iw_ep *ep) 470 { 471 struct c4iw_qp_attributes attrs; 472 int ret; 473 int state; 474 475 mutex_lock(&ep->com.mutex); 476 state = ep->com.state; 477 CTR5(KTR_IW_CXGBE, "%s:pceB ep %p so %p so->so_error %u state %s", 478 __func__, ep, ep->com.so, ep->com.so->so_error, 479 states[ep->com.state]); 480 481 switch (state) { 482 483 case MPA_REQ_WAIT: 484 STOP_EP_TIMER(ep); 485 break; 486 487 case MPA_REQ_SENT: 488 STOP_EP_TIMER(ep); 489 connect_reply_upcall(ep, -ECONNRESET); 490 break; 491 492 case MPA_REP_SENT: 493 ep->com.rpl_err = ECONNRESET; 494 CTR1(KTR_IW_CXGBE, "waking up ep %p", ep); 495 break; 496 497 case MPA_REQ_RCVD: 498 499 /* 500 * We're gonna mark this puppy DEAD, but keep 501 * the reference on it until the ULP accepts or 502 * rejects the CR. 503 */ 504 c4iw_get_ep(&ep->com); 505 break; 506 507 case MORIBUND: 508 case CLOSING: 509 STOP_EP_TIMER(ep); 510 /*FALLTHROUGH*/ 511 case FPDU_MODE: 512 513 if (ep->com.cm_id && ep->com.qp) { 514 515 attrs.next_state = C4IW_QP_STATE_ERROR; 516 ret = c4iw_modify_qp(ep->com.qp->rhp, 517 ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, 518 &attrs, 1); 519 if (ret) 520 log(LOG_ERR, 521 "%s - qp <- error failed!\n", 522 __func__); 523 } 524 peer_abort_upcall(ep); 525 break; 526 527 case ABORTING: 528 break; 529 530 case DEAD: 531 CTR2(KTR_IW_CXGBE, "%s so_error %d IN DEAD STATE!!!!", 532 __func__, ep->com.so->so_error); 533 mutex_unlock(&ep->com.mutex); 534 return; 535 536 default: 537 panic("%s: ep %p state %d", __func__, ep, state); 538 break; 539 } 540 541 if (state != ABORTING) { 542 close_socket(ep->com.so); 543 __state_set(&ep->com, DEAD); 544 c4iw_put_ep(&ep->com); 545 } 546 mutex_unlock(&ep->com.mutex); 547 CTR2(KTR_IW_CXGBE, "%s:pceE %p", __func__, ep); 548 return; 549 } 550 551 static void 552 process_close_complete(struct c4iw_ep *ep) 553 { 554 struct c4iw_qp_attributes attrs; 555 int release = 0; 556 557 CTR4(KTR_IW_CXGBE, "%s:pccB ep %p so %p state %s", __func__, ep, 558 ep->com.so, states[ep->com.state]); 559 560 /* The cm_id may be null if we failed to connect */ 561 mutex_lock(&ep->com.mutex); 562 set_bit(CLOSE_CON_RPL, &ep->com.history); 563 564 switch (ep->com.state) { 565 566 case CLOSING: 567 CTR2(KTR_IW_CXGBE, "%s:pcc1 %p CLOSING MORIBUND", 568 __func__, ep); 569 __state_set(&ep->com, MORIBUND); 570 break; 571 572 case MORIBUND: 573 CTR2(KTR_IW_CXGBE, "%s:pcc1 %p MORIBUND DEAD", __func__, 574 ep); 575 STOP_EP_TIMER(ep); 576 577 if ((ep->com.cm_id) && (ep->com.qp)) { 578 579 CTR2(KTR_IW_CXGBE, "%s:pcc2 %p QP_STATE_IDLE", 580 __func__, ep); 581 attrs.next_state = C4IW_QP_STATE_IDLE; 582 c4iw_modify_qp(ep->com.dev, 583 ep->com.qp, 584 C4IW_QP_ATTR_NEXT_STATE, 585 &attrs, 1); 586 } 587 588 close_socket(ep->com.so); 589 close_complete_upcall(ep, 0); 590 __state_set(&ep->com, DEAD); 591 release = 1; 592 break; 593 594 case ABORTING: 595 CTR2(KTR_IW_CXGBE, "%s:pcc5 %p ABORTING", __func__, ep); 596 break; 597 598 case DEAD: 599 CTR2(KTR_IW_CXGBE, "%s:pcc6 %p DEAD", __func__, ep); 600 break; 601 default: 602 CTR2(KTR_IW_CXGBE, "%s:pcc7 %p unknown ep state", 603 __func__, ep); 604 panic("%s:pcc6 %p unknown ep state", __func__, ep); 605 break; 606 } 607 mutex_unlock(&ep->com.mutex); 608 609 if (release) { 610 611 CTR2(KTR_IW_CXGBE, "%s:pcc8 %p", __func__, ep); 612 c4iw_put_ep(&ep->com); 613 } 614 CTR2(KTR_IW_CXGBE, "%s:pccE %p", __func__, ep); 615 return; 616 } 617 618 static void 619 init_iwarp_socket(struct socket *so, void *arg) 620 { 621 int rc; 622 struct sockopt sopt; 623 int on = 1; 624 625 SOCKBUF_LOCK(&so->so_rcv); 626 soupcall_set(so, SO_RCV, c4iw_so_upcall, arg); 627 so->so_state |= SS_NBIO; 628 SOCKBUF_UNLOCK(&so->so_rcv); 629 sopt.sopt_dir = SOPT_SET; 630 sopt.sopt_level = IPPROTO_TCP; 631 sopt.sopt_name = TCP_NODELAY; 632 sopt.sopt_val = (caddr_t)&on; 633 sopt.sopt_valsize = sizeof on; 634 sopt.sopt_td = NULL; 635 rc = sosetopt(so, &sopt); 636 if (rc) { 637 log(LOG_ERR, "%s: can't set TCP_NODELAY on so %p (%d)\n", 638 __func__, so, rc); 639 } 640 } 641 642 static void 643 uninit_iwarp_socket(struct socket *so) 644 { 645 646 SOCKBUF_LOCK(&so->so_rcv); 647 soupcall_clear(so, SO_RCV); 648 SOCKBUF_UNLOCK(&so->so_rcv); 649 } 650 651 static void 652 process_data(struct c4iw_ep *ep) 653 { 654 struct sockaddr_in *local, *remote; 655 int disconnect = 0; 656 657 CTR5(KTR_IW_CXGBE, "%s: so %p, ep %p, state %s, sbused %d", __func__, 658 ep->com.so, ep, states[ep->com.state], sbused(&ep->com.so->so_rcv)); 659 660 switch (state_read(&ep->com)) { 661 case MPA_REQ_SENT: 662 disconnect = process_mpa_reply(ep); 663 break; 664 case MPA_REQ_WAIT: 665 in_getsockaddr(ep->com.so, (struct sockaddr **)&local); 666 in_getpeeraddr(ep->com.so, (struct sockaddr **)&remote); 667 ep->com.local_addr = *local; 668 ep->com.remote_addr = *remote; 669 free(local, M_SONAME); 670 free(remote, M_SONAME); 671 disconnect = process_mpa_request(ep); 672 break; 673 default: 674 if (sbused(&ep->com.so->so_rcv)) 675 log(LOG_ERR, "%s: Unexpected streaming data. ep %p, " 676 "state %d, so %p, so_state 0x%x, sbused %u\n", 677 __func__, ep, state_read(&ep->com), ep->com.so, 678 ep->com.so->so_state, sbused(&ep->com.so->so_rcv)); 679 break; 680 } 681 if (disconnect) 682 c4iw_ep_disconnect(ep, disconnect == 2, GFP_KERNEL); 683 684 } 685 686 static void 687 process_connected(struct c4iw_ep *ep) 688 { 689 struct socket *so = ep->com.so; 690 691 if ((so->so_state & SS_ISCONNECTED) && !so->so_error) { 692 if (send_mpa_req(ep)) 693 goto err; 694 } else { 695 connect_reply_upcall(ep, -so->so_error); 696 goto err; 697 } 698 return; 699 err: 700 close_socket(so); 701 state_set(&ep->com, DEAD); 702 c4iw_put_ep(&ep->com); 703 return; 704 } 705 706 void 707 process_newconn(struct iw_cm_id *parent_cm_id, struct socket *child_so) 708 { 709 struct c4iw_ep *child_ep; 710 struct sockaddr_in *local; 711 struct sockaddr_in *remote; 712 struct c4iw_ep *parent_ep = parent_cm_id->provider_data; 713 int ret = 0; 714 715 MPASS(child_so != NULL); 716 717 child_ep = alloc_ep(sizeof(*child_ep), GFP_KERNEL); 718 719 CTR5(KTR_IW_CXGBE, 720 "%s: parent so %p, parent ep %p, child so %p, child ep %p", 721 __func__, parent_ep->com.so, parent_ep, child_so, child_ep); 722 723 in_getsockaddr(child_so, (struct sockaddr **)&local); 724 in_getpeeraddr(child_so, (struct sockaddr **)&remote); 725 726 child_ep->com.local_addr = *local; 727 child_ep->com.remote_addr = *remote; 728 child_ep->com.dev = parent_ep->com.dev; 729 child_ep->com.so = child_so; 730 child_ep->com.cm_id = NULL; 731 child_ep->com.thread = parent_ep->com.thread; 732 child_ep->parent_ep = parent_ep; 733 734 free(local, M_SONAME); 735 free(remote, M_SONAME); 736 737 init_iwarp_socket(child_so, &child_ep->com); 738 c4iw_get_ep(&parent_ep->com); 739 init_timer(&child_ep->timer); 740 state_set(&child_ep->com, MPA_REQ_WAIT); 741 START_EP_TIMER(child_ep); 742 743 /* maybe the request has already been queued up on the socket... */ 744 ret = process_mpa_request(child_ep); 745 if (ret == 2) 746 /* ABORT */ 747 c4iw_ep_disconnect(child_ep, 1, GFP_KERNEL); 748 else if (ret == 1) 749 /* CLOSE */ 750 c4iw_ep_disconnect(child_ep, 0, GFP_KERNEL); 751 752 return; 753 } 754 755 static int 756 add_ep_to_req_list(struct c4iw_ep *ep, int new_ep_event) 757 { 758 unsigned long flag; 759 760 spin_lock_irqsave(&req_lock, flag); 761 if (ep && ep->com.so) { 762 ep->com.ep_events |= new_ep_event; 763 if (!ep->com.entry.tqe_prev) { 764 c4iw_get_ep(&ep->com); 765 TAILQ_INSERT_TAIL(&req_list, &ep->com, entry); 766 queue_work(c4iw_taskq, &c4iw_task); 767 } 768 } 769 spin_unlock_irqrestore(&req_lock, flag); 770 771 return (0); 772 } 773 774 static int 775 c4iw_so_upcall(struct socket *so, void *arg, int waitflag) 776 { 777 struct c4iw_ep *ep = arg; 778 779 CTR6(KTR_IW_CXGBE, 780 "%s: so %p, so_state 0x%x, ep %p, ep_state %s, tqe_prev %p", 781 __func__, so, so->so_state, ep, states[ep->com.state], 782 ep->com.entry.tqe_prev); 783 784 MPASS(ep->com.so == so); 785 add_ep_to_req_list(ep, C4IW_EVENT_SOCKET); 786 787 return (SU_OK); 788 } 789 790 791 static int 792 terminate(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 793 { 794 struct adapter *sc = iq->adapter; 795 const struct cpl_rdma_terminate *cpl = mtod(m, const void *); 796 unsigned int tid = GET_TID(cpl); 797 struct toepcb *toep = lookup_tid(sc, tid); 798 struct socket *so; 799 struct c4iw_ep *ep; 800 801 INP_WLOCK(toep->inp); 802 so = inp_inpcbtosocket(toep->inp); 803 ep = so->so_rcv.sb_upcallarg; 804 INP_WUNLOCK(toep->inp); 805 806 CTR3(KTR_IW_CXGBE, "%s: so %p, ep %p", __func__, so, ep); 807 add_ep_to_req_list(ep, C4IW_EVENT_TERM); 808 809 return 0; 810 } 811 812 static void 813 process_socket_event(struct c4iw_ep *ep) 814 { 815 int state = state_read(&ep->com); 816 struct socket *so = ep->com.so; 817 818 CTR6(KTR_IW_CXGBE, "process_socket_event: so %p, so_state 0x%x, " 819 "so_err %d, sb_state 0x%x, ep %p, ep_state %s", so, so->so_state, 820 so->so_error, so->so_rcv.sb_state, ep, states[state]); 821 822 if (state == CONNECTING) { 823 process_connected(ep); 824 return; 825 } 826 827 if (state == LISTEN) { 828 /* socket listening events are handled at IWCM */ 829 CTR3(KTR_IW_CXGBE, "%s Invalid ep state:%u, ep:%p", __func__, 830 ep->com.state, ep); 831 BUG(); 832 return; 833 } 834 835 /* connection error */ 836 if (so->so_error) { 837 process_conn_error(ep); 838 return; 839 } 840 841 /* peer close */ 842 if ((so->so_rcv.sb_state & SBS_CANTRCVMORE) && state <= CLOSING) { 843 process_peer_close(ep); 844 /* 845 * check whether socket disconnect event is pending before 846 * returning. Fallthrough if yes. 847 */ 848 if (!(so->so_state & SS_ISDISCONNECTED)) 849 return; 850 } 851 852 /* close complete */ 853 if (so->so_state & SS_ISDISCONNECTED) { 854 process_close_complete(ep); 855 return; 856 } 857 858 /* rx data */ 859 process_data(ep); 860 } 861 862 SYSCTL_NODE(_hw, OID_AUTO, iw_cxgbe, CTLFLAG_RD, 0, "iw_cxgbe driver parameters"); 863 864 static int dack_mode = 0; 865 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, dack_mode, CTLFLAG_RWTUN, &dack_mode, 0, 866 "Delayed ack mode (default = 0)"); 867 868 int c4iw_max_read_depth = 8; 869 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, c4iw_max_read_depth, CTLFLAG_RWTUN, &c4iw_max_read_depth, 0, 870 "Per-connection max ORD/IRD (default = 8)"); 871 872 static int enable_tcp_timestamps; 873 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, enable_tcp_timestamps, CTLFLAG_RWTUN, &enable_tcp_timestamps, 0, 874 "Enable tcp timestamps (default = 0)"); 875 876 static int enable_tcp_sack; 877 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, enable_tcp_sack, CTLFLAG_RWTUN, &enable_tcp_sack, 0, 878 "Enable tcp SACK (default = 0)"); 879 880 static int enable_tcp_window_scaling = 1; 881 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, enable_tcp_window_scaling, CTLFLAG_RWTUN, &enable_tcp_window_scaling, 0, 882 "Enable tcp window scaling (default = 1)"); 883 884 int c4iw_debug = 0; 885 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, c4iw_debug, CTLFLAG_RWTUN, &c4iw_debug, 0, 886 "Enable debug logging (default = 0)"); 887 888 static int peer2peer = 1; 889 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, peer2peer, CTLFLAG_RWTUN, &peer2peer, 0, 890 "Support peer2peer ULPs (default = 1)"); 891 892 static int p2p_type = FW_RI_INIT_P2PTYPE_READ_REQ; 893 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, p2p_type, CTLFLAG_RWTUN, &p2p_type, 0, 894 "RDMAP opcode to use for the RTR message: 1 = RDMA_READ 0 = RDMA_WRITE (default 1)"); 895 896 static int ep_timeout_secs = 60; 897 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, ep_timeout_secs, CTLFLAG_RWTUN, &ep_timeout_secs, 0, 898 "CM Endpoint operation timeout in seconds (default = 60)"); 899 900 static int mpa_rev = 1; 901 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, mpa_rev, CTLFLAG_RWTUN, &mpa_rev, 0, 902 "MPA Revision, 0 supports amso1100, 1 is RFC5044 spec compliant, 2 is IETF MPA Peer Connect Draft compliant (default = 1)"); 903 904 static int markers_enabled; 905 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, markers_enabled, CTLFLAG_RWTUN, &markers_enabled, 0, 906 "Enable MPA MARKERS (default(0) = disabled)"); 907 908 static int crc_enabled = 1; 909 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, crc_enabled, CTLFLAG_RWTUN, &crc_enabled, 0, 910 "Enable MPA CRC (default(1) = enabled)"); 911 912 static int rcv_win = 256 * 1024; 913 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, rcv_win, CTLFLAG_RWTUN, &rcv_win, 0, 914 "TCP receive window in bytes (default = 256KB)"); 915 916 static int snd_win = 128 * 1024; 917 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, snd_win, CTLFLAG_RWTUN, &snd_win, 0, 918 "TCP send window in bytes (default = 128KB)"); 919 920 static void 921 start_ep_timer(struct c4iw_ep *ep) 922 { 923 924 if (timer_pending(&ep->timer)) { 925 CTR2(KTR_IW_CXGBE, "%s: ep %p, already started", __func__, ep); 926 printk(KERN_ERR "%s timer already started! ep %p\n", __func__, 927 ep); 928 return; 929 } 930 clear_bit(TIMEOUT, &ep->com.flags); 931 c4iw_get_ep(&ep->com); 932 ep->timer.expires = jiffies + ep_timeout_secs * HZ; 933 ep->timer.data = (unsigned long)ep; 934 ep->timer.function = ep_timeout; 935 add_timer(&ep->timer); 936 } 937 938 static int 939 stop_ep_timer(struct c4iw_ep *ep) 940 { 941 942 del_timer_sync(&ep->timer); 943 if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) { 944 c4iw_put_ep(&ep->com); 945 return 0; 946 } 947 return 1; 948 } 949 950 static enum 951 c4iw_ep_state state_read(struct c4iw_ep_common *epc) 952 { 953 enum c4iw_ep_state state; 954 955 mutex_lock(&epc->mutex); 956 state = epc->state; 957 mutex_unlock(&epc->mutex); 958 959 return (state); 960 } 961 962 static void 963 __state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new) 964 { 965 966 epc->state = new; 967 } 968 969 static void 970 state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new) 971 { 972 973 mutex_lock(&epc->mutex); 974 __state_set(epc, new); 975 mutex_unlock(&epc->mutex); 976 } 977 978 static void * 979 alloc_ep(int size, gfp_t gfp) 980 { 981 struct c4iw_ep_common *epc; 982 983 epc = kzalloc(size, gfp); 984 if (epc == NULL) 985 return (NULL); 986 987 kref_init(&epc->kref); 988 mutex_init(&epc->mutex); 989 c4iw_init_wr_wait(&epc->wr_wait); 990 991 return (epc); 992 } 993 994 void _c4iw_free_ep(struct kref *kref) 995 { 996 struct c4iw_ep *ep; 997 struct c4iw_ep_common *epc; 998 999 ep = container_of(kref, struct c4iw_ep, com.kref); 1000 epc = &ep->com; 1001 KASSERT(!epc->entry.tqe_prev, ("%s epc %p still on req list", 1002 __func__, epc)); 1003 if (test_bit(QP_REFERENCED, &ep->com.flags)) 1004 deref_qp(ep); 1005 CTR4(KTR_IW_CXGBE, "%s: ep %p, history 0x%lx, flags 0x%lx", 1006 __func__, ep, epc->history, epc->flags); 1007 kfree(ep); 1008 } 1009 1010 static void release_ep_resources(struct c4iw_ep *ep) 1011 { 1012 CTR2(KTR_IW_CXGBE, "%s:rerB %p", __func__, ep); 1013 set_bit(RELEASE_RESOURCES, &ep->com.flags); 1014 c4iw_put_ep(&ep->com); 1015 CTR2(KTR_IW_CXGBE, "%s:rerE %p", __func__, ep); 1016 } 1017 1018 static int 1019 send_mpa_req(struct c4iw_ep *ep) 1020 { 1021 int mpalen; 1022 struct mpa_message *mpa; 1023 struct mpa_v2_conn_params mpa_v2_params; 1024 struct mbuf *m; 1025 char mpa_rev_to_use = mpa_rev; 1026 int err = 0; 1027 1028 if (ep->retry_with_mpa_v1) 1029 mpa_rev_to_use = 1; 1030 mpalen = sizeof(*mpa) + ep->plen; 1031 if (mpa_rev_to_use == 2) 1032 mpalen += sizeof(struct mpa_v2_conn_params); 1033 1034 mpa = malloc(mpalen, M_CXGBE, M_NOWAIT); 1035 if (mpa == NULL) { 1036 err = -ENOMEM; 1037 CTR3(KTR_IW_CXGBE, "%s:smr1 ep: %p , error: %d", 1038 __func__, ep, err); 1039 goto err; 1040 } 1041 1042 memset(mpa, 0, mpalen); 1043 memcpy(mpa->key, MPA_KEY_REQ, sizeof(mpa->key)); 1044 mpa->flags = (crc_enabled ? MPA_CRC : 0) | 1045 (markers_enabled ? MPA_MARKERS : 0) | 1046 (mpa_rev_to_use == 2 ? MPA_ENHANCED_RDMA_CONN : 0); 1047 mpa->private_data_size = htons(ep->plen); 1048 mpa->revision = mpa_rev_to_use; 1049 1050 if (mpa_rev_to_use == 1) { 1051 ep->tried_with_mpa_v1 = 1; 1052 ep->retry_with_mpa_v1 = 0; 1053 } 1054 1055 if (mpa_rev_to_use == 2) { 1056 mpa->private_data_size += 1057 htons(sizeof(struct mpa_v2_conn_params)); 1058 mpa_v2_params.ird = htons((u16)ep->ird); 1059 mpa_v2_params.ord = htons((u16)ep->ord); 1060 1061 if (peer2peer) { 1062 mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); 1063 1064 if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) { 1065 mpa_v2_params.ord |= 1066 htons(MPA_V2_RDMA_WRITE_RTR); 1067 } else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) { 1068 mpa_v2_params.ord |= 1069 htons(MPA_V2_RDMA_READ_RTR); 1070 } 1071 } 1072 memcpy(mpa->private_data, &mpa_v2_params, 1073 sizeof(struct mpa_v2_conn_params)); 1074 1075 if (ep->plen) { 1076 1077 memcpy(mpa->private_data + 1078 sizeof(struct mpa_v2_conn_params), 1079 ep->mpa_pkt + sizeof(*mpa), ep->plen); 1080 } 1081 } else { 1082 1083 if (ep->plen) 1084 memcpy(mpa->private_data, 1085 ep->mpa_pkt + sizeof(*mpa), ep->plen); 1086 CTR2(KTR_IW_CXGBE, "%s:smr7 %p", __func__, ep); 1087 } 1088 1089 m = m_getm(NULL, mpalen, M_NOWAIT, MT_DATA); 1090 if (m == NULL) { 1091 err = -ENOMEM; 1092 CTR3(KTR_IW_CXGBE, "%s:smr2 ep: %p , error: %d", 1093 __func__, ep, err); 1094 free(mpa, M_CXGBE); 1095 goto err; 1096 } 1097 m_copyback(m, 0, mpalen, (void *)mpa); 1098 free(mpa, M_CXGBE); 1099 1100 err = -sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, 1101 ep->com.thread); 1102 if (err) { 1103 CTR3(KTR_IW_CXGBE, "%s:smr3 ep: %p , error: %d", 1104 __func__, ep, err); 1105 goto err; 1106 } 1107 1108 START_EP_TIMER(ep); 1109 state_set(&ep->com, MPA_REQ_SENT); 1110 ep->mpa_attr.initiator = 1; 1111 CTR3(KTR_IW_CXGBE, "%s:smrE %p, error: %d", __func__, ep, err); 1112 return 0; 1113 err: 1114 connect_reply_upcall(ep, err); 1115 CTR3(KTR_IW_CXGBE, "%s:smrE %p, error: %d", __func__, ep, err); 1116 return err; 1117 } 1118 1119 static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen) 1120 { 1121 int mpalen ; 1122 struct mpa_message *mpa; 1123 struct mpa_v2_conn_params mpa_v2_params; 1124 struct mbuf *m; 1125 int err; 1126 1127 CTR4(KTR_IW_CXGBE, "%s:smrejB %p %u %d", __func__, ep, ep->hwtid, 1128 ep->plen); 1129 1130 mpalen = sizeof(*mpa) + plen; 1131 1132 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 1133 1134 mpalen += sizeof(struct mpa_v2_conn_params); 1135 CTR4(KTR_IW_CXGBE, "%s:smrej1 %p %u %d", __func__, ep, 1136 ep->mpa_attr.version, mpalen); 1137 } 1138 1139 mpa = malloc(mpalen, M_CXGBE, M_NOWAIT); 1140 if (mpa == NULL) 1141 return (-ENOMEM); 1142 1143 memset(mpa, 0, mpalen); 1144 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 1145 mpa->flags = MPA_REJECT; 1146 mpa->revision = mpa_rev; 1147 mpa->private_data_size = htons(plen); 1148 1149 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 1150 1151 mpa->flags |= MPA_ENHANCED_RDMA_CONN; 1152 mpa->private_data_size += 1153 htons(sizeof(struct mpa_v2_conn_params)); 1154 mpa_v2_params.ird = htons(((u16)ep->ird) | 1155 (peer2peer ? MPA_V2_PEER2PEER_MODEL : 1156 0)); 1157 mpa_v2_params.ord = htons(((u16)ep->ord) | (peer2peer ? 1158 (p2p_type == 1159 FW_RI_INIT_P2PTYPE_RDMA_WRITE ? 1160 MPA_V2_RDMA_WRITE_RTR : p2p_type == 1161 FW_RI_INIT_P2PTYPE_READ_REQ ? 1162 MPA_V2_RDMA_READ_RTR : 0) : 0)); 1163 memcpy(mpa->private_data, &mpa_v2_params, 1164 sizeof(struct mpa_v2_conn_params)); 1165 1166 if (ep->plen) 1167 memcpy(mpa->private_data + 1168 sizeof(struct mpa_v2_conn_params), pdata, plen); 1169 CTR5(KTR_IW_CXGBE, "%s:smrej3 %p %d %d %d", __func__, ep, 1170 mpa_v2_params.ird, mpa_v2_params.ord, ep->plen); 1171 } else 1172 if (plen) 1173 memcpy(mpa->private_data, pdata, plen); 1174 1175 m = m_getm(NULL, mpalen, M_NOWAIT, MT_DATA); 1176 if (m == NULL) { 1177 free(mpa, M_CXGBE); 1178 return (-ENOMEM); 1179 } 1180 m_copyback(m, 0, mpalen, (void *)mpa); 1181 free(mpa, M_CXGBE); 1182 1183 err = -sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, ep->com.thread); 1184 if (!err) 1185 ep->snd_seq += mpalen; 1186 CTR4(KTR_IW_CXGBE, "%s:smrejE %p %u %d", __func__, ep, ep->hwtid, err); 1187 return err; 1188 } 1189 1190 static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen) 1191 { 1192 int mpalen; 1193 struct mpa_message *mpa; 1194 struct mbuf *m; 1195 struct mpa_v2_conn_params mpa_v2_params; 1196 int err; 1197 1198 CTR2(KTR_IW_CXGBE, "%s:smrepB %p", __func__, ep); 1199 1200 mpalen = sizeof(*mpa) + plen; 1201 1202 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 1203 1204 CTR3(KTR_IW_CXGBE, "%s:smrep1 %p %d", __func__, ep, 1205 ep->mpa_attr.version); 1206 mpalen += sizeof(struct mpa_v2_conn_params); 1207 } 1208 1209 mpa = malloc(mpalen, M_CXGBE, M_NOWAIT); 1210 if (mpa == NULL) 1211 return (-ENOMEM); 1212 1213 memset(mpa, 0, sizeof(*mpa)); 1214 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 1215 mpa->flags = (ep->mpa_attr.crc_enabled ? MPA_CRC : 0) | 1216 (markers_enabled ? MPA_MARKERS : 0); 1217 mpa->revision = ep->mpa_attr.version; 1218 mpa->private_data_size = htons(plen); 1219 1220 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 1221 1222 mpa->flags |= MPA_ENHANCED_RDMA_CONN; 1223 mpa->private_data_size += 1224 htons(sizeof(struct mpa_v2_conn_params)); 1225 mpa_v2_params.ird = htons((u16)ep->ird); 1226 mpa_v2_params.ord = htons((u16)ep->ord); 1227 CTR5(KTR_IW_CXGBE, "%s:smrep3 %p %d %d %d", __func__, ep, 1228 ep->mpa_attr.version, mpa_v2_params.ird, mpa_v2_params.ord); 1229 1230 if (peer2peer && (ep->mpa_attr.p2p_type != 1231 FW_RI_INIT_P2PTYPE_DISABLED)) { 1232 1233 mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); 1234 1235 if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) { 1236 1237 mpa_v2_params.ord |= 1238 htons(MPA_V2_RDMA_WRITE_RTR); 1239 CTR5(KTR_IW_CXGBE, "%s:smrep4 %p %d %d %d", 1240 __func__, ep, p2p_type, mpa_v2_params.ird, 1241 mpa_v2_params.ord); 1242 } 1243 else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) { 1244 1245 mpa_v2_params.ord |= 1246 htons(MPA_V2_RDMA_READ_RTR); 1247 CTR5(KTR_IW_CXGBE, "%s:smrep5 %p %d %d %d", 1248 __func__, ep, p2p_type, mpa_v2_params.ird, 1249 mpa_v2_params.ord); 1250 } 1251 } 1252 1253 memcpy(mpa->private_data, &mpa_v2_params, 1254 sizeof(struct mpa_v2_conn_params)); 1255 1256 if (ep->plen) 1257 memcpy(mpa->private_data + 1258 sizeof(struct mpa_v2_conn_params), pdata, plen); 1259 } else 1260 if (plen) 1261 memcpy(mpa->private_data, pdata, plen); 1262 1263 m = m_getm(NULL, mpalen, M_NOWAIT, MT_DATA); 1264 if (m == NULL) { 1265 free(mpa, M_CXGBE); 1266 return (-ENOMEM); 1267 } 1268 m_copyback(m, 0, mpalen, (void *)mpa); 1269 free(mpa, M_CXGBE); 1270 1271 1272 state_set(&ep->com, MPA_REP_SENT); 1273 ep->snd_seq += mpalen; 1274 err = -sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, 1275 ep->com.thread); 1276 CTR3(KTR_IW_CXGBE, "%s:smrepE %p %d", __func__, ep, err); 1277 return err; 1278 } 1279 1280 1281 1282 static void close_complete_upcall(struct c4iw_ep *ep, int status) 1283 { 1284 struct iw_cm_event event; 1285 1286 CTR2(KTR_IW_CXGBE, "%s:ccuB %p", __func__, ep); 1287 memset(&event, 0, sizeof(event)); 1288 event.event = IW_CM_EVENT_CLOSE; 1289 event.status = status; 1290 1291 if (ep->com.cm_id) { 1292 1293 CTR2(KTR_IW_CXGBE, "%s:ccu1 %1", __func__, ep); 1294 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1295 deref_cm_id(&ep->com); 1296 set_bit(CLOSE_UPCALL, &ep->com.history); 1297 } 1298 CTR2(KTR_IW_CXGBE, "%s:ccuE %p", __func__, ep); 1299 } 1300 1301 static int 1302 send_abort(struct c4iw_ep *ep) 1303 { 1304 struct socket *so = ep->com.so; 1305 struct sockopt sopt; 1306 int rc; 1307 struct linger l; 1308 1309 CTR5(KTR_IW_CXGBE, "%s ep %p so %p state %s tid %d", __func__, ep, so, 1310 states[ep->com.state], ep->hwtid); 1311 1312 l.l_onoff = 1; 1313 l.l_linger = 0; 1314 1315 /* linger_time of 0 forces RST to be sent */ 1316 sopt.sopt_dir = SOPT_SET; 1317 sopt.sopt_level = SOL_SOCKET; 1318 sopt.sopt_name = SO_LINGER; 1319 sopt.sopt_val = (caddr_t)&l; 1320 sopt.sopt_valsize = sizeof l; 1321 sopt.sopt_td = NULL; 1322 rc = sosetopt(so, &sopt); 1323 if (rc != 0) { 1324 log(LOG_ERR, "%s: sosetopt(%p, linger = 0) failed with %d.\n", 1325 __func__, so, rc); 1326 } 1327 1328 uninit_iwarp_socket(so); 1329 sodisconnect(so); 1330 set_bit(ABORT_CONN, &ep->com.history); 1331 1332 /* 1333 * TBD: iw_cxgbe driver should receive ABORT reply for every ABORT 1334 * request it has sent. But the current TOE driver is not propagating 1335 * this ABORT reply event (via do_abort_rpl) to iw_cxgbe. So as a work- 1336 * around de-refer 'ep' (which was refered before sending ABORT request) 1337 * here instead of doing it in abort_rpl() handler of iw_cxgbe driver. 1338 */ 1339 c4iw_put_ep(&ep->com); 1340 1341 return (0); 1342 } 1343 1344 static void peer_close_upcall(struct c4iw_ep *ep) 1345 { 1346 struct iw_cm_event event; 1347 1348 CTR2(KTR_IW_CXGBE, "%s:pcuB %p", __func__, ep); 1349 memset(&event, 0, sizeof(event)); 1350 event.event = IW_CM_EVENT_DISCONNECT; 1351 1352 if (ep->com.cm_id) { 1353 1354 CTR2(KTR_IW_CXGBE, "%s:pcu1 %p", __func__, ep); 1355 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1356 set_bit(DISCONN_UPCALL, &ep->com.history); 1357 } 1358 CTR2(KTR_IW_CXGBE, "%s:pcuE %p", __func__, ep); 1359 } 1360 1361 static void peer_abort_upcall(struct c4iw_ep *ep) 1362 { 1363 struct iw_cm_event event; 1364 1365 CTR2(KTR_IW_CXGBE, "%s:pauB %p", __func__, ep); 1366 memset(&event, 0, sizeof(event)); 1367 event.event = IW_CM_EVENT_CLOSE; 1368 event.status = -ECONNRESET; 1369 1370 if (ep->com.cm_id) { 1371 1372 CTR2(KTR_IW_CXGBE, "%s:pau1 %p", __func__, ep); 1373 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1374 deref_cm_id(&ep->com); 1375 set_bit(ABORT_UPCALL, &ep->com.history); 1376 } 1377 CTR2(KTR_IW_CXGBE, "%s:pauE %p", __func__, ep); 1378 } 1379 1380 static void connect_reply_upcall(struct c4iw_ep *ep, int status) 1381 { 1382 struct iw_cm_event event; 1383 1384 CTR3(KTR_IW_CXGBE, "%s:cruB %p, status: %d", __func__, ep, status); 1385 memset(&event, 0, sizeof(event)); 1386 event.event = IW_CM_EVENT_CONNECT_REPLY; 1387 event.status = ((status == -ECONNABORTED) || (status == -EPIPE)) ? 1388 -ECONNRESET : status; 1389 event.local_addr = ep->com.local_addr; 1390 event.remote_addr = ep->com.remote_addr; 1391 1392 if ((status == 0) || (status == -ECONNREFUSED)) { 1393 1394 if (!ep->tried_with_mpa_v1) { 1395 1396 CTR2(KTR_IW_CXGBE, "%s:cru1 %p", __func__, ep); 1397 /* this means MPA_v2 is used */ 1398 event.private_data_len = ep->plen - 1399 sizeof(struct mpa_v2_conn_params); 1400 event.private_data = ep->mpa_pkt + 1401 sizeof(struct mpa_message) + 1402 sizeof(struct mpa_v2_conn_params); 1403 } else { 1404 1405 CTR2(KTR_IW_CXGBE, "%s:cru2 %p", __func__, ep); 1406 /* this means MPA_v1 is used */ 1407 event.private_data_len = ep->plen; 1408 event.private_data = ep->mpa_pkt + 1409 sizeof(struct mpa_message); 1410 } 1411 } 1412 1413 if (ep->com.cm_id) { 1414 1415 CTR2(KTR_IW_CXGBE, "%s:cru3 %p", __func__, ep); 1416 set_bit(CONN_RPL_UPCALL, &ep->com.history); 1417 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1418 } 1419 1420 if(status == -ECONNABORTED) { 1421 1422 CTR3(KTR_IW_CXGBE, "%s:cruE %p %d", __func__, ep, status); 1423 return; 1424 } 1425 1426 if (status < 0) { 1427 1428 CTR3(KTR_IW_CXGBE, "%s:cru4 %p %d", __func__, ep, status); 1429 deref_cm_id(&ep->com); 1430 } 1431 1432 CTR2(KTR_IW_CXGBE, "%s:cruE %p", __func__, ep); 1433 } 1434 1435 static int connect_request_upcall(struct c4iw_ep *ep) 1436 { 1437 struct iw_cm_event event; 1438 int ret; 1439 1440 CTR3(KTR_IW_CXGBE, "%s: ep %p, mpa_v1 %d", __func__, ep, 1441 ep->tried_with_mpa_v1); 1442 1443 memset(&event, 0, sizeof(event)); 1444 event.event = IW_CM_EVENT_CONNECT_REQUEST; 1445 event.local_addr = ep->com.local_addr; 1446 event.remote_addr = ep->com.remote_addr; 1447 event.provider_data = ep; 1448 event.so = ep->com.so; 1449 1450 if (!ep->tried_with_mpa_v1) { 1451 /* this means MPA_v2 is used */ 1452 event.ord = ep->ord; 1453 event.ird = ep->ird; 1454 event.private_data_len = ep->plen - 1455 sizeof(struct mpa_v2_conn_params); 1456 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message) + 1457 sizeof(struct mpa_v2_conn_params); 1458 } else { 1459 1460 /* this means MPA_v1 is used. Send max supported */ 1461 event.ord = c4iw_max_read_depth; 1462 event.ird = c4iw_max_read_depth; 1463 event.private_data_len = ep->plen; 1464 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message); 1465 } 1466 1467 c4iw_get_ep(&ep->com); 1468 ret = ep->parent_ep->com.cm_id->event_handler(ep->parent_ep->com.cm_id, 1469 &event); 1470 if(ret) 1471 c4iw_put_ep(&ep->com); 1472 1473 set_bit(CONNREQ_UPCALL, &ep->com.history); 1474 c4iw_put_ep(&ep->parent_ep->com); 1475 return ret; 1476 } 1477 1478 static void established_upcall(struct c4iw_ep *ep) 1479 { 1480 struct iw_cm_event event; 1481 1482 CTR2(KTR_IW_CXGBE, "%s:euB %p", __func__, ep); 1483 memset(&event, 0, sizeof(event)); 1484 event.event = IW_CM_EVENT_ESTABLISHED; 1485 event.ird = ep->ird; 1486 event.ord = ep->ord; 1487 1488 if (ep->com.cm_id) { 1489 1490 CTR2(KTR_IW_CXGBE, "%s:eu1 %p", __func__, ep); 1491 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1492 set_bit(ESTAB_UPCALL, &ep->com.history); 1493 } 1494 CTR2(KTR_IW_CXGBE, "%s:euE %p", __func__, ep); 1495 } 1496 1497 1498 /* 1499 * process_mpa_reply - process streaming mode MPA reply 1500 * 1501 * Returns: 1502 * 1503 * 0 upon success indicating a connect request was delivered to the ULP 1504 * or the mpa request is incomplete but valid so far. 1505 * 1506 * 1 if a failure requires the caller to close the connection. 1507 * 1508 * 2 if a failure requires the caller to abort the connection. 1509 */ 1510 static int process_mpa_reply(struct c4iw_ep *ep) 1511 { 1512 struct mpa_message *mpa; 1513 struct mpa_v2_conn_params *mpa_v2_params; 1514 u16 plen; 1515 u16 resp_ird, resp_ord; 1516 u8 rtr_mismatch = 0, insuff_ird = 0; 1517 struct c4iw_qp_attributes attrs; 1518 enum c4iw_qp_attr_mask mask; 1519 int err; 1520 struct mbuf *top, *m; 1521 int flags = MSG_DONTWAIT; 1522 struct uio uio; 1523 int disconnect = 0; 1524 1525 CTR2(KTR_IW_CXGBE, "%s:pmrB %p", __func__, ep); 1526 1527 /* 1528 * Stop mpa timer. If it expired, then 1529 * we ignore the MPA reply. process_timeout() 1530 * will abort the connection. 1531 */ 1532 if (STOP_EP_TIMER(ep)) 1533 return 0; 1534 1535 uio.uio_resid = 1000000; 1536 uio.uio_td = ep->com.thread; 1537 err = soreceive(ep->com.so, NULL, &uio, &top, NULL, &flags); 1538 1539 if (err) { 1540 1541 if (err == EWOULDBLOCK) { 1542 1543 CTR2(KTR_IW_CXGBE, "%s:pmr1 %p", __func__, ep); 1544 START_EP_TIMER(ep); 1545 return 0; 1546 } 1547 err = -err; 1548 CTR2(KTR_IW_CXGBE, "%s:pmr2 %p", __func__, ep); 1549 goto err; 1550 } 1551 1552 if (ep->com.so->so_rcv.sb_mb) { 1553 1554 CTR2(KTR_IW_CXGBE, "%s:pmr3 %p", __func__, ep); 1555 printf("%s data after soreceive called! so %p sb_mb %p top %p\n", 1556 __func__, ep->com.so, ep->com.so->so_rcv.sb_mb, top); 1557 } 1558 1559 m = top; 1560 1561 do { 1562 1563 CTR2(KTR_IW_CXGBE, "%s:pmr4 %p", __func__, ep); 1564 /* 1565 * If we get more than the supported amount of private data 1566 * then we must fail this connection. 1567 */ 1568 if (ep->mpa_pkt_len + m->m_len > sizeof(ep->mpa_pkt)) { 1569 1570 CTR3(KTR_IW_CXGBE, "%s:pmr5 %p %d", __func__, ep, 1571 ep->mpa_pkt_len + m->m_len); 1572 err = (-EINVAL); 1573 goto err_stop_timer; 1574 } 1575 1576 /* 1577 * copy the new data into our accumulation buffer. 1578 */ 1579 m_copydata(m, 0, m->m_len, &(ep->mpa_pkt[ep->mpa_pkt_len])); 1580 ep->mpa_pkt_len += m->m_len; 1581 if (!m->m_next) 1582 m = m->m_nextpkt; 1583 else 1584 m = m->m_next; 1585 } while (m); 1586 1587 m_freem(top); 1588 /* 1589 * if we don't even have the mpa message, then bail. 1590 */ 1591 if (ep->mpa_pkt_len < sizeof(*mpa)) { 1592 return 0; 1593 } 1594 mpa = (struct mpa_message *) ep->mpa_pkt; 1595 1596 /* Validate MPA header. */ 1597 if (mpa->revision > mpa_rev) { 1598 1599 CTR4(KTR_IW_CXGBE, "%s:pmr6 %p %d %d", __func__, ep, 1600 mpa->revision, mpa_rev); 1601 printk(KERN_ERR MOD "%s MPA version mismatch. Local = %d, " 1602 " Received = %d\n", __func__, mpa_rev, mpa->revision); 1603 err = -EPROTO; 1604 goto err_stop_timer; 1605 } 1606 1607 if (memcmp(mpa->key, MPA_KEY_REP, sizeof(mpa->key))) { 1608 1609 CTR2(KTR_IW_CXGBE, "%s:pmr7 %p", __func__, ep); 1610 err = -EPROTO; 1611 goto err_stop_timer; 1612 } 1613 1614 plen = ntohs(mpa->private_data_size); 1615 1616 /* 1617 * Fail if there's too much private data. 1618 */ 1619 if (plen > MPA_MAX_PRIVATE_DATA) { 1620 1621 CTR2(KTR_IW_CXGBE, "%s:pmr8 %p", __func__, ep); 1622 err = -EPROTO; 1623 goto err_stop_timer; 1624 } 1625 1626 /* 1627 * If plen does not account for pkt size 1628 */ 1629 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { 1630 1631 CTR2(KTR_IW_CXGBE, "%s:pmr9 %p", __func__, ep); 1632 STOP_EP_TIMER(ep); 1633 err = -EPROTO; 1634 goto err_stop_timer; 1635 } 1636 1637 ep->plen = (u8) plen; 1638 1639 /* 1640 * If we don't have all the pdata yet, then bail. 1641 * We'll continue process when more data arrives. 1642 */ 1643 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) { 1644 1645 CTR2(KTR_IW_CXGBE, "%s:pmra %p", __func__, ep); 1646 return 0; 1647 } 1648 1649 if (mpa->flags & MPA_REJECT) { 1650 1651 CTR2(KTR_IW_CXGBE, "%s:pmrb %p", __func__, ep); 1652 err = -ECONNREFUSED; 1653 goto err_stop_timer; 1654 } 1655 1656 /* 1657 * If we get here we have accumulated the entire mpa 1658 * start reply message including private data. And 1659 * the MPA header is valid. 1660 */ 1661 state_set(&ep->com, FPDU_MODE); 1662 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 1663 ep->mpa_attr.recv_marker_enabled = markers_enabled; 1664 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 1665 ep->mpa_attr.version = mpa->revision; 1666 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1667 1668 if (mpa->revision == 2) { 1669 1670 CTR2(KTR_IW_CXGBE, "%s:pmrc %p", __func__, ep); 1671 ep->mpa_attr.enhanced_rdma_conn = 1672 mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; 1673 1674 if (ep->mpa_attr.enhanced_rdma_conn) { 1675 1676 CTR2(KTR_IW_CXGBE, "%s:pmrd %p", __func__, ep); 1677 mpa_v2_params = (struct mpa_v2_conn_params *) 1678 (ep->mpa_pkt + sizeof(*mpa)); 1679 resp_ird = ntohs(mpa_v2_params->ird) & 1680 MPA_V2_IRD_ORD_MASK; 1681 resp_ord = ntohs(mpa_v2_params->ord) & 1682 MPA_V2_IRD_ORD_MASK; 1683 1684 /* 1685 * This is a double-check. Ideally, below checks are 1686 * not required since ird/ord stuff has been taken 1687 * care of in c4iw_accept_cr 1688 */ 1689 if ((ep->ird < resp_ord) || (ep->ord > resp_ird)) { 1690 1691 CTR2(KTR_IW_CXGBE, "%s:pmre %p", __func__, ep); 1692 err = -ENOMEM; 1693 ep->ird = resp_ord; 1694 ep->ord = resp_ird; 1695 insuff_ird = 1; 1696 } 1697 1698 if (ntohs(mpa_v2_params->ird) & 1699 MPA_V2_PEER2PEER_MODEL) { 1700 1701 CTR2(KTR_IW_CXGBE, "%s:pmrf %p", __func__, ep); 1702 if (ntohs(mpa_v2_params->ord) & 1703 MPA_V2_RDMA_WRITE_RTR) { 1704 1705 CTR2(KTR_IW_CXGBE, "%s:pmrg %p", __func__, ep); 1706 ep->mpa_attr.p2p_type = 1707 FW_RI_INIT_P2PTYPE_RDMA_WRITE; 1708 } 1709 else if (ntohs(mpa_v2_params->ord) & 1710 MPA_V2_RDMA_READ_RTR) { 1711 1712 CTR2(KTR_IW_CXGBE, "%s:pmrh %p", __func__, ep); 1713 ep->mpa_attr.p2p_type = 1714 FW_RI_INIT_P2PTYPE_READ_REQ; 1715 } 1716 } 1717 } 1718 } else { 1719 1720 CTR2(KTR_IW_CXGBE, "%s:pmri %p", __func__, ep); 1721 1722 if (mpa->revision == 1) { 1723 1724 CTR2(KTR_IW_CXGBE, "%s:pmrj %p", __func__, ep); 1725 1726 if (peer2peer) { 1727 1728 CTR2(KTR_IW_CXGBE, "%s:pmrk %p", __func__, ep); 1729 ep->mpa_attr.p2p_type = p2p_type; 1730 } 1731 } 1732 } 1733 1734 if (set_tcpinfo(ep)) { 1735 1736 CTR2(KTR_IW_CXGBE, "%s:pmrl %p", __func__, ep); 1737 printf("%s set_tcpinfo error\n", __func__); 1738 err = -ECONNRESET; 1739 goto err; 1740 } 1741 1742 CTR6(KTR_IW_CXGBE, "%s - crc_enabled = %d, recv_marker_enabled = %d, " 1743 "xmit_marker_enabled = %d, version = %d p2p_type = %d", __func__, 1744 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled, 1745 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version, 1746 ep->mpa_attr.p2p_type); 1747 1748 /* 1749 * If responder's RTR does not match with that of initiator, assign 1750 * FW_RI_INIT_P2PTYPE_DISABLED in mpa attributes so that RTR is not 1751 * generated when moving QP to RTS state. 1752 * A TERM message will be sent after QP has moved to RTS state 1753 */ 1754 if ((ep->mpa_attr.version == 2) && peer2peer && 1755 (ep->mpa_attr.p2p_type != p2p_type)) { 1756 1757 CTR2(KTR_IW_CXGBE, "%s:pmrm %p", __func__, ep); 1758 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1759 rtr_mismatch = 1; 1760 } 1761 1762 1763 //ep->ofld_txq = TOEPCB(ep->com.so)->ofld_txq; 1764 attrs.mpa_attr = ep->mpa_attr; 1765 attrs.max_ird = ep->ird; 1766 attrs.max_ord = ep->ord; 1767 attrs.llp_stream_handle = ep; 1768 attrs.next_state = C4IW_QP_STATE_RTS; 1769 1770 mask = C4IW_QP_ATTR_NEXT_STATE | 1771 C4IW_QP_ATTR_LLP_STREAM_HANDLE | C4IW_QP_ATTR_MPA_ATTR | 1772 C4IW_QP_ATTR_MAX_IRD | C4IW_QP_ATTR_MAX_ORD; 1773 1774 /* bind QP and TID with INIT_WR */ 1775 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, mask, &attrs, 1); 1776 1777 if (err) { 1778 1779 CTR2(KTR_IW_CXGBE, "%s:pmrn %p", __func__, ep); 1780 goto err; 1781 } 1782 1783 /* 1784 * If responder's RTR requirement did not match with what initiator 1785 * supports, generate TERM message 1786 */ 1787 if (rtr_mismatch) { 1788 1789 CTR2(KTR_IW_CXGBE, "%s:pmro %p", __func__, ep); 1790 printk(KERN_ERR "%s: RTR mismatch, sending TERM\n", __func__); 1791 attrs.layer_etype = LAYER_MPA | DDP_LLP; 1792 attrs.ecode = MPA_NOMATCH_RTR; 1793 attrs.next_state = C4IW_QP_STATE_TERMINATE; 1794 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1795 C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); 1796 err = -ENOMEM; 1797 disconnect = 1; 1798 goto out; 1799 } 1800 1801 /* 1802 * Generate TERM if initiator IRD is not sufficient for responder 1803 * provided ORD. Currently, we do the same behaviour even when 1804 * responder provided IRD is also not sufficient as regards to 1805 * initiator ORD. 1806 */ 1807 if (insuff_ird) { 1808 1809 CTR2(KTR_IW_CXGBE, "%s:pmrp %p", __func__, ep); 1810 printk(KERN_ERR "%s: Insufficient IRD, sending TERM\n", 1811 __func__); 1812 attrs.layer_etype = LAYER_MPA | DDP_LLP; 1813 attrs.ecode = MPA_INSUFF_IRD; 1814 attrs.next_state = C4IW_QP_STATE_TERMINATE; 1815 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1816 C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); 1817 err = -ENOMEM; 1818 disconnect = 1; 1819 goto out; 1820 } 1821 goto out; 1822 err_stop_timer: 1823 STOP_EP_TIMER(ep); 1824 err: 1825 disconnect = 2; 1826 out: 1827 connect_reply_upcall(ep, err); 1828 CTR2(KTR_IW_CXGBE, "%s:pmrE %p", __func__, ep); 1829 return disconnect; 1830 } 1831 1832 /* 1833 * process_mpa_request - process streaming mode MPA request 1834 * 1835 * Returns: 1836 * 1837 * 0 upon success indicating a connect request was delivered to the ULP 1838 * or the mpa request is incomplete but valid so far. 1839 * 1840 * 1 if a failure requires the caller to close the connection. 1841 * 1842 * 2 if a failure requires the caller to abort the connection. 1843 */ 1844 static int 1845 process_mpa_request(struct c4iw_ep *ep) 1846 { 1847 struct mpa_message *mpa; 1848 u16 plen; 1849 int flags = MSG_DONTWAIT; 1850 int rc; 1851 struct iovec iov; 1852 struct uio uio; 1853 enum c4iw_ep_state state = state_read(&ep->com); 1854 1855 CTR3(KTR_IW_CXGBE, "%s: ep %p, state %s", __func__, ep, states[state]); 1856 1857 if (state != MPA_REQ_WAIT) 1858 return 0; 1859 1860 iov.iov_base = &ep->mpa_pkt[ep->mpa_pkt_len]; 1861 iov.iov_len = sizeof(ep->mpa_pkt) - ep->mpa_pkt_len; 1862 uio.uio_iov = &iov; 1863 uio.uio_iovcnt = 1; 1864 uio.uio_offset = 0; 1865 uio.uio_resid = sizeof(ep->mpa_pkt) - ep->mpa_pkt_len; 1866 uio.uio_segflg = UIO_SYSSPACE; 1867 uio.uio_rw = UIO_READ; 1868 uio.uio_td = NULL; /* uio.uio_td = ep->com.thread; */ 1869 1870 rc = soreceive(ep->com.so, NULL, &uio, NULL, NULL, &flags); 1871 if (rc == EAGAIN) 1872 return 0; 1873 else if (rc) 1874 goto err_stop_timer; 1875 1876 KASSERT(uio.uio_offset > 0, ("%s: sorecieve on so %p read no data", 1877 __func__, ep->com.so)); 1878 ep->mpa_pkt_len += uio.uio_offset; 1879 1880 /* 1881 * If we get more than the supported amount of private data then we must 1882 * fail this connection. XXX: check so_rcv->sb_cc, or peek with another 1883 * soreceive, or increase the size of mpa_pkt by 1 and abort if the last 1884 * byte is filled by the soreceive above. 1885 */ 1886 1887 /* Don't even have the MPA message. Wait for more data to arrive. */ 1888 if (ep->mpa_pkt_len < sizeof(*mpa)) 1889 return 0; 1890 mpa = (struct mpa_message *) ep->mpa_pkt; 1891 1892 /* 1893 * Validate MPA Header. 1894 */ 1895 if (mpa->revision > mpa_rev) { 1896 log(LOG_ERR, "%s: MPA version mismatch. Local = %d," 1897 " Received = %d\n", __func__, mpa_rev, mpa->revision); 1898 goto err_stop_timer; 1899 } 1900 1901 if (memcmp(mpa->key, MPA_KEY_REQ, sizeof(mpa->key))) 1902 goto err_stop_timer; 1903 1904 /* 1905 * Fail if there's too much private data. 1906 */ 1907 plen = ntohs(mpa->private_data_size); 1908 if (plen > MPA_MAX_PRIVATE_DATA) 1909 goto err_stop_timer; 1910 1911 /* 1912 * If plen does not account for pkt size 1913 */ 1914 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) 1915 goto err_stop_timer; 1916 1917 ep->plen = (u8) plen; 1918 1919 /* 1920 * If we don't have all the pdata yet, then bail. 1921 */ 1922 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) 1923 return 0; 1924 1925 /* 1926 * If we get here we have accumulated the entire mpa 1927 * start reply message including private data. 1928 */ 1929 ep->mpa_attr.initiator = 0; 1930 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 1931 ep->mpa_attr.recv_marker_enabled = markers_enabled; 1932 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 1933 ep->mpa_attr.version = mpa->revision; 1934 if (mpa->revision == 1) 1935 ep->tried_with_mpa_v1 = 1; 1936 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1937 1938 if (mpa->revision == 2) { 1939 ep->mpa_attr.enhanced_rdma_conn = 1940 mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; 1941 if (ep->mpa_attr.enhanced_rdma_conn) { 1942 struct mpa_v2_conn_params *mpa_v2_params; 1943 u16 ird, ord; 1944 1945 mpa_v2_params = (void *)&ep->mpa_pkt[sizeof(*mpa)]; 1946 ird = ntohs(mpa_v2_params->ird); 1947 ord = ntohs(mpa_v2_params->ord); 1948 1949 ep->ird = ird & MPA_V2_IRD_ORD_MASK; 1950 ep->ord = ord & MPA_V2_IRD_ORD_MASK; 1951 if (ird & MPA_V2_PEER2PEER_MODEL && peer2peer) { 1952 if (ord & MPA_V2_RDMA_WRITE_RTR) { 1953 ep->mpa_attr.p2p_type = 1954 FW_RI_INIT_P2PTYPE_RDMA_WRITE; 1955 } else if (ord & MPA_V2_RDMA_READ_RTR) { 1956 ep->mpa_attr.p2p_type = 1957 FW_RI_INIT_P2PTYPE_READ_REQ; 1958 } 1959 } 1960 } 1961 } else if (mpa->revision == 1 && peer2peer) 1962 ep->mpa_attr.p2p_type = p2p_type; 1963 1964 if (set_tcpinfo(ep)) 1965 goto err_stop_timer; 1966 1967 CTR5(KTR_IW_CXGBE, "%s: crc_enabled = %d, recv_marker_enabled = %d, " 1968 "xmit_marker_enabled = %d, version = %d", __func__, 1969 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled, 1970 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version); 1971 1972 state_set(&ep->com, MPA_REQ_RCVD); 1973 STOP_EP_TIMER(ep); 1974 1975 /* drive upcall */ 1976 mutex_lock(&ep->parent_ep->com.mutex); 1977 if (ep->parent_ep->com.state != DEAD) { 1978 if (connect_request_upcall(ep)) 1979 goto err_unlock_parent; 1980 } else 1981 goto err_unlock_parent; 1982 mutex_unlock(&ep->parent_ep->com.mutex); 1983 return 0; 1984 1985 err_unlock_parent: 1986 mutex_unlock(&ep->parent_ep->com.mutex); 1987 goto err_out; 1988 err_stop_timer: 1989 STOP_EP_TIMER(ep); 1990 err_out: 1991 return 2; 1992 } 1993 1994 /* 1995 * Upcall from the adapter indicating data has been transmitted. 1996 * For us its just the single MPA request or reply. We can now free 1997 * the skb holding the mpa message. 1998 */ 1999 int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) 2000 { 2001 int err; 2002 struct c4iw_ep *ep = to_ep(cm_id); 2003 CTR2(KTR_IW_CXGBE, "%s:crcB %p", __func__, ep); 2004 int abort = 0; 2005 2006 if ((state_read(&ep->com) == DEAD) || 2007 (state_read(&ep->com) != MPA_REQ_RCVD)) { 2008 2009 CTR2(KTR_IW_CXGBE, "%s:crc1 %p", __func__, ep); 2010 c4iw_put_ep(&ep->com); 2011 return -ECONNRESET; 2012 } 2013 set_bit(ULP_REJECT, &ep->com.history); 2014 2015 if (mpa_rev == 0) { 2016 2017 CTR2(KTR_IW_CXGBE, "%s:crc2 %p", __func__, ep); 2018 abort = 1; 2019 } 2020 else { 2021 2022 CTR2(KTR_IW_CXGBE, "%s:crc3 %p", __func__, ep); 2023 abort = send_mpa_reject(ep, pdata, pdata_len); 2024 } 2025 stop_ep_timer(ep); 2026 err = c4iw_ep_disconnect(ep, abort != 0, GFP_KERNEL); 2027 c4iw_put_ep(&ep->com); 2028 CTR3(KTR_IW_CXGBE, "%s:crc4 %p, err: %d", __func__, ep, err); 2029 return 0; 2030 } 2031 2032 int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 2033 { 2034 int err; 2035 struct c4iw_qp_attributes attrs; 2036 enum c4iw_qp_attr_mask mask; 2037 struct c4iw_ep *ep = to_ep(cm_id); 2038 struct c4iw_dev *h = to_c4iw_dev(cm_id->device); 2039 struct c4iw_qp *qp = get_qhp(h, conn_param->qpn); 2040 int abort = 0; 2041 2042 CTR2(KTR_IW_CXGBE, "%s:cacB %p", __func__, ep); 2043 2044 if (state_read(&ep->com) == DEAD) { 2045 2046 CTR2(KTR_IW_CXGBE, "%s:cac1 %p", __func__, ep); 2047 err = -ECONNRESET; 2048 goto err_out; 2049 } 2050 2051 BUG_ON(state_read(&ep->com) != MPA_REQ_RCVD); 2052 BUG_ON(!qp); 2053 2054 set_bit(ULP_ACCEPT, &ep->com.history); 2055 2056 if ((conn_param->ord > c4iw_max_read_depth) || 2057 (conn_param->ird > c4iw_max_read_depth)) { 2058 2059 CTR2(KTR_IW_CXGBE, "%s:cac2 %p", __func__, ep); 2060 err = -EINVAL; 2061 goto err_abort; 2062 } 2063 2064 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 2065 2066 CTR2(KTR_IW_CXGBE, "%s:cac3 %p", __func__, ep); 2067 2068 if (conn_param->ord > ep->ird) { 2069 2070 CTR2(KTR_IW_CXGBE, "%s:cac4 %p", __func__, ep); 2071 ep->ird = conn_param->ird; 2072 ep->ord = conn_param->ord; 2073 send_mpa_reject(ep, conn_param->private_data, 2074 conn_param->private_data_len); 2075 err = -ENOMEM; 2076 goto err_abort; 2077 } 2078 2079 if (conn_param->ird > ep->ord) { 2080 2081 CTR2(KTR_IW_CXGBE, "%s:cac5 %p", __func__, ep); 2082 2083 if (!ep->ord) { 2084 2085 CTR2(KTR_IW_CXGBE, "%s:cac6 %p", __func__, ep); 2086 conn_param->ird = 1; 2087 } 2088 else { 2089 CTR2(KTR_IW_CXGBE, "%s:cac7 %p", __func__, ep); 2090 err = -ENOMEM; 2091 goto err_abort; 2092 } 2093 } 2094 2095 } 2096 ep->ird = conn_param->ird; 2097 ep->ord = conn_param->ord; 2098 2099 if (ep->mpa_attr.version != 2) { 2100 2101 CTR2(KTR_IW_CXGBE, "%s:cac8 %p", __func__, ep); 2102 2103 if (peer2peer && ep->ird == 0) { 2104 2105 CTR2(KTR_IW_CXGBE, "%s:cac9 %p", __func__, ep); 2106 ep->ird = 1; 2107 } 2108 } 2109 2110 2111 ep->com.cm_id = cm_id; 2112 ref_cm_id(&ep->com); 2113 ep->com.qp = qp; 2114 ref_qp(ep); 2115 //ep->ofld_txq = TOEPCB(ep->com.so)->ofld_txq; 2116 2117 /* bind QP to EP and move to RTS */ 2118 attrs.mpa_attr = ep->mpa_attr; 2119 attrs.max_ird = ep->ird; 2120 attrs.max_ord = ep->ord; 2121 attrs.llp_stream_handle = ep; 2122 attrs.next_state = C4IW_QP_STATE_RTS; 2123 2124 /* bind QP and TID with INIT_WR */ 2125 mask = C4IW_QP_ATTR_NEXT_STATE | 2126 C4IW_QP_ATTR_LLP_STREAM_HANDLE | 2127 C4IW_QP_ATTR_MPA_ATTR | 2128 C4IW_QP_ATTR_MAX_IRD | 2129 C4IW_QP_ATTR_MAX_ORD; 2130 2131 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, mask, &attrs, 1); 2132 2133 if (err) { 2134 2135 CTR2(KTR_IW_CXGBE, "%s:caca %p", __func__, ep); 2136 goto err_defef_cm_id; 2137 } 2138 err = send_mpa_reply(ep, conn_param->private_data, 2139 conn_param->private_data_len); 2140 2141 if (err) { 2142 2143 CTR2(KTR_IW_CXGBE, "%s:caca %p", __func__, ep); 2144 goto err_defef_cm_id; 2145 } 2146 2147 state_set(&ep->com, FPDU_MODE); 2148 established_upcall(ep); 2149 c4iw_put_ep(&ep->com); 2150 CTR2(KTR_IW_CXGBE, "%s:cacE %p", __func__, ep); 2151 return 0; 2152 err_defef_cm_id: 2153 deref_cm_id(&ep->com); 2154 err_abort: 2155 abort = 1; 2156 err_out: 2157 if (abort) 2158 c4iw_ep_disconnect(ep, 1, GFP_KERNEL); 2159 c4iw_put_ep(&ep->com); 2160 CTR2(KTR_IW_CXGBE, "%s:cacE err %p", __func__, ep); 2161 return err; 2162 } 2163 2164 2165 2166 int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 2167 { 2168 int err = 0; 2169 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); 2170 struct c4iw_ep *ep = NULL; 2171 struct nhop4_extended nh4; 2172 2173 CTR2(KTR_IW_CXGBE, "%s:ccB %p", __func__, cm_id); 2174 2175 if ((conn_param->ord > c4iw_max_read_depth) || 2176 (conn_param->ird > c4iw_max_read_depth)) { 2177 2178 CTR2(KTR_IW_CXGBE, "%s:cc1 %p", __func__, cm_id); 2179 err = -EINVAL; 2180 goto out; 2181 } 2182 ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 2183 init_timer(&ep->timer); 2184 ep->plen = conn_param->private_data_len; 2185 2186 if (ep->plen) { 2187 2188 CTR2(KTR_IW_CXGBE, "%s:cc3 %p", __func__, ep); 2189 memcpy(ep->mpa_pkt + sizeof(struct mpa_message), 2190 conn_param->private_data, ep->plen); 2191 } 2192 ep->ird = conn_param->ird; 2193 ep->ord = conn_param->ord; 2194 2195 if (peer2peer && ep->ord == 0) { 2196 2197 CTR2(KTR_IW_CXGBE, "%s:cc4 %p", __func__, ep); 2198 ep->ord = 1; 2199 } 2200 2201 ep->com.dev = dev; 2202 ep->com.cm_id = cm_id; 2203 ref_cm_id(&ep->com); 2204 ep->com.qp = get_qhp(dev, conn_param->qpn); 2205 2206 if (!ep->com.qp) { 2207 2208 CTR2(KTR_IW_CXGBE, "%s:cc5 %p", __func__, ep); 2209 err = -EINVAL; 2210 goto fail2; 2211 } 2212 ref_qp(ep); 2213 ep->com.thread = curthread; 2214 ep->com.so = cm_id->so; 2215 2216 /* find a route */ 2217 err = find_route( 2218 cm_id->local_addr.sin_addr.s_addr, 2219 cm_id->remote_addr.sin_addr.s_addr, 2220 cm_id->local_addr.sin_port, 2221 cm_id->remote_addr.sin_port, 0, &nh4); 2222 2223 if (err) { 2224 2225 CTR2(KTR_IW_CXGBE, "%s:cc7 %p", __func__, ep); 2226 printk(KERN_ERR MOD "%s - cannot find route.\n", __func__); 2227 err = -EHOSTUNREACH; 2228 goto fail2; 2229 } 2230 2231 if (!(nh4.nh_ifp->if_capenable & IFCAP_TOE) || 2232 TOEDEV(nh4.nh_ifp) == NULL) { 2233 err = -ENOPROTOOPT; 2234 goto fail3; 2235 } 2236 fib4_free_nh_ext(RT_DEFAULT_FIB, &nh4); 2237 2238 state_set(&ep->com, CONNECTING); 2239 ep->tos = 0; 2240 ep->com.local_addr = cm_id->local_addr; 2241 ep->com.remote_addr = cm_id->remote_addr; 2242 err = -soconnect(ep->com.so, (struct sockaddr *)&ep->com.remote_addr, 2243 ep->com.thread); 2244 2245 if (!err) { 2246 init_iwarp_socket(cm_id->so, &ep->com); 2247 goto out; 2248 } else { 2249 goto fail2; 2250 } 2251 2252 fail3: 2253 fib4_free_nh_ext(RT_DEFAULT_FIB, &nh4); 2254 fail2: 2255 deref_cm_id(&ep->com); 2256 c4iw_put_ep(&ep->com); 2257 ep = NULL; /* CTR shouldn't display already-freed ep. */ 2258 out: 2259 CTR2(KTR_IW_CXGBE, "%s:ccE %p", __func__, ep); 2260 return err; 2261 } 2262 2263 /* 2264 * iwcm->create_listen_ep. Returns -errno on failure. 2265 */ 2266 int 2267 c4iw_create_listen_ep(struct iw_cm_id *cm_id, int backlog) 2268 { 2269 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); 2270 struct c4iw_listen_ep *ep; 2271 struct socket *so = cm_id->so; 2272 2273 ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 2274 ep->com.cm_id = cm_id; 2275 ref_cm_id(&ep->com); 2276 ep->com.dev = dev; 2277 ep->backlog = backlog; 2278 ep->com.local_addr = cm_id->local_addr; 2279 ep->com.thread = curthread; 2280 state_set(&ep->com, LISTEN); 2281 ep->com.so = so; 2282 2283 cm_id->provider_data = ep; 2284 return (0); 2285 } 2286 2287 void 2288 c4iw_destroy_listen_ep(struct iw_cm_id *cm_id) 2289 { 2290 struct c4iw_listen_ep *ep = to_listen_ep(cm_id); 2291 2292 CTR4(KTR_IW_CXGBE, "%s: cm_id %p, so %p, state %s", __func__, cm_id, 2293 cm_id->so, states[ep->com.state]); 2294 2295 state_set(&ep->com, DEAD); 2296 deref_cm_id(&ep->com); 2297 c4iw_put_ep(&ep->com); 2298 2299 return; 2300 } 2301 2302 int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp) 2303 { 2304 int ret = 0; 2305 int close = 0; 2306 int fatal = 0; 2307 struct c4iw_rdev *rdev; 2308 2309 mutex_lock(&ep->com.mutex); 2310 2311 CTR2(KTR_IW_CXGBE, "%s:cedB %p", __func__, ep); 2312 2313 rdev = &ep->com.dev->rdev; 2314 2315 if (c4iw_fatal_error(rdev)) { 2316 2317 CTR2(KTR_IW_CXGBE, "%s:ced1 %p", __func__, ep); 2318 fatal = 1; 2319 close_complete_upcall(ep, -ECONNRESET); 2320 ep->com.state = DEAD; 2321 } 2322 CTR3(KTR_IW_CXGBE, "%s:ced2 %p %s", __func__, ep, 2323 states[ep->com.state]); 2324 2325 switch (ep->com.state) { 2326 2327 case MPA_REQ_WAIT: 2328 case MPA_REQ_SENT: 2329 case MPA_REQ_RCVD: 2330 case MPA_REP_SENT: 2331 case FPDU_MODE: 2332 close = 1; 2333 if (abrupt) 2334 ep->com.state = ABORTING; 2335 else { 2336 ep->com.state = CLOSING; 2337 START_EP_TIMER(ep); 2338 } 2339 set_bit(CLOSE_SENT, &ep->com.flags); 2340 break; 2341 2342 case CLOSING: 2343 2344 if (!test_and_set_bit(CLOSE_SENT, &ep->com.flags)) { 2345 2346 close = 1; 2347 if (abrupt) { 2348 STOP_EP_TIMER(ep); 2349 ep->com.state = ABORTING; 2350 } else 2351 ep->com.state = MORIBUND; 2352 } 2353 break; 2354 2355 case MORIBUND: 2356 case ABORTING: 2357 case DEAD: 2358 CTR3(KTR_IW_CXGBE, 2359 "%s ignoring disconnect ep %p state %u", __func__, 2360 ep, ep->com.state); 2361 break; 2362 2363 default: 2364 BUG(); 2365 break; 2366 } 2367 2368 mutex_unlock(&ep->com.mutex); 2369 2370 if (close) { 2371 2372 CTR2(KTR_IW_CXGBE, "%s:ced3 %p", __func__, ep); 2373 2374 if (abrupt) { 2375 2376 CTR2(KTR_IW_CXGBE, "%s:ced4 %p", __func__, ep); 2377 set_bit(EP_DISC_ABORT, &ep->com.history); 2378 close_complete_upcall(ep, -ECONNRESET); 2379 ret = send_abort(ep); 2380 if (ret) 2381 fatal = 1; 2382 } else { 2383 2384 CTR2(KTR_IW_CXGBE, "%s:ced5 %p", __func__, ep); 2385 set_bit(EP_DISC_CLOSE, &ep->com.history); 2386 2387 if (!ep->parent_ep) 2388 __state_set(&ep->com, MORIBUND); 2389 sodisconnect(ep->com.so); 2390 } 2391 2392 } 2393 2394 if (fatal) { 2395 set_bit(EP_DISC_FAIL, &ep->com.history); 2396 if (!abrupt) { 2397 STOP_EP_TIMER(ep); 2398 close_complete_upcall(ep, -EIO); 2399 } 2400 if (ep->com.qp) { 2401 struct c4iw_qp_attributes attrs; 2402 2403 attrs.next_state = C4IW_QP_STATE_ERROR; 2404 ret = c4iw_modify_qp(ep->com.dev, ep->com.qp, 2405 C4IW_QP_ATTR_NEXT_STATE, 2406 &attrs, 1); 2407 if (ret) { 2408 CTR2(KTR_IW_CXGBE, "%s:ced7 %p", __func__, ep); 2409 printf("%s - qp <- error failed!\n", __func__); 2410 } 2411 } 2412 release_ep_resources(ep); 2413 ep->com.state = DEAD; 2414 CTR2(KTR_IW_CXGBE, "%s:ced6 %p", __func__, ep); 2415 } 2416 CTR2(KTR_IW_CXGBE, "%s:cedE %p", __func__, ep); 2417 return ret; 2418 } 2419 2420 #ifdef C4IW_EP_REDIRECT 2421 int c4iw_ep_redirect(void *ctx, struct dst_entry *old, struct dst_entry *new, 2422 struct l2t_entry *l2t) 2423 { 2424 struct c4iw_ep *ep = ctx; 2425 2426 if (ep->dst != old) 2427 return 0; 2428 2429 PDBG("%s ep %p redirect to dst %p l2t %p\n", __func__, ep, new, 2430 l2t); 2431 dst_hold(new); 2432 cxgb4_l2t_release(ep->l2t); 2433 ep->l2t = l2t; 2434 dst_release(old); 2435 ep->dst = new; 2436 return 1; 2437 } 2438 #endif 2439 2440 2441 2442 static void ep_timeout(unsigned long arg) 2443 { 2444 struct c4iw_ep *ep = (struct c4iw_ep *)arg; 2445 2446 if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) { 2447 2448 /* 2449 * Only insert if it is not already on the list. 2450 */ 2451 if (!(ep->com.ep_events & C4IW_EVENT_TIMEOUT)) { 2452 CTR2(KTR_IW_CXGBE, "%s:et1 %p", __func__, ep); 2453 add_ep_to_req_list(ep, C4IW_EVENT_TIMEOUT); 2454 } 2455 } 2456 } 2457 2458 static int fw6_wr_rpl(struct adapter *sc, const __be64 *rpl) 2459 { 2460 uint64_t val = be64toh(*rpl); 2461 int ret; 2462 struct c4iw_wr_wait *wr_waitp; 2463 2464 ret = (int)((val >> 8) & 0xff); 2465 wr_waitp = (struct c4iw_wr_wait *)rpl[1]; 2466 CTR3(KTR_IW_CXGBE, "%s wr_waitp %p ret %u", __func__, wr_waitp, ret); 2467 if (wr_waitp) 2468 c4iw_wake_up(wr_waitp, ret ? -ret : 0); 2469 2470 return (0); 2471 } 2472 2473 static int fw6_cqe_handler(struct adapter *sc, const __be64 *rpl) 2474 { 2475 struct cqe_list_entry *cle; 2476 unsigned long flag; 2477 2478 cle = malloc(sizeof(*cle), M_CXGBE, M_NOWAIT); 2479 cle->rhp = sc->iwarp_softc; 2480 cle->err_cqe = *(const struct t4_cqe *)(&rpl[0]); 2481 2482 spin_lock_irqsave(&err_cqe_lock, flag); 2483 list_add_tail(&cle->entry, &err_cqe_list); 2484 queue_work(c4iw_taskq, &c4iw_task); 2485 spin_unlock_irqrestore(&err_cqe_lock, flag); 2486 2487 return (0); 2488 } 2489 2490 static int 2491 process_terminate(struct c4iw_ep *ep) 2492 { 2493 struct c4iw_qp_attributes attrs; 2494 2495 CTR2(KTR_IW_CXGBE, "%s:tB %p %d", __func__, ep); 2496 2497 if (ep && ep->com.qp) { 2498 2499 printk(KERN_WARNING MOD "TERM received tid %u qpid %u\n", 2500 ep->hwtid, ep->com.qp->wq.sq.qid); 2501 attrs.next_state = C4IW_QP_STATE_TERMINATE; 2502 c4iw_modify_qp(ep->com.dev, ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 2503 1); 2504 } else 2505 printk(KERN_WARNING MOD "TERM received tid %u no ep/qp\n", 2506 ep->hwtid); 2507 CTR2(KTR_IW_CXGBE, "%s:tE %p %d", __func__, ep); 2508 2509 return 0; 2510 } 2511 2512 int __init c4iw_cm_init(void) 2513 { 2514 2515 t4_register_cpl_handler(CPL_RDMA_TERMINATE, terminate); 2516 t4_register_fw_msg_handler(FW6_TYPE_WR_RPL, fw6_wr_rpl); 2517 t4_register_fw_msg_handler(FW6_TYPE_CQE, fw6_cqe_handler); 2518 t4_register_an_handler(c4iw_ev_handler); 2519 2520 TAILQ_INIT(&req_list); 2521 spin_lock_init(&req_lock); 2522 INIT_LIST_HEAD(&err_cqe_list); 2523 spin_lock_init(&err_cqe_lock); 2524 2525 INIT_WORK(&c4iw_task, process_req); 2526 2527 c4iw_taskq = create_singlethread_workqueue("iw_cxgbe"); 2528 if (!c4iw_taskq) 2529 return -ENOMEM; 2530 2531 return 0; 2532 } 2533 2534 void __exit c4iw_cm_term(void) 2535 { 2536 WARN_ON(!TAILQ_EMPTY(&req_list)); 2537 WARN_ON(!list_empty(&err_cqe_list)); 2538 flush_workqueue(c4iw_taskq); 2539 destroy_workqueue(c4iw_taskq); 2540 2541 t4_register_cpl_handler(CPL_RDMA_TERMINATE, NULL); 2542 t4_register_fw_msg_handler(FW6_TYPE_WR_RPL, NULL); 2543 t4_register_fw_msg_handler(FW6_TYPE_CQE, NULL); 2544 t4_register_an_handler(NULL); 2545 } 2546 #endif 2547