1 /* 2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <linux/module.h> 33 #include <linux/list.h> 34 #include <linux/workqueue.h> 35 #include <linux/skbuff.h> 36 #include <linux/timer.h> 37 #include <linux/notifier.h> 38 #include <linux/inetdevice.h> 39 #include <linux/ip.h> 40 #include <linux/tcp.h> 41 42 #include <net/neighbour.h> 43 #include <net/netevent.h> 44 #include <net/route.h> 45 46 #include "iw_cxgb4.h" 47 48 static char *states[] = { 49 "idle", 50 "listen", 51 "connecting", 52 "mpa_wait_req", 53 "mpa_req_sent", 54 "mpa_req_rcvd", 55 "mpa_rep_sent", 56 "fpdu_mode", 57 "aborting", 58 "closing", 59 "moribund", 60 "dead", 61 NULL, 62 }; 63 64 static int dack_mode = 1; 65 module_param(dack_mode, int, 0644); 66 MODULE_PARM_DESC(dack_mode, "Delayed ack mode (default=1)"); 67 68 int c4iw_max_read_depth = 8; 69 module_param(c4iw_max_read_depth, int, 0644); 70 MODULE_PARM_DESC(c4iw_max_read_depth, "Per-connection max ORD/IRD (default=8)"); 71 72 static int enable_tcp_timestamps; 73 module_param(enable_tcp_timestamps, int, 0644); 74 MODULE_PARM_DESC(enable_tcp_timestamps, "Enable tcp timestamps (default=0)"); 75 76 static int enable_tcp_sack; 77 module_param(enable_tcp_sack, int, 0644); 78 MODULE_PARM_DESC(enable_tcp_sack, "Enable tcp SACK (default=0)"); 79 80 static int enable_tcp_window_scaling = 1; 81 module_param(enable_tcp_window_scaling, int, 0644); 82 MODULE_PARM_DESC(enable_tcp_window_scaling, 83 "Enable tcp window scaling (default=1)"); 84 85 int c4iw_debug; 86 module_param(c4iw_debug, int, 0644); 87 MODULE_PARM_DESC(c4iw_debug, "Enable debug logging (default=0)"); 88 89 static int peer2peer; 90 module_param(peer2peer, int, 0644); 91 MODULE_PARM_DESC(peer2peer, "Support peer2peer ULPs (default=0)"); 92 93 static int p2p_type = FW_RI_INIT_P2PTYPE_READ_REQ; 94 module_param(p2p_type, int, 0644); 95 MODULE_PARM_DESC(p2p_type, "RDMAP opcode to use for the RTR message: " 96 "1=RDMA_READ 0=RDMA_WRITE (default 1)"); 97 98 static int ep_timeout_secs = 60; 99 module_param(ep_timeout_secs, int, 0644); 100 MODULE_PARM_DESC(ep_timeout_secs, "CM Endpoint operation timeout " 101 "in seconds (default=60)"); 102 103 static int mpa_rev = 1; 104 module_param(mpa_rev, int, 0644); 105 MODULE_PARM_DESC(mpa_rev, "MPA Revision, 0 supports amso1100, " 106 "1 is RFC0544 spec compliant, 2 is IETF MPA Peer Connect Draft" 107 " compliant (default=1)"); 108 109 static int markers_enabled; 110 module_param(markers_enabled, int, 0644); 111 MODULE_PARM_DESC(markers_enabled, "Enable MPA MARKERS (default(0)=disabled)"); 112 113 static int crc_enabled = 1; 114 module_param(crc_enabled, int, 0644); 115 MODULE_PARM_DESC(crc_enabled, "Enable MPA CRC (default(1)=enabled)"); 116 117 static int rcv_win = 256 * 1024; 118 module_param(rcv_win, int, 0644); 119 MODULE_PARM_DESC(rcv_win, "TCP receive window in bytes (default=256KB)"); 120 121 static int snd_win = 128 * 1024; 122 module_param(snd_win, int, 0644); 123 MODULE_PARM_DESC(snd_win, "TCP send window in bytes (default=128KB)"); 124 125 static struct workqueue_struct *workq; 126 127 static struct sk_buff_head rxq; 128 129 static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp); 130 static void ep_timeout(unsigned long arg); 131 static void connect_reply_upcall(struct c4iw_ep *ep, int status); 132 133 static LIST_HEAD(timeout_list); 134 static spinlock_t timeout_lock; 135 136 static void start_ep_timer(struct c4iw_ep *ep) 137 { 138 PDBG("%s ep %p\n", __func__, ep); 139 if (timer_pending(&ep->timer)) { 140 PDBG("%s stopped / restarted timer ep %p\n", __func__, ep); 141 del_timer_sync(&ep->timer); 142 } else 143 c4iw_get_ep(&ep->com); 144 ep->timer.expires = jiffies + ep_timeout_secs * HZ; 145 ep->timer.data = (unsigned long)ep; 146 ep->timer.function = ep_timeout; 147 add_timer(&ep->timer); 148 } 149 150 static void stop_ep_timer(struct c4iw_ep *ep) 151 { 152 PDBG("%s ep %p\n", __func__, ep); 153 if (!timer_pending(&ep->timer)) { 154 printk(KERN_ERR "%s timer stopped when its not running! " 155 "ep %p state %u\n", __func__, ep, ep->com.state); 156 WARN_ON(1); 157 return; 158 } 159 del_timer_sync(&ep->timer); 160 c4iw_put_ep(&ep->com); 161 } 162 163 static int c4iw_l2t_send(struct c4iw_rdev *rdev, struct sk_buff *skb, 164 struct l2t_entry *l2e) 165 { 166 int error = 0; 167 168 if (c4iw_fatal_error(rdev)) { 169 kfree_skb(skb); 170 PDBG("%s - device in error state - dropping\n", __func__); 171 return -EIO; 172 } 173 error = cxgb4_l2t_send(rdev->lldi.ports[0], skb, l2e); 174 if (error < 0) 175 kfree_skb(skb); 176 return error < 0 ? error : 0; 177 } 178 179 int c4iw_ofld_send(struct c4iw_rdev *rdev, struct sk_buff *skb) 180 { 181 int error = 0; 182 183 if (c4iw_fatal_error(rdev)) { 184 kfree_skb(skb); 185 PDBG("%s - device in error state - dropping\n", __func__); 186 return -EIO; 187 } 188 error = cxgb4_ofld_send(rdev->lldi.ports[0], skb); 189 if (error < 0) 190 kfree_skb(skb); 191 return error < 0 ? error : 0; 192 } 193 194 static void release_tid(struct c4iw_rdev *rdev, u32 hwtid, struct sk_buff *skb) 195 { 196 struct cpl_tid_release *req; 197 198 skb = get_skb(skb, sizeof *req, GFP_KERNEL); 199 if (!skb) 200 return; 201 req = (struct cpl_tid_release *) skb_put(skb, sizeof(*req)); 202 INIT_TP_WR(req, hwtid); 203 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_TID_RELEASE, hwtid)); 204 set_wr_txq(skb, CPL_PRIORITY_SETUP, 0); 205 c4iw_ofld_send(rdev, skb); 206 return; 207 } 208 209 static void set_emss(struct c4iw_ep *ep, u16 opt) 210 { 211 ep->emss = ep->com.dev->rdev.lldi.mtus[GET_TCPOPT_MSS(opt)] - 40; 212 ep->mss = ep->emss; 213 if (GET_TCPOPT_TSTAMP(opt)) 214 ep->emss -= 12; 215 if (ep->emss < 128) 216 ep->emss = 128; 217 PDBG("%s mss_idx %u mss %u emss=%u\n", __func__, GET_TCPOPT_MSS(opt), 218 ep->mss, ep->emss); 219 } 220 221 static enum c4iw_ep_state state_read(struct c4iw_ep_common *epc) 222 { 223 enum c4iw_ep_state state; 224 225 mutex_lock(&epc->mutex); 226 state = epc->state; 227 mutex_unlock(&epc->mutex); 228 return state; 229 } 230 231 static void __state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new) 232 { 233 epc->state = new; 234 } 235 236 static void state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new) 237 { 238 mutex_lock(&epc->mutex); 239 PDBG("%s - %s -> %s\n", __func__, states[epc->state], states[new]); 240 __state_set(epc, new); 241 mutex_unlock(&epc->mutex); 242 return; 243 } 244 245 static void *alloc_ep(int size, gfp_t gfp) 246 { 247 struct c4iw_ep_common *epc; 248 249 epc = kzalloc(size, gfp); 250 if (epc) { 251 kref_init(&epc->kref); 252 mutex_init(&epc->mutex); 253 c4iw_init_wr_wait(&epc->wr_wait); 254 } 255 PDBG("%s alloc ep %p\n", __func__, epc); 256 return epc; 257 } 258 259 void _c4iw_free_ep(struct kref *kref) 260 { 261 struct c4iw_ep *ep; 262 263 ep = container_of(kref, struct c4iw_ep, com.kref); 264 PDBG("%s ep %p state %s\n", __func__, ep, states[state_read(&ep->com)]); 265 if (test_bit(RELEASE_RESOURCES, &ep->com.flags)) { 266 cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, ep->hwtid); 267 dst_release(ep->dst); 268 cxgb4_l2t_release(ep->l2t); 269 } 270 kfree(ep); 271 } 272 273 static void release_ep_resources(struct c4iw_ep *ep) 274 { 275 set_bit(RELEASE_RESOURCES, &ep->com.flags); 276 c4iw_put_ep(&ep->com); 277 } 278 279 static int status2errno(int status) 280 { 281 switch (status) { 282 case CPL_ERR_NONE: 283 return 0; 284 case CPL_ERR_CONN_RESET: 285 return -ECONNRESET; 286 case CPL_ERR_ARP_MISS: 287 return -EHOSTUNREACH; 288 case CPL_ERR_CONN_TIMEDOUT: 289 return -ETIMEDOUT; 290 case CPL_ERR_TCAM_FULL: 291 return -ENOMEM; 292 case CPL_ERR_CONN_EXIST: 293 return -EADDRINUSE; 294 default: 295 return -EIO; 296 } 297 } 298 299 /* 300 * Try and reuse skbs already allocated... 301 */ 302 static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp) 303 { 304 if (skb && !skb_is_nonlinear(skb) && !skb_cloned(skb)) { 305 skb_trim(skb, 0); 306 skb_get(skb); 307 skb_reset_transport_header(skb); 308 } else { 309 skb = alloc_skb(len, gfp); 310 } 311 return skb; 312 } 313 314 static struct rtable *find_route(struct c4iw_dev *dev, __be32 local_ip, 315 __be32 peer_ip, __be16 local_port, 316 __be16 peer_port, u8 tos) 317 { 318 struct rtable *rt; 319 struct flowi4 fl4; 320 321 rt = ip_route_output_ports(&init_net, &fl4, NULL, peer_ip, local_ip, 322 peer_port, local_port, IPPROTO_TCP, 323 tos, 0); 324 if (IS_ERR(rt)) 325 return NULL; 326 return rt; 327 } 328 329 static void arp_failure_discard(void *handle, struct sk_buff *skb) 330 { 331 PDBG("%s c4iw_dev %p\n", __func__, handle); 332 kfree_skb(skb); 333 } 334 335 /* 336 * Handle an ARP failure for an active open. 337 */ 338 static void act_open_req_arp_failure(void *handle, struct sk_buff *skb) 339 { 340 printk(KERN_ERR MOD "ARP failure duing connect\n"); 341 kfree_skb(skb); 342 } 343 344 /* 345 * Handle an ARP failure for a CPL_ABORT_REQ. Change it into a no RST variant 346 * and send it along. 347 */ 348 static void abort_arp_failure(void *handle, struct sk_buff *skb) 349 { 350 struct c4iw_rdev *rdev = handle; 351 struct cpl_abort_req *req = cplhdr(skb); 352 353 PDBG("%s rdev %p\n", __func__, rdev); 354 req->cmd = CPL_ABORT_NO_RST; 355 c4iw_ofld_send(rdev, skb); 356 } 357 358 static void send_flowc(struct c4iw_ep *ep, struct sk_buff *skb) 359 { 360 unsigned int flowclen = 80; 361 struct fw_flowc_wr *flowc; 362 int i; 363 364 skb = get_skb(skb, flowclen, GFP_KERNEL); 365 flowc = (struct fw_flowc_wr *)__skb_put(skb, flowclen); 366 367 flowc->op_to_nparams = cpu_to_be32(FW_WR_OP(FW_FLOWC_WR) | 368 FW_FLOWC_WR_NPARAMS(8)); 369 flowc->flowid_len16 = cpu_to_be32(FW_WR_LEN16(DIV_ROUND_UP(flowclen, 370 16)) | FW_WR_FLOWID(ep->hwtid)); 371 372 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN; 373 flowc->mnemval[0].val = cpu_to_be32(PCI_FUNC(ep->com.dev->rdev.lldi.pdev->devfn) << 8); 374 flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH; 375 flowc->mnemval[1].val = cpu_to_be32(ep->tx_chan); 376 flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT; 377 flowc->mnemval[2].val = cpu_to_be32(ep->tx_chan); 378 flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID; 379 flowc->mnemval[3].val = cpu_to_be32(ep->rss_qid); 380 flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT; 381 flowc->mnemval[4].val = cpu_to_be32(ep->snd_seq); 382 flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT; 383 flowc->mnemval[5].val = cpu_to_be32(ep->rcv_seq); 384 flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF; 385 flowc->mnemval[6].val = cpu_to_be32(snd_win); 386 flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS; 387 flowc->mnemval[7].val = cpu_to_be32(ep->emss); 388 /* Pad WR to 16 byte boundary */ 389 flowc->mnemval[8].mnemonic = 0; 390 flowc->mnemval[8].val = 0; 391 for (i = 0; i < 9; i++) { 392 flowc->mnemval[i].r4[0] = 0; 393 flowc->mnemval[i].r4[1] = 0; 394 flowc->mnemval[i].r4[2] = 0; 395 } 396 397 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 398 c4iw_ofld_send(&ep->com.dev->rdev, skb); 399 } 400 401 static int send_halfclose(struct c4iw_ep *ep, gfp_t gfp) 402 { 403 struct cpl_close_con_req *req; 404 struct sk_buff *skb; 405 int wrlen = roundup(sizeof *req, 16); 406 407 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 408 skb = get_skb(NULL, wrlen, gfp); 409 if (!skb) { 410 printk(KERN_ERR MOD "%s - failed to alloc skb\n", __func__); 411 return -ENOMEM; 412 } 413 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 414 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 415 req = (struct cpl_close_con_req *) skb_put(skb, wrlen); 416 memset(req, 0, wrlen); 417 INIT_TP_WR(req, ep->hwtid); 418 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, 419 ep->hwtid)); 420 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 421 } 422 423 static int send_abort(struct c4iw_ep *ep, struct sk_buff *skb, gfp_t gfp) 424 { 425 struct cpl_abort_req *req; 426 int wrlen = roundup(sizeof *req, 16); 427 428 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 429 skb = get_skb(skb, wrlen, gfp); 430 if (!skb) { 431 printk(KERN_ERR MOD "%s - failed to alloc skb.\n", 432 __func__); 433 return -ENOMEM; 434 } 435 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 436 t4_set_arp_err_handler(skb, &ep->com.dev->rdev, abort_arp_failure); 437 req = (struct cpl_abort_req *) skb_put(skb, wrlen); 438 memset(req, 0, wrlen); 439 INIT_TP_WR(req, ep->hwtid); 440 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ, ep->hwtid)); 441 req->cmd = CPL_ABORT_SEND_RST; 442 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 443 } 444 445 static int send_connect(struct c4iw_ep *ep) 446 { 447 struct cpl_act_open_req *req; 448 struct sk_buff *skb; 449 u64 opt0; 450 u32 opt2; 451 unsigned int mtu_idx; 452 int wscale; 453 int wrlen = roundup(sizeof *req, 16); 454 455 PDBG("%s ep %p atid %u\n", __func__, ep, ep->atid); 456 457 skb = get_skb(NULL, wrlen, GFP_KERNEL); 458 if (!skb) { 459 printk(KERN_ERR MOD "%s - failed to alloc skb.\n", 460 __func__); 461 return -ENOMEM; 462 } 463 set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->ctrlq_idx); 464 465 cxgb4_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx); 466 wscale = compute_wscale(rcv_win); 467 opt0 = KEEP_ALIVE(1) | 468 DELACK(1) | 469 WND_SCALE(wscale) | 470 MSS_IDX(mtu_idx) | 471 L2T_IDX(ep->l2t->idx) | 472 TX_CHAN(ep->tx_chan) | 473 SMAC_SEL(ep->smac_idx) | 474 DSCP(ep->tos) | 475 ULP_MODE(ULP_MODE_TCPDDP) | 476 RCV_BUFSIZ(rcv_win>>10); 477 opt2 = RX_CHANNEL(0) | 478 RSS_QUEUE_VALID | RSS_QUEUE(ep->rss_qid); 479 if (enable_tcp_timestamps) 480 opt2 |= TSTAMPS_EN(1); 481 if (enable_tcp_sack) 482 opt2 |= SACK_EN(1); 483 if (wscale && enable_tcp_window_scaling) 484 opt2 |= WND_SCALE_EN(1); 485 t4_set_arp_err_handler(skb, NULL, act_open_req_arp_failure); 486 487 req = (struct cpl_act_open_req *) skb_put(skb, wrlen); 488 INIT_TP_WR(req, 0); 489 OPCODE_TID(req) = cpu_to_be32( 490 MK_OPCODE_TID(CPL_ACT_OPEN_REQ, ((ep->rss_qid<<14)|ep->atid))); 491 req->local_port = ep->com.local_addr.sin_port; 492 req->peer_port = ep->com.remote_addr.sin_port; 493 req->local_ip = ep->com.local_addr.sin_addr.s_addr; 494 req->peer_ip = ep->com.remote_addr.sin_addr.s_addr; 495 req->opt0 = cpu_to_be64(opt0); 496 req->params = 0; 497 req->opt2 = cpu_to_be32(opt2); 498 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 499 } 500 501 static void send_mpa_req(struct c4iw_ep *ep, struct sk_buff *skb, 502 u8 mpa_rev_to_use) 503 { 504 int mpalen, wrlen; 505 struct fw_ofld_tx_data_wr *req; 506 struct mpa_message *mpa; 507 struct mpa_v2_conn_params mpa_v2_params; 508 509 PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen); 510 511 BUG_ON(skb_cloned(skb)); 512 513 mpalen = sizeof(*mpa) + ep->plen; 514 if (mpa_rev_to_use == 2) 515 mpalen += sizeof(struct mpa_v2_conn_params); 516 wrlen = roundup(mpalen + sizeof *req, 16); 517 skb = get_skb(skb, wrlen, GFP_KERNEL); 518 if (!skb) { 519 connect_reply_upcall(ep, -ENOMEM); 520 return; 521 } 522 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 523 524 req = (struct fw_ofld_tx_data_wr *)skb_put(skb, wrlen); 525 memset(req, 0, wrlen); 526 req->op_to_immdlen = cpu_to_be32( 527 FW_WR_OP(FW_OFLD_TX_DATA_WR) | 528 FW_WR_COMPL(1) | 529 FW_WR_IMMDLEN(mpalen)); 530 req->flowid_len16 = cpu_to_be32( 531 FW_WR_FLOWID(ep->hwtid) | 532 FW_WR_LEN16(wrlen >> 4)); 533 req->plen = cpu_to_be32(mpalen); 534 req->tunnel_to_proxy = cpu_to_be32( 535 FW_OFLD_TX_DATA_WR_FLUSH(1) | 536 FW_OFLD_TX_DATA_WR_SHOVE(1)); 537 538 mpa = (struct mpa_message *)(req + 1); 539 memcpy(mpa->key, MPA_KEY_REQ, sizeof(mpa->key)); 540 mpa->flags = (crc_enabled ? MPA_CRC : 0) | 541 (markers_enabled ? MPA_MARKERS : 0) | 542 (mpa_rev_to_use == 2 ? MPA_ENHANCED_RDMA_CONN : 0); 543 mpa->private_data_size = htons(ep->plen); 544 mpa->revision = mpa_rev_to_use; 545 if (mpa_rev_to_use == 1) { 546 ep->tried_with_mpa_v1 = 1; 547 ep->retry_with_mpa_v1 = 0; 548 } 549 550 if (mpa_rev_to_use == 2) { 551 mpa->private_data_size += 552 htons(sizeof(struct mpa_v2_conn_params)); 553 mpa_v2_params.ird = htons((u16)ep->ird); 554 mpa_v2_params.ord = htons((u16)ep->ord); 555 556 if (peer2peer) { 557 mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); 558 if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) 559 mpa_v2_params.ord |= 560 htons(MPA_V2_RDMA_WRITE_RTR); 561 else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) 562 mpa_v2_params.ord |= 563 htons(MPA_V2_RDMA_READ_RTR); 564 } 565 memcpy(mpa->private_data, &mpa_v2_params, 566 sizeof(struct mpa_v2_conn_params)); 567 568 if (ep->plen) 569 memcpy(mpa->private_data + 570 sizeof(struct mpa_v2_conn_params), 571 ep->mpa_pkt + sizeof(*mpa), ep->plen); 572 } else 573 if (ep->plen) 574 memcpy(mpa->private_data, 575 ep->mpa_pkt + sizeof(*mpa), ep->plen); 576 577 /* 578 * Reference the mpa skb. This ensures the data area 579 * will remain in memory until the hw acks the tx. 580 * Function fw4_ack() will deref it. 581 */ 582 skb_get(skb); 583 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 584 BUG_ON(ep->mpa_skb); 585 ep->mpa_skb = skb; 586 c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 587 start_ep_timer(ep); 588 state_set(&ep->com, MPA_REQ_SENT); 589 ep->mpa_attr.initiator = 1; 590 return; 591 } 592 593 static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen) 594 { 595 int mpalen, wrlen; 596 struct fw_ofld_tx_data_wr *req; 597 struct mpa_message *mpa; 598 struct sk_buff *skb; 599 struct mpa_v2_conn_params mpa_v2_params; 600 601 PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen); 602 603 mpalen = sizeof(*mpa) + plen; 604 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) 605 mpalen += sizeof(struct mpa_v2_conn_params); 606 wrlen = roundup(mpalen + sizeof *req, 16); 607 608 skb = get_skb(NULL, wrlen, GFP_KERNEL); 609 if (!skb) { 610 printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __func__); 611 return -ENOMEM; 612 } 613 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 614 615 req = (struct fw_ofld_tx_data_wr *)skb_put(skb, wrlen); 616 memset(req, 0, wrlen); 617 req->op_to_immdlen = cpu_to_be32( 618 FW_WR_OP(FW_OFLD_TX_DATA_WR) | 619 FW_WR_COMPL(1) | 620 FW_WR_IMMDLEN(mpalen)); 621 req->flowid_len16 = cpu_to_be32( 622 FW_WR_FLOWID(ep->hwtid) | 623 FW_WR_LEN16(wrlen >> 4)); 624 req->plen = cpu_to_be32(mpalen); 625 req->tunnel_to_proxy = cpu_to_be32( 626 FW_OFLD_TX_DATA_WR_FLUSH(1) | 627 FW_OFLD_TX_DATA_WR_SHOVE(1)); 628 629 mpa = (struct mpa_message *)(req + 1); 630 memset(mpa, 0, sizeof(*mpa)); 631 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 632 mpa->flags = MPA_REJECT; 633 mpa->revision = mpa_rev; 634 mpa->private_data_size = htons(plen); 635 636 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 637 mpa->flags |= MPA_ENHANCED_RDMA_CONN; 638 mpa->private_data_size += 639 htons(sizeof(struct mpa_v2_conn_params)); 640 mpa_v2_params.ird = htons(((u16)ep->ird) | 641 (peer2peer ? MPA_V2_PEER2PEER_MODEL : 642 0)); 643 mpa_v2_params.ord = htons(((u16)ep->ord) | (peer2peer ? 644 (p2p_type == 645 FW_RI_INIT_P2PTYPE_RDMA_WRITE ? 646 MPA_V2_RDMA_WRITE_RTR : p2p_type == 647 FW_RI_INIT_P2PTYPE_READ_REQ ? 648 MPA_V2_RDMA_READ_RTR : 0) : 0)); 649 memcpy(mpa->private_data, &mpa_v2_params, 650 sizeof(struct mpa_v2_conn_params)); 651 652 if (ep->plen) 653 memcpy(mpa->private_data + 654 sizeof(struct mpa_v2_conn_params), pdata, plen); 655 } else 656 if (plen) 657 memcpy(mpa->private_data, pdata, plen); 658 659 /* 660 * Reference the mpa skb again. This ensures the data area 661 * will remain in memory until the hw acks the tx. 662 * Function fw4_ack() will deref it. 663 */ 664 skb_get(skb); 665 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 666 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 667 BUG_ON(ep->mpa_skb); 668 ep->mpa_skb = skb; 669 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 670 } 671 672 static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen) 673 { 674 int mpalen, wrlen; 675 struct fw_ofld_tx_data_wr *req; 676 struct mpa_message *mpa; 677 struct sk_buff *skb; 678 struct mpa_v2_conn_params mpa_v2_params; 679 680 PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen); 681 682 mpalen = sizeof(*mpa) + plen; 683 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) 684 mpalen += sizeof(struct mpa_v2_conn_params); 685 wrlen = roundup(mpalen + sizeof *req, 16); 686 687 skb = get_skb(NULL, wrlen, GFP_KERNEL); 688 if (!skb) { 689 printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __func__); 690 return -ENOMEM; 691 } 692 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 693 694 req = (struct fw_ofld_tx_data_wr *) skb_put(skb, wrlen); 695 memset(req, 0, wrlen); 696 req->op_to_immdlen = cpu_to_be32( 697 FW_WR_OP(FW_OFLD_TX_DATA_WR) | 698 FW_WR_COMPL(1) | 699 FW_WR_IMMDLEN(mpalen)); 700 req->flowid_len16 = cpu_to_be32( 701 FW_WR_FLOWID(ep->hwtid) | 702 FW_WR_LEN16(wrlen >> 4)); 703 req->plen = cpu_to_be32(mpalen); 704 req->tunnel_to_proxy = cpu_to_be32( 705 FW_OFLD_TX_DATA_WR_FLUSH(1) | 706 FW_OFLD_TX_DATA_WR_SHOVE(1)); 707 708 mpa = (struct mpa_message *)(req + 1); 709 memset(mpa, 0, sizeof(*mpa)); 710 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 711 mpa->flags = (ep->mpa_attr.crc_enabled ? MPA_CRC : 0) | 712 (markers_enabled ? MPA_MARKERS : 0); 713 mpa->revision = ep->mpa_attr.version; 714 mpa->private_data_size = htons(plen); 715 716 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 717 mpa->flags |= MPA_ENHANCED_RDMA_CONN; 718 mpa->private_data_size += 719 htons(sizeof(struct mpa_v2_conn_params)); 720 mpa_v2_params.ird = htons((u16)ep->ird); 721 mpa_v2_params.ord = htons((u16)ep->ord); 722 if (peer2peer && (ep->mpa_attr.p2p_type != 723 FW_RI_INIT_P2PTYPE_DISABLED)) { 724 mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); 725 726 if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) 727 mpa_v2_params.ord |= 728 htons(MPA_V2_RDMA_WRITE_RTR); 729 else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) 730 mpa_v2_params.ord |= 731 htons(MPA_V2_RDMA_READ_RTR); 732 } 733 734 memcpy(mpa->private_data, &mpa_v2_params, 735 sizeof(struct mpa_v2_conn_params)); 736 737 if (ep->plen) 738 memcpy(mpa->private_data + 739 sizeof(struct mpa_v2_conn_params), pdata, plen); 740 } else 741 if (plen) 742 memcpy(mpa->private_data, pdata, plen); 743 744 /* 745 * Reference the mpa skb. This ensures the data area 746 * will remain in memory until the hw acks the tx. 747 * Function fw4_ack() will deref it. 748 */ 749 skb_get(skb); 750 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 751 ep->mpa_skb = skb; 752 state_set(&ep->com, MPA_REP_SENT); 753 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 754 } 755 756 static int act_establish(struct c4iw_dev *dev, struct sk_buff *skb) 757 { 758 struct c4iw_ep *ep; 759 struct cpl_act_establish *req = cplhdr(skb); 760 unsigned int tid = GET_TID(req); 761 unsigned int atid = GET_TID_TID(ntohl(req->tos_atid)); 762 struct tid_info *t = dev->rdev.lldi.tids; 763 764 ep = lookup_atid(t, atid); 765 766 PDBG("%s ep %p tid %u snd_isn %u rcv_isn %u\n", __func__, ep, tid, 767 be32_to_cpu(req->snd_isn), be32_to_cpu(req->rcv_isn)); 768 769 dst_confirm(ep->dst); 770 771 /* setup the hwtid for this connection */ 772 ep->hwtid = tid; 773 cxgb4_insert_tid(t, ep, tid); 774 775 ep->snd_seq = be32_to_cpu(req->snd_isn); 776 ep->rcv_seq = be32_to_cpu(req->rcv_isn); 777 778 set_emss(ep, ntohs(req->tcp_opt)); 779 780 /* dealloc the atid */ 781 cxgb4_free_atid(t, atid); 782 783 /* start MPA negotiation */ 784 send_flowc(ep, NULL); 785 if (ep->retry_with_mpa_v1) 786 send_mpa_req(ep, skb, 1); 787 else 788 send_mpa_req(ep, skb, mpa_rev); 789 790 return 0; 791 } 792 793 static void close_complete_upcall(struct c4iw_ep *ep) 794 { 795 struct iw_cm_event event; 796 797 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 798 memset(&event, 0, sizeof(event)); 799 event.event = IW_CM_EVENT_CLOSE; 800 if (ep->com.cm_id) { 801 PDBG("close complete delivered ep %p cm_id %p tid %u\n", 802 ep, ep->com.cm_id, ep->hwtid); 803 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 804 ep->com.cm_id->rem_ref(ep->com.cm_id); 805 ep->com.cm_id = NULL; 806 ep->com.qp = NULL; 807 } 808 } 809 810 static int abort_connection(struct c4iw_ep *ep, struct sk_buff *skb, gfp_t gfp) 811 { 812 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 813 close_complete_upcall(ep); 814 state_set(&ep->com, ABORTING); 815 return send_abort(ep, skb, gfp); 816 } 817 818 static void peer_close_upcall(struct c4iw_ep *ep) 819 { 820 struct iw_cm_event event; 821 822 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 823 memset(&event, 0, sizeof(event)); 824 event.event = IW_CM_EVENT_DISCONNECT; 825 if (ep->com.cm_id) { 826 PDBG("peer close delivered ep %p cm_id %p tid %u\n", 827 ep, ep->com.cm_id, ep->hwtid); 828 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 829 } 830 } 831 832 static void peer_abort_upcall(struct c4iw_ep *ep) 833 { 834 struct iw_cm_event event; 835 836 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 837 memset(&event, 0, sizeof(event)); 838 event.event = IW_CM_EVENT_CLOSE; 839 event.status = -ECONNRESET; 840 if (ep->com.cm_id) { 841 PDBG("abort delivered ep %p cm_id %p tid %u\n", ep, 842 ep->com.cm_id, ep->hwtid); 843 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 844 ep->com.cm_id->rem_ref(ep->com.cm_id); 845 ep->com.cm_id = NULL; 846 ep->com.qp = NULL; 847 } 848 } 849 850 static void connect_reply_upcall(struct c4iw_ep *ep, int status) 851 { 852 struct iw_cm_event event; 853 854 PDBG("%s ep %p tid %u status %d\n", __func__, ep, ep->hwtid, status); 855 memset(&event, 0, sizeof(event)); 856 event.event = IW_CM_EVENT_CONNECT_REPLY; 857 event.status = status; 858 event.local_addr = ep->com.local_addr; 859 event.remote_addr = ep->com.remote_addr; 860 861 if ((status == 0) || (status == -ECONNREFUSED)) { 862 if (!ep->tried_with_mpa_v1) { 863 /* this means MPA_v2 is used */ 864 event.private_data_len = ep->plen - 865 sizeof(struct mpa_v2_conn_params); 866 event.private_data = ep->mpa_pkt + 867 sizeof(struct mpa_message) + 868 sizeof(struct mpa_v2_conn_params); 869 } else { 870 /* this means MPA_v1 is used */ 871 event.private_data_len = ep->plen; 872 event.private_data = ep->mpa_pkt + 873 sizeof(struct mpa_message); 874 } 875 } 876 877 PDBG("%s ep %p tid %u status %d\n", __func__, ep, 878 ep->hwtid, status); 879 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 880 881 if (status < 0) { 882 ep->com.cm_id->rem_ref(ep->com.cm_id); 883 ep->com.cm_id = NULL; 884 ep->com.qp = NULL; 885 } 886 } 887 888 static void connect_request_upcall(struct c4iw_ep *ep) 889 { 890 struct iw_cm_event event; 891 892 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 893 memset(&event, 0, sizeof(event)); 894 event.event = IW_CM_EVENT_CONNECT_REQUEST; 895 event.local_addr = ep->com.local_addr; 896 event.remote_addr = ep->com.remote_addr; 897 event.provider_data = ep; 898 if (!ep->tried_with_mpa_v1) { 899 /* this means MPA_v2 is used */ 900 event.ord = ep->ord; 901 event.ird = ep->ird; 902 event.private_data_len = ep->plen - 903 sizeof(struct mpa_v2_conn_params); 904 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message) + 905 sizeof(struct mpa_v2_conn_params); 906 } else { 907 /* this means MPA_v1 is used. Send max supported */ 908 event.ord = c4iw_max_read_depth; 909 event.ird = c4iw_max_read_depth; 910 event.private_data_len = ep->plen; 911 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message); 912 } 913 if (state_read(&ep->parent_ep->com) != DEAD) { 914 c4iw_get_ep(&ep->com); 915 ep->parent_ep->com.cm_id->event_handler( 916 ep->parent_ep->com.cm_id, 917 &event); 918 } 919 c4iw_put_ep(&ep->parent_ep->com); 920 ep->parent_ep = NULL; 921 } 922 923 static void established_upcall(struct c4iw_ep *ep) 924 { 925 struct iw_cm_event event; 926 927 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 928 memset(&event, 0, sizeof(event)); 929 event.event = IW_CM_EVENT_ESTABLISHED; 930 event.ird = ep->ird; 931 event.ord = ep->ord; 932 if (ep->com.cm_id) { 933 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 934 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 935 } 936 } 937 938 static int update_rx_credits(struct c4iw_ep *ep, u32 credits) 939 { 940 struct cpl_rx_data_ack *req; 941 struct sk_buff *skb; 942 int wrlen = roundup(sizeof *req, 16); 943 944 PDBG("%s ep %p tid %u credits %u\n", __func__, ep, ep->hwtid, credits); 945 skb = get_skb(NULL, wrlen, GFP_KERNEL); 946 if (!skb) { 947 printk(KERN_ERR MOD "update_rx_credits - cannot alloc skb!\n"); 948 return 0; 949 } 950 951 req = (struct cpl_rx_data_ack *) skb_put(skb, wrlen); 952 memset(req, 0, wrlen); 953 INIT_TP_WR(req, ep->hwtid); 954 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK, 955 ep->hwtid)); 956 req->credit_dack = cpu_to_be32(credits | RX_FORCE_ACK(1) | 957 F_RX_DACK_CHANGE | 958 V_RX_DACK_MODE(dack_mode)); 959 set_wr_txq(skb, CPL_PRIORITY_ACK, ep->ctrlq_idx); 960 c4iw_ofld_send(&ep->com.dev->rdev, skb); 961 return credits; 962 } 963 964 static void process_mpa_reply(struct c4iw_ep *ep, struct sk_buff *skb) 965 { 966 struct mpa_message *mpa; 967 struct mpa_v2_conn_params *mpa_v2_params; 968 u16 plen; 969 u16 resp_ird, resp_ord; 970 u8 rtr_mismatch = 0, insuff_ird = 0; 971 struct c4iw_qp_attributes attrs; 972 enum c4iw_qp_attr_mask mask; 973 int err; 974 975 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 976 977 /* 978 * Stop mpa timer. If it expired, then the state has 979 * changed and we bail since ep_timeout already aborted 980 * the connection. 981 */ 982 stop_ep_timer(ep); 983 if (state_read(&ep->com) != MPA_REQ_SENT) 984 return; 985 986 /* 987 * If we get more than the supported amount of private data 988 * then we must fail this connection. 989 */ 990 if (ep->mpa_pkt_len + skb->len > sizeof(ep->mpa_pkt)) { 991 err = -EINVAL; 992 goto err; 993 } 994 995 /* 996 * copy the new data into our accumulation buffer. 997 */ 998 skb_copy_from_linear_data(skb, &(ep->mpa_pkt[ep->mpa_pkt_len]), 999 skb->len); 1000 ep->mpa_pkt_len += skb->len; 1001 1002 /* 1003 * if we don't even have the mpa message, then bail. 1004 */ 1005 if (ep->mpa_pkt_len < sizeof(*mpa)) 1006 return; 1007 mpa = (struct mpa_message *) ep->mpa_pkt; 1008 1009 /* Validate MPA header. */ 1010 if (mpa->revision > mpa_rev) { 1011 printk(KERN_ERR MOD "%s MPA version mismatch. Local = %d," 1012 " Received = %d\n", __func__, mpa_rev, mpa->revision); 1013 err = -EPROTO; 1014 goto err; 1015 } 1016 if (memcmp(mpa->key, MPA_KEY_REP, sizeof(mpa->key))) { 1017 err = -EPROTO; 1018 goto err; 1019 } 1020 1021 plen = ntohs(mpa->private_data_size); 1022 1023 /* 1024 * Fail if there's too much private data. 1025 */ 1026 if (plen > MPA_MAX_PRIVATE_DATA) { 1027 err = -EPROTO; 1028 goto err; 1029 } 1030 1031 /* 1032 * If plen does not account for pkt size 1033 */ 1034 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { 1035 err = -EPROTO; 1036 goto err; 1037 } 1038 1039 ep->plen = (u8) plen; 1040 1041 /* 1042 * If we don't have all the pdata yet, then bail. 1043 * We'll continue process when more data arrives. 1044 */ 1045 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) 1046 return; 1047 1048 if (mpa->flags & MPA_REJECT) { 1049 err = -ECONNREFUSED; 1050 goto err; 1051 } 1052 1053 /* 1054 * If we get here we have accumulated the entire mpa 1055 * start reply message including private data. And 1056 * the MPA header is valid. 1057 */ 1058 state_set(&ep->com, FPDU_MODE); 1059 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 1060 ep->mpa_attr.recv_marker_enabled = markers_enabled; 1061 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 1062 ep->mpa_attr.version = mpa->revision; 1063 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1064 1065 if (mpa->revision == 2) { 1066 ep->mpa_attr.enhanced_rdma_conn = 1067 mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; 1068 if (ep->mpa_attr.enhanced_rdma_conn) { 1069 mpa_v2_params = (struct mpa_v2_conn_params *) 1070 (ep->mpa_pkt + sizeof(*mpa)); 1071 resp_ird = ntohs(mpa_v2_params->ird) & 1072 MPA_V2_IRD_ORD_MASK; 1073 resp_ord = ntohs(mpa_v2_params->ord) & 1074 MPA_V2_IRD_ORD_MASK; 1075 1076 /* 1077 * This is a double-check. Ideally, below checks are 1078 * not required since ird/ord stuff has been taken 1079 * care of in c4iw_accept_cr 1080 */ 1081 if ((ep->ird < resp_ord) || (ep->ord > resp_ird)) { 1082 err = -ENOMEM; 1083 ep->ird = resp_ord; 1084 ep->ord = resp_ird; 1085 insuff_ird = 1; 1086 } 1087 1088 if (ntohs(mpa_v2_params->ird) & 1089 MPA_V2_PEER2PEER_MODEL) { 1090 if (ntohs(mpa_v2_params->ord) & 1091 MPA_V2_RDMA_WRITE_RTR) 1092 ep->mpa_attr.p2p_type = 1093 FW_RI_INIT_P2PTYPE_RDMA_WRITE; 1094 else if (ntohs(mpa_v2_params->ord) & 1095 MPA_V2_RDMA_READ_RTR) 1096 ep->mpa_attr.p2p_type = 1097 FW_RI_INIT_P2PTYPE_READ_REQ; 1098 } 1099 } 1100 } else if (mpa->revision == 1) 1101 if (peer2peer) 1102 ep->mpa_attr.p2p_type = p2p_type; 1103 1104 PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, " 1105 "xmit_marker_enabled=%d, version=%d p2p_type=%d local-p2p_type = " 1106 "%d\n", __func__, ep->mpa_attr.crc_enabled, 1107 ep->mpa_attr.recv_marker_enabled, 1108 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version, 1109 ep->mpa_attr.p2p_type, p2p_type); 1110 1111 /* 1112 * If responder's RTR does not match with that of initiator, assign 1113 * FW_RI_INIT_P2PTYPE_DISABLED in mpa attributes so that RTR is not 1114 * generated when moving QP to RTS state. 1115 * A TERM message will be sent after QP has moved to RTS state 1116 */ 1117 if ((ep->mpa_attr.version == 2) && 1118 (ep->mpa_attr.p2p_type != p2p_type)) { 1119 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1120 rtr_mismatch = 1; 1121 } 1122 1123 attrs.mpa_attr = ep->mpa_attr; 1124 attrs.max_ird = ep->ird; 1125 attrs.max_ord = ep->ord; 1126 attrs.llp_stream_handle = ep; 1127 attrs.next_state = C4IW_QP_STATE_RTS; 1128 1129 mask = C4IW_QP_ATTR_NEXT_STATE | 1130 C4IW_QP_ATTR_LLP_STREAM_HANDLE | C4IW_QP_ATTR_MPA_ATTR | 1131 C4IW_QP_ATTR_MAX_IRD | C4IW_QP_ATTR_MAX_ORD; 1132 1133 /* bind QP and TID with INIT_WR */ 1134 err = c4iw_modify_qp(ep->com.qp->rhp, 1135 ep->com.qp, mask, &attrs, 1); 1136 if (err) 1137 goto err; 1138 1139 /* 1140 * If responder's RTR requirement did not match with what initiator 1141 * supports, generate TERM message 1142 */ 1143 if (rtr_mismatch) { 1144 printk(KERN_ERR "%s: RTR mismatch, sending TERM\n", __func__); 1145 attrs.layer_etype = LAYER_MPA | DDP_LLP; 1146 attrs.ecode = MPA_NOMATCH_RTR; 1147 attrs.next_state = C4IW_QP_STATE_TERMINATE; 1148 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1149 C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); 1150 err = -ENOMEM; 1151 goto out; 1152 } 1153 1154 /* 1155 * Generate TERM if initiator IRD is not sufficient for responder 1156 * provided ORD. Currently, we do the same behaviour even when 1157 * responder provided IRD is also not sufficient as regards to 1158 * initiator ORD. 1159 */ 1160 if (insuff_ird) { 1161 printk(KERN_ERR "%s: Insufficient IRD, sending TERM\n", 1162 __func__); 1163 attrs.layer_etype = LAYER_MPA | DDP_LLP; 1164 attrs.ecode = MPA_INSUFF_IRD; 1165 attrs.next_state = C4IW_QP_STATE_TERMINATE; 1166 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1167 C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); 1168 err = -ENOMEM; 1169 goto out; 1170 } 1171 goto out; 1172 err: 1173 state_set(&ep->com, ABORTING); 1174 send_abort(ep, skb, GFP_KERNEL); 1175 out: 1176 connect_reply_upcall(ep, err); 1177 return; 1178 } 1179 1180 static void process_mpa_request(struct c4iw_ep *ep, struct sk_buff *skb) 1181 { 1182 struct mpa_message *mpa; 1183 struct mpa_v2_conn_params *mpa_v2_params; 1184 u16 plen; 1185 1186 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1187 1188 if (state_read(&ep->com) != MPA_REQ_WAIT) 1189 return; 1190 1191 /* 1192 * If we get more than the supported amount of private data 1193 * then we must fail this connection. 1194 */ 1195 if (ep->mpa_pkt_len + skb->len > sizeof(ep->mpa_pkt)) { 1196 stop_ep_timer(ep); 1197 abort_connection(ep, skb, GFP_KERNEL); 1198 return; 1199 } 1200 1201 PDBG("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__); 1202 1203 /* 1204 * Copy the new data into our accumulation buffer. 1205 */ 1206 skb_copy_from_linear_data(skb, &(ep->mpa_pkt[ep->mpa_pkt_len]), 1207 skb->len); 1208 ep->mpa_pkt_len += skb->len; 1209 1210 /* 1211 * If we don't even have the mpa message, then bail. 1212 * We'll continue process when more data arrives. 1213 */ 1214 if (ep->mpa_pkt_len < sizeof(*mpa)) 1215 return; 1216 1217 PDBG("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__); 1218 stop_ep_timer(ep); 1219 mpa = (struct mpa_message *) ep->mpa_pkt; 1220 1221 /* 1222 * Validate MPA Header. 1223 */ 1224 if (mpa->revision > mpa_rev) { 1225 printk(KERN_ERR MOD "%s MPA version mismatch. Local = %d," 1226 " Received = %d\n", __func__, mpa_rev, mpa->revision); 1227 abort_connection(ep, skb, GFP_KERNEL); 1228 return; 1229 } 1230 1231 if (memcmp(mpa->key, MPA_KEY_REQ, sizeof(mpa->key))) { 1232 abort_connection(ep, skb, GFP_KERNEL); 1233 return; 1234 } 1235 1236 plen = ntohs(mpa->private_data_size); 1237 1238 /* 1239 * Fail if there's too much private data. 1240 */ 1241 if (plen > MPA_MAX_PRIVATE_DATA) { 1242 abort_connection(ep, skb, GFP_KERNEL); 1243 return; 1244 } 1245 1246 /* 1247 * If plen does not account for pkt size 1248 */ 1249 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { 1250 abort_connection(ep, skb, GFP_KERNEL); 1251 return; 1252 } 1253 ep->plen = (u8) plen; 1254 1255 /* 1256 * If we don't have all the pdata yet, then bail. 1257 */ 1258 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) 1259 return; 1260 1261 /* 1262 * If we get here we have accumulated the entire mpa 1263 * start reply message including private data. 1264 */ 1265 ep->mpa_attr.initiator = 0; 1266 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 1267 ep->mpa_attr.recv_marker_enabled = markers_enabled; 1268 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 1269 ep->mpa_attr.version = mpa->revision; 1270 if (mpa->revision == 1) 1271 ep->tried_with_mpa_v1 = 1; 1272 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1273 1274 if (mpa->revision == 2) { 1275 ep->mpa_attr.enhanced_rdma_conn = 1276 mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; 1277 if (ep->mpa_attr.enhanced_rdma_conn) { 1278 mpa_v2_params = (struct mpa_v2_conn_params *) 1279 (ep->mpa_pkt + sizeof(*mpa)); 1280 ep->ird = ntohs(mpa_v2_params->ird) & 1281 MPA_V2_IRD_ORD_MASK; 1282 ep->ord = ntohs(mpa_v2_params->ord) & 1283 MPA_V2_IRD_ORD_MASK; 1284 if (ntohs(mpa_v2_params->ird) & MPA_V2_PEER2PEER_MODEL) 1285 if (peer2peer) { 1286 if (ntohs(mpa_v2_params->ord) & 1287 MPA_V2_RDMA_WRITE_RTR) 1288 ep->mpa_attr.p2p_type = 1289 FW_RI_INIT_P2PTYPE_RDMA_WRITE; 1290 else if (ntohs(mpa_v2_params->ord) & 1291 MPA_V2_RDMA_READ_RTR) 1292 ep->mpa_attr.p2p_type = 1293 FW_RI_INIT_P2PTYPE_READ_REQ; 1294 } 1295 } 1296 } else if (mpa->revision == 1) 1297 if (peer2peer) 1298 ep->mpa_attr.p2p_type = p2p_type; 1299 1300 PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, " 1301 "xmit_marker_enabled=%d, version=%d p2p_type=%d\n", __func__, 1302 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled, 1303 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version, 1304 ep->mpa_attr.p2p_type); 1305 1306 state_set(&ep->com, MPA_REQ_RCVD); 1307 1308 /* drive upcall */ 1309 connect_request_upcall(ep); 1310 return; 1311 } 1312 1313 static int rx_data(struct c4iw_dev *dev, struct sk_buff *skb) 1314 { 1315 struct c4iw_ep *ep; 1316 struct cpl_rx_data *hdr = cplhdr(skb); 1317 unsigned int dlen = ntohs(hdr->len); 1318 unsigned int tid = GET_TID(hdr); 1319 struct tid_info *t = dev->rdev.lldi.tids; 1320 1321 ep = lookup_tid(t, tid); 1322 PDBG("%s ep %p tid %u dlen %u\n", __func__, ep, ep->hwtid, dlen); 1323 skb_pull(skb, sizeof(*hdr)); 1324 skb_trim(skb, dlen); 1325 1326 ep->rcv_seq += dlen; 1327 BUG_ON(ep->rcv_seq != (ntohl(hdr->seq) + dlen)); 1328 1329 /* update RX credits */ 1330 update_rx_credits(ep, dlen); 1331 1332 switch (state_read(&ep->com)) { 1333 case MPA_REQ_SENT: 1334 process_mpa_reply(ep, skb); 1335 break; 1336 case MPA_REQ_WAIT: 1337 process_mpa_request(ep, skb); 1338 break; 1339 case MPA_REP_SENT: 1340 break; 1341 default: 1342 printk(KERN_ERR MOD "%s Unexpected streaming data." 1343 " ep %p state %d tid %u\n", 1344 __func__, ep, state_read(&ep->com), ep->hwtid); 1345 1346 /* 1347 * The ep will timeout and inform the ULP of the failure. 1348 * See ep_timeout(). 1349 */ 1350 break; 1351 } 1352 return 0; 1353 } 1354 1355 static int abort_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1356 { 1357 struct c4iw_ep *ep; 1358 struct cpl_abort_rpl_rss *rpl = cplhdr(skb); 1359 int release = 0; 1360 unsigned int tid = GET_TID(rpl); 1361 struct tid_info *t = dev->rdev.lldi.tids; 1362 1363 ep = lookup_tid(t, tid); 1364 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1365 BUG_ON(!ep); 1366 mutex_lock(&ep->com.mutex); 1367 switch (ep->com.state) { 1368 case ABORTING: 1369 __state_set(&ep->com, DEAD); 1370 release = 1; 1371 break; 1372 default: 1373 printk(KERN_ERR "%s ep %p state %d\n", 1374 __func__, ep, ep->com.state); 1375 break; 1376 } 1377 mutex_unlock(&ep->com.mutex); 1378 1379 if (release) 1380 release_ep_resources(ep); 1381 return 0; 1382 } 1383 1384 /* 1385 * Return whether a failed active open has allocated a TID 1386 */ 1387 static inline int act_open_has_tid(int status) 1388 { 1389 return status != CPL_ERR_TCAM_FULL && status != CPL_ERR_CONN_EXIST && 1390 status != CPL_ERR_ARP_MISS; 1391 } 1392 1393 static int act_open_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1394 { 1395 struct c4iw_ep *ep; 1396 struct cpl_act_open_rpl *rpl = cplhdr(skb); 1397 unsigned int atid = GET_TID_TID(GET_AOPEN_ATID( 1398 ntohl(rpl->atid_status))); 1399 struct tid_info *t = dev->rdev.lldi.tids; 1400 int status = GET_AOPEN_STATUS(ntohl(rpl->atid_status)); 1401 1402 ep = lookup_atid(t, atid); 1403 1404 PDBG("%s ep %p atid %u status %u errno %d\n", __func__, ep, atid, 1405 status, status2errno(status)); 1406 1407 if (status == CPL_ERR_RTX_NEG_ADVICE) { 1408 printk(KERN_WARNING MOD "Connection problems for atid %u\n", 1409 atid); 1410 return 0; 1411 } 1412 1413 connect_reply_upcall(ep, status2errno(status)); 1414 state_set(&ep->com, DEAD); 1415 1416 if (status && act_open_has_tid(status)) 1417 cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, GET_TID(rpl)); 1418 1419 cxgb4_free_atid(t, atid); 1420 dst_release(ep->dst); 1421 cxgb4_l2t_release(ep->l2t); 1422 c4iw_put_ep(&ep->com); 1423 1424 return 0; 1425 } 1426 1427 static int pass_open_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1428 { 1429 struct cpl_pass_open_rpl *rpl = cplhdr(skb); 1430 struct tid_info *t = dev->rdev.lldi.tids; 1431 unsigned int stid = GET_TID(rpl); 1432 struct c4iw_listen_ep *ep = lookup_stid(t, stid); 1433 1434 if (!ep) { 1435 printk(KERN_ERR MOD "stid %d lookup failure!\n", stid); 1436 return 0; 1437 } 1438 PDBG("%s ep %p status %d error %d\n", __func__, ep, 1439 rpl->status, status2errno(rpl->status)); 1440 c4iw_wake_up(&ep->com.wr_wait, status2errno(rpl->status)); 1441 1442 return 0; 1443 } 1444 1445 static int listen_stop(struct c4iw_listen_ep *ep) 1446 { 1447 struct sk_buff *skb; 1448 struct cpl_close_listsvr_req *req; 1449 1450 PDBG("%s ep %p\n", __func__, ep); 1451 skb = get_skb(NULL, sizeof(*req), GFP_KERNEL); 1452 if (!skb) { 1453 printk(KERN_ERR MOD "%s - failed to alloc skb\n", __func__); 1454 return -ENOMEM; 1455 } 1456 req = (struct cpl_close_listsvr_req *) skb_put(skb, sizeof(*req)); 1457 INIT_TP_WR(req, 0); 1458 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, 1459 ep->stid)); 1460 req->reply_ctrl = cpu_to_be16( 1461 QUEUENO(ep->com.dev->rdev.lldi.rxq_ids[0])); 1462 set_wr_txq(skb, CPL_PRIORITY_SETUP, 0); 1463 return c4iw_ofld_send(&ep->com.dev->rdev, skb); 1464 } 1465 1466 static int close_listsrv_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1467 { 1468 struct cpl_close_listsvr_rpl *rpl = cplhdr(skb); 1469 struct tid_info *t = dev->rdev.lldi.tids; 1470 unsigned int stid = GET_TID(rpl); 1471 struct c4iw_listen_ep *ep = lookup_stid(t, stid); 1472 1473 PDBG("%s ep %p\n", __func__, ep); 1474 c4iw_wake_up(&ep->com.wr_wait, status2errno(rpl->status)); 1475 return 0; 1476 } 1477 1478 static void accept_cr(struct c4iw_ep *ep, __be32 peer_ip, struct sk_buff *skb, 1479 struct cpl_pass_accept_req *req) 1480 { 1481 struct cpl_pass_accept_rpl *rpl; 1482 unsigned int mtu_idx; 1483 u64 opt0; 1484 u32 opt2; 1485 int wscale; 1486 1487 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1488 BUG_ON(skb_cloned(skb)); 1489 skb_trim(skb, sizeof(*rpl)); 1490 skb_get(skb); 1491 cxgb4_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx); 1492 wscale = compute_wscale(rcv_win); 1493 opt0 = KEEP_ALIVE(1) | 1494 DELACK(1) | 1495 WND_SCALE(wscale) | 1496 MSS_IDX(mtu_idx) | 1497 L2T_IDX(ep->l2t->idx) | 1498 TX_CHAN(ep->tx_chan) | 1499 SMAC_SEL(ep->smac_idx) | 1500 DSCP(ep->tos) | 1501 ULP_MODE(ULP_MODE_TCPDDP) | 1502 RCV_BUFSIZ(rcv_win>>10); 1503 opt2 = RX_CHANNEL(0) | 1504 RSS_QUEUE_VALID | RSS_QUEUE(ep->rss_qid); 1505 1506 if (enable_tcp_timestamps && req->tcpopt.tstamp) 1507 opt2 |= TSTAMPS_EN(1); 1508 if (enable_tcp_sack && req->tcpopt.sack) 1509 opt2 |= SACK_EN(1); 1510 if (wscale && enable_tcp_window_scaling) 1511 opt2 |= WND_SCALE_EN(1); 1512 1513 rpl = cplhdr(skb); 1514 INIT_TP_WR(rpl, ep->hwtid); 1515 OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL, 1516 ep->hwtid)); 1517 rpl->opt0 = cpu_to_be64(opt0); 1518 rpl->opt2 = cpu_to_be32(opt2); 1519 set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->ctrlq_idx); 1520 c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 1521 1522 return; 1523 } 1524 1525 static void reject_cr(struct c4iw_dev *dev, u32 hwtid, __be32 peer_ip, 1526 struct sk_buff *skb) 1527 { 1528 PDBG("%s c4iw_dev %p tid %u peer_ip %x\n", __func__, dev, hwtid, 1529 peer_ip); 1530 BUG_ON(skb_cloned(skb)); 1531 skb_trim(skb, sizeof(struct cpl_tid_release)); 1532 skb_get(skb); 1533 release_tid(&dev->rdev, hwtid, skb); 1534 return; 1535 } 1536 1537 static void get_4tuple(struct cpl_pass_accept_req *req, 1538 __be32 *local_ip, __be32 *peer_ip, 1539 __be16 *local_port, __be16 *peer_port) 1540 { 1541 int eth_len = G_ETH_HDR_LEN(be32_to_cpu(req->hdr_len)); 1542 int ip_len = G_IP_HDR_LEN(be32_to_cpu(req->hdr_len)); 1543 struct iphdr *ip = (struct iphdr *)((u8 *)(req + 1) + eth_len); 1544 struct tcphdr *tcp = (struct tcphdr *) 1545 ((u8 *)(req + 1) + eth_len + ip_len); 1546 1547 PDBG("%s saddr 0x%x daddr 0x%x sport %u dport %u\n", __func__, 1548 ntohl(ip->saddr), ntohl(ip->daddr), ntohs(tcp->source), 1549 ntohs(tcp->dest)); 1550 1551 *peer_ip = ip->saddr; 1552 *local_ip = ip->daddr; 1553 *peer_port = tcp->source; 1554 *local_port = tcp->dest; 1555 1556 return; 1557 } 1558 1559 static int import_ep(struct c4iw_ep *ep, __be32 peer_ip, struct dst_entry *dst, 1560 struct c4iw_dev *cdev, bool clear_mpa_v1) 1561 { 1562 struct neighbour *n; 1563 int err, step; 1564 1565 rcu_read_lock(); 1566 n = dst_get_neighbour_noref(dst); 1567 err = -ENODEV; 1568 if (!n) 1569 goto out; 1570 err = -ENOMEM; 1571 if (n->dev->flags & IFF_LOOPBACK) { 1572 struct net_device *pdev; 1573 1574 pdev = ip_dev_find(&init_net, peer_ip); 1575 ep->l2t = cxgb4_l2t_get(cdev->rdev.lldi.l2t, 1576 n, pdev, 0); 1577 if (!ep->l2t) 1578 goto out; 1579 ep->mtu = pdev->mtu; 1580 ep->tx_chan = cxgb4_port_chan(pdev); 1581 ep->smac_idx = (cxgb4_port_viid(pdev) & 0x7F) << 1; 1582 step = cdev->rdev.lldi.ntxq / 1583 cdev->rdev.lldi.nchan; 1584 ep->txq_idx = cxgb4_port_idx(pdev) * step; 1585 step = cdev->rdev.lldi.nrxq / 1586 cdev->rdev.lldi.nchan; 1587 ep->ctrlq_idx = cxgb4_port_idx(pdev); 1588 ep->rss_qid = cdev->rdev.lldi.rxq_ids[ 1589 cxgb4_port_idx(pdev) * step]; 1590 dev_put(pdev); 1591 } else { 1592 ep->l2t = cxgb4_l2t_get(cdev->rdev.lldi.l2t, 1593 n, n->dev, 0); 1594 if (!ep->l2t) 1595 goto out; 1596 ep->mtu = dst_mtu(ep->dst); 1597 ep->tx_chan = cxgb4_port_chan(n->dev); 1598 ep->smac_idx = (cxgb4_port_viid(n->dev) & 0x7F) << 1; 1599 step = cdev->rdev.lldi.ntxq / 1600 cdev->rdev.lldi.nchan; 1601 ep->txq_idx = cxgb4_port_idx(n->dev) * step; 1602 ep->ctrlq_idx = cxgb4_port_idx(n->dev); 1603 step = cdev->rdev.lldi.nrxq / 1604 cdev->rdev.lldi.nchan; 1605 ep->rss_qid = cdev->rdev.lldi.rxq_ids[ 1606 cxgb4_port_idx(n->dev) * step]; 1607 1608 if (clear_mpa_v1) { 1609 ep->retry_with_mpa_v1 = 0; 1610 ep->tried_with_mpa_v1 = 0; 1611 } 1612 } 1613 err = 0; 1614 out: 1615 rcu_read_unlock(); 1616 1617 return err; 1618 } 1619 1620 static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb) 1621 { 1622 struct c4iw_ep *child_ep, *parent_ep; 1623 struct cpl_pass_accept_req *req = cplhdr(skb); 1624 unsigned int stid = GET_POPEN_TID(ntohl(req->tos_stid)); 1625 struct tid_info *t = dev->rdev.lldi.tids; 1626 unsigned int hwtid = GET_TID(req); 1627 struct dst_entry *dst; 1628 struct rtable *rt; 1629 __be32 local_ip, peer_ip; 1630 __be16 local_port, peer_port; 1631 int err; 1632 1633 parent_ep = lookup_stid(t, stid); 1634 PDBG("%s parent ep %p tid %u\n", __func__, parent_ep, hwtid); 1635 1636 get_4tuple(req, &local_ip, &peer_ip, &local_port, &peer_port); 1637 1638 if (state_read(&parent_ep->com) != LISTEN) { 1639 printk(KERN_ERR "%s - listening ep not in LISTEN\n", 1640 __func__); 1641 goto reject; 1642 } 1643 1644 /* Find output route */ 1645 rt = find_route(dev, local_ip, peer_ip, local_port, peer_port, 1646 GET_POPEN_TOS(ntohl(req->tos_stid))); 1647 if (!rt) { 1648 printk(KERN_ERR MOD "%s - failed to find dst entry!\n", 1649 __func__); 1650 goto reject; 1651 } 1652 dst = &rt->dst; 1653 1654 child_ep = alloc_ep(sizeof(*child_ep), GFP_KERNEL); 1655 if (!child_ep) { 1656 printk(KERN_ERR MOD "%s - failed to allocate ep entry!\n", 1657 __func__); 1658 dst_release(dst); 1659 goto reject; 1660 } 1661 1662 err = import_ep(child_ep, peer_ip, dst, dev, false); 1663 if (err) { 1664 printk(KERN_ERR MOD "%s - failed to allocate l2t entry!\n", 1665 __func__); 1666 dst_release(dst); 1667 kfree(child_ep); 1668 goto reject; 1669 } 1670 1671 state_set(&child_ep->com, CONNECTING); 1672 child_ep->com.dev = dev; 1673 child_ep->com.cm_id = NULL; 1674 child_ep->com.local_addr.sin_family = PF_INET; 1675 child_ep->com.local_addr.sin_port = local_port; 1676 child_ep->com.local_addr.sin_addr.s_addr = local_ip; 1677 child_ep->com.remote_addr.sin_family = PF_INET; 1678 child_ep->com.remote_addr.sin_port = peer_port; 1679 child_ep->com.remote_addr.sin_addr.s_addr = peer_ip; 1680 c4iw_get_ep(&parent_ep->com); 1681 child_ep->parent_ep = parent_ep; 1682 child_ep->tos = GET_POPEN_TOS(ntohl(req->tos_stid)); 1683 child_ep->dst = dst; 1684 child_ep->hwtid = hwtid; 1685 1686 PDBG("%s tx_chan %u smac_idx %u rss_qid %u\n", __func__, 1687 child_ep->tx_chan, child_ep->smac_idx, child_ep->rss_qid); 1688 1689 init_timer(&child_ep->timer); 1690 cxgb4_insert_tid(t, child_ep, hwtid); 1691 accept_cr(child_ep, peer_ip, skb, req); 1692 goto out; 1693 reject: 1694 reject_cr(dev, hwtid, peer_ip, skb); 1695 out: 1696 return 0; 1697 } 1698 1699 static int pass_establish(struct c4iw_dev *dev, struct sk_buff *skb) 1700 { 1701 struct c4iw_ep *ep; 1702 struct cpl_pass_establish *req = cplhdr(skb); 1703 struct tid_info *t = dev->rdev.lldi.tids; 1704 unsigned int tid = GET_TID(req); 1705 1706 ep = lookup_tid(t, tid); 1707 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1708 ep->snd_seq = be32_to_cpu(req->snd_isn); 1709 ep->rcv_seq = be32_to_cpu(req->rcv_isn); 1710 1711 set_emss(ep, ntohs(req->tcp_opt)); 1712 1713 dst_confirm(ep->dst); 1714 state_set(&ep->com, MPA_REQ_WAIT); 1715 start_ep_timer(ep); 1716 send_flowc(ep, skb); 1717 1718 return 0; 1719 } 1720 1721 static int peer_close(struct c4iw_dev *dev, struct sk_buff *skb) 1722 { 1723 struct cpl_peer_close *hdr = cplhdr(skb); 1724 struct c4iw_ep *ep; 1725 struct c4iw_qp_attributes attrs; 1726 int disconnect = 1; 1727 int release = 0; 1728 struct tid_info *t = dev->rdev.lldi.tids; 1729 unsigned int tid = GET_TID(hdr); 1730 int ret; 1731 1732 ep = lookup_tid(t, tid); 1733 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1734 dst_confirm(ep->dst); 1735 1736 mutex_lock(&ep->com.mutex); 1737 switch (ep->com.state) { 1738 case MPA_REQ_WAIT: 1739 __state_set(&ep->com, CLOSING); 1740 break; 1741 case MPA_REQ_SENT: 1742 __state_set(&ep->com, CLOSING); 1743 connect_reply_upcall(ep, -ECONNRESET); 1744 break; 1745 case MPA_REQ_RCVD: 1746 1747 /* 1748 * We're gonna mark this puppy DEAD, but keep 1749 * the reference on it until the ULP accepts or 1750 * rejects the CR. Also wake up anyone waiting 1751 * in rdma connection migration (see c4iw_accept_cr()). 1752 */ 1753 __state_set(&ep->com, CLOSING); 1754 PDBG("waking up ep %p tid %u\n", ep, ep->hwtid); 1755 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 1756 break; 1757 case MPA_REP_SENT: 1758 __state_set(&ep->com, CLOSING); 1759 PDBG("waking up ep %p tid %u\n", ep, ep->hwtid); 1760 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 1761 break; 1762 case FPDU_MODE: 1763 start_ep_timer(ep); 1764 __state_set(&ep->com, CLOSING); 1765 attrs.next_state = C4IW_QP_STATE_CLOSING; 1766 ret = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1767 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 1768 if (ret != -ECONNRESET) { 1769 peer_close_upcall(ep); 1770 disconnect = 1; 1771 } 1772 break; 1773 case ABORTING: 1774 disconnect = 0; 1775 break; 1776 case CLOSING: 1777 __state_set(&ep->com, MORIBUND); 1778 disconnect = 0; 1779 break; 1780 case MORIBUND: 1781 stop_ep_timer(ep); 1782 if (ep->com.cm_id && ep->com.qp) { 1783 attrs.next_state = C4IW_QP_STATE_IDLE; 1784 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1785 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 1786 } 1787 close_complete_upcall(ep); 1788 __state_set(&ep->com, DEAD); 1789 release = 1; 1790 disconnect = 0; 1791 break; 1792 case DEAD: 1793 disconnect = 0; 1794 break; 1795 default: 1796 BUG_ON(1); 1797 } 1798 mutex_unlock(&ep->com.mutex); 1799 if (disconnect) 1800 c4iw_ep_disconnect(ep, 0, GFP_KERNEL); 1801 if (release) 1802 release_ep_resources(ep); 1803 return 0; 1804 } 1805 1806 /* 1807 * Returns whether an ABORT_REQ_RSS message is a negative advice. 1808 */ 1809 static int is_neg_adv_abort(unsigned int status) 1810 { 1811 return status == CPL_ERR_RTX_NEG_ADVICE || 1812 status == CPL_ERR_PERSIST_NEG_ADVICE; 1813 } 1814 1815 static int c4iw_reconnect(struct c4iw_ep *ep) 1816 { 1817 struct rtable *rt; 1818 int err = 0; 1819 1820 PDBG("%s qp %p cm_id %p\n", __func__, ep->com.qp, ep->com.cm_id); 1821 init_timer(&ep->timer); 1822 1823 /* 1824 * Allocate an active TID to initiate a TCP connection. 1825 */ 1826 ep->atid = cxgb4_alloc_atid(ep->com.dev->rdev.lldi.tids, ep); 1827 if (ep->atid == -1) { 1828 printk(KERN_ERR MOD "%s - cannot alloc atid.\n", __func__); 1829 err = -ENOMEM; 1830 goto fail2; 1831 } 1832 1833 /* find a route */ 1834 rt = find_route(ep->com.dev, 1835 ep->com.cm_id->local_addr.sin_addr.s_addr, 1836 ep->com.cm_id->remote_addr.sin_addr.s_addr, 1837 ep->com.cm_id->local_addr.sin_port, 1838 ep->com.cm_id->remote_addr.sin_port, 0); 1839 if (!rt) { 1840 printk(KERN_ERR MOD "%s - cannot find route.\n", __func__); 1841 err = -EHOSTUNREACH; 1842 goto fail3; 1843 } 1844 ep->dst = &rt->dst; 1845 1846 err = import_ep(ep, ep->com.cm_id->remote_addr.sin_addr.s_addr, 1847 ep->dst, ep->com.dev, false); 1848 if (err) { 1849 printk(KERN_ERR MOD "%s - cannot alloc l2e.\n", __func__); 1850 goto fail4; 1851 } 1852 1853 PDBG("%s txq_idx %u tx_chan %u smac_idx %u rss_qid %u l2t_idx %u\n", 1854 __func__, ep->txq_idx, ep->tx_chan, ep->smac_idx, ep->rss_qid, 1855 ep->l2t->idx); 1856 1857 state_set(&ep->com, CONNECTING); 1858 ep->tos = 0; 1859 1860 /* send connect request to rnic */ 1861 err = send_connect(ep); 1862 if (!err) 1863 goto out; 1864 1865 cxgb4_l2t_release(ep->l2t); 1866 fail4: 1867 dst_release(ep->dst); 1868 fail3: 1869 cxgb4_free_atid(ep->com.dev->rdev.lldi.tids, ep->atid); 1870 fail2: 1871 /* 1872 * remember to send notification to upper layer. 1873 * We are in here so the upper layer is not aware that this is 1874 * re-connect attempt and so, upper layer is still waiting for 1875 * response of 1st connect request. 1876 */ 1877 connect_reply_upcall(ep, -ECONNRESET); 1878 c4iw_put_ep(&ep->com); 1879 out: 1880 return err; 1881 } 1882 1883 static int peer_abort(struct c4iw_dev *dev, struct sk_buff *skb) 1884 { 1885 struct cpl_abort_req_rss *req = cplhdr(skb); 1886 struct c4iw_ep *ep; 1887 struct cpl_abort_rpl *rpl; 1888 struct sk_buff *rpl_skb; 1889 struct c4iw_qp_attributes attrs; 1890 int ret; 1891 int release = 0; 1892 struct tid_info *t = dev->rdev.lldi.tids; 1893 unsigned int tid = GET_TID(req); 1894 1895 ep = lookup_tid(t, tid); 1896 if (is_neg_adv_abort(req->status)) { 1897 PDBG("%s neg_adv_abort ep %p tid %u\n", __func__, ep, 1898 ep->hwtid); 1899 return 0; 1900 } 1901 PDBG("%s ep %p tid %u state %u\n", __func__, ep, ep->hwtid, 1902 ep->com.state); 1903 1904 /* 1905 * Wake up any threads in rdma_init() or rdma_fini(). 1906 * However, this is not needed if com state is just 1907 * MPA_REQ_SENT 1908 */ 1909 if (ep->com.state != MPA_REQ_SENT) 1910 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 1911 1912 mutex_lock(&ep->com.mutex); 1913 switch (ep->com.state) { 1914 case CONNECTING: 1915 break; 1916 case MPA_REQ_WAIT: 1917 stop_ep_timer(ep); 1918 break; 1919 case MPA_REQ_SENT: 1920 stop_ep_timer(ep); 1921 if (mpa_rev == 2 && ep->tried_with_mpa_v1) 1922 connect_reply_upcall(ep, -ECONNRESET); 1923 else { 1924 /* 1925 * we just don't send notification upwards because we 1926 * want to retry with mpa_v1 without upper layers even 1927 * knowing it. 1928 * 1929 * do some housekeeping so as to re-initiate the 1930 * connection 1931 */ 1932 PDBG("%s: mpa_rev=%d. Retrying with mpav1\n", __func__, 1933 mpa_rev); 1934 ep->retry_with_mpa_v1 = 1; 1935 } 1936 break; 1937 case MPA_REP_SENT: 1938 break; 1939 case MPA_REQ_RCVD: 1940 break; 1941 case MORIBUND: 1942 case CLOSING: 1943 stop_ep_timer(ep); 1944 /*FALLTHROUGH*/ 1945 case FPDU_MODE: 1946 if (ep->com.cm_id && ep->com.qp) { 1947 attrs.next_state = C4IW_QP_STATE_ERROR; 1948 ret = c4iw_modify_qp(ep->com.qp->rhp, 1949 ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, 1950 &attrs, 1); 1951 if (ret) 1952 printk(KERN_ERR MOD 1953 "%s - qp <- error failed!\n", 1954 __func__); 1955 } 1956 peer_abort_upcall(ep); 1957 break; 1958 case ABORTING: 1959 break; 1960 case DEAD: 1961 PDBG("%s PEER_ABORT IN DEAD STATE!!!!\n", __func__); 1962 mutex_unlock(&ep->com.mutex); 1963 return 0; 1964 default: 1965 BUG_ON(1); 1966 break; 1967 } 1968 dst_confirm(ep->dst); 1969 if (ep->com.state != ABORTING) { 1970 __state_set(&ep->com, DEAD); 1971 /* we don't release if we want to retry with mpa_v1 */ 1972 if (!ep->retry_with_mpa_v1) 1973 release = 1; 1974 } 1975 mutex_unlock(&ep->com.mutex); 1976 1977 rpl_skb = get_skb(skb, sizeof(*rpl), GFP_KERNEL); 1978 if (!rpl_skb) { 1979 printk(KERN_ERR MOD "%s - cannot allocate skb!\n", 1980 __func__); 1981 release = 1; 1982 goto out; 1983 } 1984 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 1985 rpl = (struct cpl_abort_rpl *) skb_put(rpl_skb, sizeof(*rpl)); 1986 INIT_TP_WR(rpl, ep->hwtid); 1987 OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL, ep->hwtid)); 1988 rpl->cmd = CPL_ABORT_NO_RST; 1989 c4iw_ofld_send(&ep->com.dev->rdev, rpl_skb); 1990 out: 1991 if (release) 1992 release_ep_resources(ep); 1993 1994 /* retry with mpa-v1 */ 1995 if (ep && ep->retry_with_mpa_v1) { 1996 cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, ep->hwtid); 1997 dst_release(ep->dst); 1998 cxgb4_l2t_release(ep->l2t); 1999 c4iw_reconnect(ep); 2000 } 2001 2002 return 0; 2003 } 2004 2005 static int close_con_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 2006 { 2007 struct c4iw_ep *ep; 2008 struct c4iw_qp_attributes attrs; 2009 struct cpl_close_con_rpl *rpl = cplhdr(skb); 2010 int release = 0; 2011 struct tid_info *t = dev->rdev.lldi.tids; 2012 unsigned int tid = GET_TID(rpl); 2013 2014 ep = lookup_tid(t, tid); 2015 2016 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2017 BUG_ON(!ep); 2018 2019 /* The cm_id may be null if we failed to connect */ 2020 mutex_lock(&ep->com.mutex); 2021 switch (ep->com.state) { 2022 case CLOSING: 2023 __state_set(&ep->com, MORIBUND); 2024 break; 2025 case MORIBUND: 2026 stop_ep_timer(ep); 2027 if ((ep->com.cm_id) && (ep->com.qp)) { 2028 attrs.next_state = C4IW_QP_STATE_IDLE; 2029 c4iw_modify_qp(ep->com.qp->rhp, 2030 ep->com.qp, 2031 C4IW_QP_ATTR_NEXT_STATE, 2032 &attrs, 1); 2033 } 2034 close_complete_upcall(ep); 2035 __state_set(&ep->com, DEAD); 2036 release = 1; 2037 break; 2038 case ABORTING: 2039 case DEAD: 2040 break; 2041 default: 2042 BUG_ON(1); 2043 break; 2044 } 2045 mutex_unlock(&ep->com.mutex); 2046 if (release) 2047 release_ep_resources(ep); 2048 return 0; 2049 } 2050 2051 static int terminate(struct c4iw_dev *dev, struct sk_buff *skb) 2052 { 2053 struct cpl_rdma_terminate *rpl = cplhdr(skb); 2054 struct tid_info *t = dev->rdev.lldi.tids; 2055 unsigned int tid = GET_TID(rpl); 2056 struct c4iw_ep *ep; 2057 struct c4iw_qp_attributes attrs; 2058 2059 ep = lookup_tid(t, tid); 2060 BUG_ON(!ep); 2061 2062 if (ep && ep->com.qp) { 2063 printk(KERN_WARNING MOD "TERM received tid %u qpid %u\n", tid, 2064 ep->com.qp->wq.sq.qid); 2065 attrs.next_state = C4IW_QP_STATE_TERMINATE; 2066 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 2067 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 2068 } else 2069 printk(KERN_WARNING MOD "TERM received tid %u no ep/qp\n", tid); 2070 2071 return 0; 2072 } 2073 2074 /* 2075 * Upcall from the adapter indicating data has been transmitted. 2076 * For us its just the single MPA request or reply. We can now free 2077 * the skb holding the mpa message. 2078 */ 2079 static int fw4_ack(struct c4iw_dev *dev, struct sk_buff *skb) 2080 { 2081 struct c4iw_ep *ep; 2082 struct cpl_fw4_ack *hdr = cplhdr(skb); 2083 u8 credits = hdr->credits; 2084 unsigned int tid = GET_TID(hdr); 2085 struct tid_info *t = dev->rdev.lldi.tids; 2086 2087 2088 ep = lookup_tid(t, tid); 2089 PDBG("%s ep %p tid %u credits %u\n", __func__, ep, ep->hwtid, credits); 2090 if (credits == 0) { 2091 PDBG("%s 0 credit ack ep %p tid %u state %u\n", 2092 __func__, ep, ep->hwtid, state_read(&ep->com)); 2093 return 0; 2094 } 2095 2096 dst_confirm(ep->dst); 2097 if (ep->mpa_skb) { 2098 PDBG("%s last streaming msg ack ep %p tid %u state %u " 2099 "initiator %u freeing skb\n", __func__, ep, ep->hwtid, 2100 state_read(&ep->com), ep->mpa_attr.initiator ? 1 : 0); 2101 kfree_skb(ep->mpa_skb); 2102 ep->mpa_skb = NULL; 2103 } 2104 return 0; 2105 } 2106 2107 int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) 2108 { 2109 int err; 2110 struct c4iw_ep *ep = to_ep(cm_id); 2111 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2112 2113 if (state_read(&ep->com) == DEAD) { 2114 c4iw_put_ep(&ep->com); 2115 return -ECONNRESET; 2116 } 2117 BUG_ON(state_read(&ep->com) != MPA_REQ_RCVD); 2118 if (mpa_rev == 0) 2119 abort_connection(ep, NULL, GFP_KERNEL); 2120 else { 2121 err = send_mpa_reject(ep, pdata, pdata_len); 2122 err = c4iw_ep_disconnect(ep, 0, GFP_KERNEL); 2123 } 2124 c4iw_put_ep(&ep->com); 2125 return 0; 2126 } 2127 2128 int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 2129 { 2130 int err; 2131 struct c4iw_qp_attributes attrs; 2132 enum c4iw_qp_attr_mask mask; 2133 struct c4iw_ep *ep = to_ep(cm_id); 2134 struct c4iw_dev *h = to_c4iw_dev(cm_id->device); 2135 struct c4iw_qp *qp = get_qhp(h, conn_param->qpn); 2136 2137 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2138 if (state_read(&ep->com) == DEAD) { 2139 err = -ECONNRESET; 2140 goto err; 2141 } 2142 2143 BUG_ON(state_read(&ep->com) != MPA_REQ_RCVD); 2144 BUG_ON(!qp); 2145 2146 if ((conn_param->ord > c4iw_max_read_depth) || 2147 (conn_param->ird > c4iw_max_read_depth)) { 2148 abort_connection(ep, NULL, GFP_KERNEL); 2149 err = -EINVAL; 2150 goto err; 2151 } 2152 2153 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 2154 if (conn_param->ord > ep->ird) { 2155 ep->ird = conn_param->ird; 2156 ep->ord = conn_param->ord; 2157 send_mpa_reject(ep, conn_param->private_data, 2158 conn_param->private_data_len); 2159 abort_connection(ep, NULL, GFP_KERNEL); 2160 err = -ENOMEM; 2161 goto err; 2162 } 2163 if (conn_param->ird > ep->ord) { 2164 if (!ep->ord) 2165 conn_param->ird = 1; 2166 else { 2167 abort_connection(ep, NULL, GFP_KERNEL); 2168 err = -ENOMEM; 2169 goto err; 2170 } 2171 } 2172 2173 } 2174 ep->ird = conn_param->ird; 2175 ep->ord = conn_param->ord; 2176 2177 if (ep->mpa_attr.version != 2) 2178 if (peer2peer && ep->ird == 0) 2179 ep->ird = 1; 2180 2181 PDBG("%s %d ird %d ord %d\n", __func__, __LINE__, ep->ird, ep->ord); 2182 2183 cm_id->add_ref(cm_id); 2184 ep->com.cm_id = cm_id; 2185 ep->com.qp = qp; 2186 2187 /* bind QP to EP and move to RTS */ 2188 attrs.mpa_attr = ep->mpa_attr; 2189 attrs.max_ird = ep->ird; 2190 attrs.max_ord = ep->ord; 2191 attrs.llp_stream_handle = ep; 2192 attrs.next_state = C4IW_QP_STATE_RTS; 2193 2194 /* bind QP and TID with INIT_WR */ 2195 mask = C4IW_QP_ATTR_NEXT_STATE | 2196 C4IW_QP_ATTR_LLP_STREAM_HANDLE | 2197 C4IW_QP_ATTR_MPA_ATTR | 2198 C4IW_QP_ATTR_MAX_IRD | 2199 C4IW_QP_ATTR_MAX_ORD; 2200 2201 err = c4iw_modify_qp(ep->com.qp->rhp, 2202 ep->com.qp, mask, &attrs, 1); 2203 if (err) 2204 goto err1; 2205 err = send_mpa_reply(ep, conn_param->private_data, 2206 conn_param->private_data_len); 2207 if (err) 2208 goto err1; 2209 2210 state_set(&ep->com, FPDU_MODE); 2211 established_upcall(ep); 2212 c4iw_put_ep(&ep->com); 2213 return 0; 2214 err1: 2215 ep->com.cm_id = NULL; 2216 ep->com.qp = NULL; 2217 cm_id->rem_ref(cm_id); 2218 err: 2219 c4iw_put_ep(&ep->com); 2220 return err; 2221 } 2222 2223 int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 2224 { 2225 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); 2226 struct c4iw_ep *ep; 2227 struct rtable *rt; 2228 int err = 0; 2229 2230 if ((conn_param->ord > c4iw_max_read_depth) || 2231 (conn_param->ird > c4iw_max_read_depth)) { 2232 err = -EINVAL; 2233 goto out; 2234 } 2235 ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 2236 if (!ep) { 2237 printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __func__); 2238 err = -ENOMEM; 2239 goto out; 2240 } 2241 init_timer(&ep->timer); 2242 ep->plen = conn_param->private_data_len; 2243 if (ep->plen) 2244 memcpy(ep->mpa_pkt + sizeof(struct mpa_message), 2245 conn_param->private_data, ep->plen); 2246 ep->ird = conn_param->ird; 2247 ep->ord = conn_param->ord; 2248 2249 if (peer2peer && ep->ord == 0) 2250 ep->ord = 1; 2251 2252 cm_id->add_ref(cm_id); 2253 ep->com.dev = dev; 2254 ep->com.cm_id = cm_id; 2255 ep->com.qp = get_qhp(dev, conn_param->qpn); 2256 BUG_ON(!ep->com.qp); 2257 PDBG("%s qpn 0x%x qp %p cm_id %p\n", __func__, conn_param->qpn, 2258 ep->com.qp, cm_id); 2259 2260 /* 2261 * Allocate an active TID to initiate a TCP connection. 2262 */ 2263 ep->atid = cxgb4_alloc_atid(dev->rdev.lldi.tids, ep); 2264 if (ep->atid == -1) { 2265 printk(KERN_ERR MOD "%s - cannot alloc atid.\n", __func__); 2266 err = -ENOMEM; 2267 goto fail2; 2268 } 2269 2270 PDBG("%s saddr 0x%x sport 0x%x raddr 0x%x rport 0x%x\n", __func__, 2271 ntohl(cm_id->local_addr.sin_addr.s_addr), 2272 ntohs(cm_id->local_addr.sin_port), 2273 ntohl(cm_id->remote_addr.sin_addr.s_addr), 2274 ntohs(cm_id->remote_addr.sin_port)); 2275 2276 /* find a route */ 2277 rt = find_route(dev, 2278 cm_id->local_addr.sin_addr.s_addr, 2279 cm_id->remote_addr.sin_addr.s_addr, 2280 cm_id->local_addr.sin_port, 2281 cm_id->remote_addr.sin_port, 0); 2282 if (!rt) { 2283 printk(KERN_ERR MOD "%s - cannot find route.\n", __func__); 2284 err = -EHOSTUNREACH; 2285 goto fail3; 2286 } 2287 ep->dst = &rt->dst; 2288 2289 err = import_ep(ep, cm_id->remote_addr.sin_addr.s_addr, 2290 ep->dst, ep->com.dev, true); 2291 if (err) { 2292 printk(KERN_ERR MOD "%s - cannot alloc l2e.\n", __func__); 2293 goto fail4; 2294 } 2295 2296 PDBG("%s txq_idx %u tx_chan %u smac_idx %u rss_qid %u l2t_idx %u\n", 2297 __func__, ep->txq_idx, ep->tx_chan, ep->smac_idx, ep->rss_qid, 2298 ep->l2t->idx); 2299 2300 state_set(&ep->com, CONNECTING); 2301 ep->tos = 0; 2302 ep->com.local_addr = cm_id->local_addr; 2303 ep->com.remote_addr = cm_id->remote_addr; 2304 2305 /* send connect request to rnic */ 2306 err = send_connect(ep); 2307 if (!err) 2308 goto out; 2309 2310 cxgb4_l2t_release(ep->l2t); 2311 fail4: 2312 dst_release(ep->dst); 2313 fail3: 2314 cxgb4_free_atid(ep->com.dev->rdev.lldi.tids, ep->atid); 2315 fail2: 2316 cm_id->rem_ref(cm_id); 2317 c4iw_put_ep(&ep->com); 2318 out: 2319 return err; 2320 } 2321 2322 int c4iw_create_listen(struct iw_cm_id *cm_id, int backlog) 2323 { 2324 int err = 0; 2325 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); 2326 struct c4iw_listen_ep *ep; 2327 2328 2329 might_sleep(); 2330 2331 ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 2332 if (!ep) { 2333 printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __func__); 2334 err = -ENOMEM; 2335 goto fail1; 2336 } 2337 PDBG("%s ep %p\n", __func__, ep); 2338 cm_id->add_ref(cm_id); 2339 ep->com.cm_id = cm_id; 2340 ep->com.dev = dev; 2341 ep->backlog = backlog; 2342 ep->com.local_addr = cm_id->local_addr; 2343 2344 /* 2345 * Allocate a server TID. 2346 */ 2347 ep->stid = cxgb4_alloc_stid(dev->rdev.lldi.tids, PF_INET, ep); 2348 if (ep->stid == -1) { 2349 printk(KERN_ERR MOD "%s - cannot alloc stid.\n", __func__); 2350 err = -ENOMEM; 2351 goto fail2; 2352 } 2353 2354 state_set(&ep->com, LISTEN); 2355 c4iw_init_wr_wait(&ep->com.wr_wait); 2356 err = cxgb4_create_server(ep->com.dev->rdev.lldi.ports[0], ep->stid, 2357 ep->com.local_addr.sin_addr.s_addr, 2358 ep->com.local_addr.sin_port, 2359 ep->com.dev->rdev.lldi.rxq_ids[0]); 2360 if (err) 2361 goto fail3; 2362 2363 /* wait for pass_open_rpl */ 2364 err = c4iw_wait_for_reply(&ep->com.dev->rdev, &ep->com.wr_wait, 0, 0, 2365 __func__); 2366 if (!err) { 2367 cm_id->provider_data = ep; 2368 goto out; 2369 } 2370 fail3: 2371 cxgb4_free_stid(ep->com.dev->rdev.lldi.tids, ep->stid, PF_INET); 2372 fail2: 2373 cm_id->rem_ref(cm_id); 2374 c4iw_put_ep(&ep->com); 2375 fail1: 2376 out: 2377 return err; 2378 } 2379 2380 int c4iw_destroy_listen(struct iw_cm_id *cm_id) 2381 { 2382 int err; 2383 struct c4iw_listen_ep *ep = to_listen_ep(cm_id); 2384 2385 PDBG("%s ep %p\n", __func__, ep); 2386 2387 might_sleep(); 2388 state_set(&ep->com, DEAD); 2389 c4iw_init_wr_wait(&ep->com.wr_wait); 2390 err = listen_stop(ep); 2391 if (err) 2392 goto done; 2393 err = c4iw_wait_for_reply(&ep->com.dev->rdev, &ep->com.wr_wait, 0, 0, 2394 __func__); 2395 cxgb4_free_stid(ep->com.dev->rdev.lldi.tids, ep->stid, PF_INET); 2396 done: 2397 cm_id->rem_ref(cm_id); 2398 c4iw_put_ep(&ep->com); 2399 return err; 2400 } 2401 2402 int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp) 2403 { 2404 int ret = 0; 2405 int close = 0; 2406 int fatal = 0; 2407 struct c4iw_rdev *rdev; 2408 2409 mutex_lock(&ep->com.mutex); 2410 2411 PDBG("%s ep %p state %s, abrupt %d\n", __func__, ep, 2412 states[ep->com.state], abrupt); 2413 2414 rdev = &ep->com.dev->rdev; 2415 if (c4iw_fatal_error(rdev)) { 2416 fatal = 1; 2417 close_complete_upcall(ep); 2418 ep->com.state = DEAD; 2419 } 2420 switch (ep->com.state) { 2421 case MPA_REQ_WAIT: 2422 case MPA_REQ_SENT: 2423 case MPA_REQ_RCVD: 2424 case MPA_REP_SENT: 2425 case FPDU_MODE: 2426 close = 1; 2427 if (abrupt) 2428 ep->com.state = ABORTING; 2429 else { 2430 ep->com.state = CLOSING; 2431 start_ep_timer(ep); 2432 } 2433 set_bit(CLOSE_SENT, &ep->com.flags); 2434 break; 2435 case CLOSING: 2436 if (!test_and_set_bit(CLOSE_SENT, &ep->com.flags)) { 2437 close = 1; 2438 if (abrupt) { 2439 stop_ep_timer(ep); 2440 ep->com.state = ABORTING; 2441 } else 2442 ep->com.state = MORIBUND; 2443 } 2444 break; 2445 case MORIBUND: 2446 case ABORTING: 2447 case DEAD: 2448 PDBG("%s ignoring disconnect ep %p state %u\n", 2449 __func__, ep, ep->com.state); 2450 break; 2451 default: 2452 BUG(); 2453 break; 2454 } 2455 2456 if (close) { 2457 if (abrupt) { 2458 close_complete_upcall(ep); 2459 ret = send_abort(ep, NULL, gfp); 2460 } else 2461 ret = send_halfclose(ep, gfp); 2462 if (ret) 2463 fatal = 1; 2464 } 2465 mutex_unlock(&ep->com.mutex); 2466 if (fatal) 2467 release_ep_resources(ep); 2468 return ret; 2469 } 2470 2471 static int async_event(struct c4iw_dev *dev, struct sk_buff *skb) 2472 { 2473 struct cpl_fw6_msg *rpl = cplhdr(skb); 2474 c4iw_ev_dispatch(dev, (struct t4_cqe *)&rpl->data[0]); 2475 return 0; 2476 } 2477 2478 /* 2479 * These are the real handlers that are called from a 2480 * work queue. 2481 */ 2482 static c4iw_handler_func work_handlers[NUM_CPL_CMDS] = { 2483 [CPL_ACT_ESTABLISH] = act_establish, 2484 [CPL_ACT_OPEN_RPL] = act_open_rpl, 2485 [CPL_RX_DATA] = rx_data, 2486 [CPL_ABORT_RPL_RSS] = abort_rpl, 2487 [CPL_ABORT_RPL] = abort_rpl, 2488 [CPL_PASS_OPEN_RPL] = pass_open_rpl, 2489 [CPL_CLOSE_LISTSRV_RPL] = close_listsrv_rpl, 2490 [CPL_PASS_ACCEPT_REQ] = pass_accept_req, 2491 [CPL_PASS_ESTABLISH] = pass_establish, 2492 [CPL_PEER_CLOSE] = peer_close, 2493 [CPL_ABORT_REQ_RSS] = peer_abort, 2494 [CPL_CLOSE_CON_RPL] = close_con_rpl, 2495 [CPL_RDMA_TERMINATE] = terminate, 2496 [CPL_FW4_ACK] = fw4_ack, 2497 [CPL_FW6_MSG] = async_event 2498 }; 2499 2500 static void process_timeout(struct c4iw_ep *ep) 2501 { 2502 struct c4iw_qp_attributes attrs; 2503 int abort = 1; 2504 2505 mutex_lock(&ep->com.mutex); 2506 PDBG("%s ep %p tid %u state %d\n", __func__, ep, ep->hwtid, 2507 ep->com.state); 2508 switch (ep->com.state) { 2509 case MPA_REQ_SENT: 2510 __state_set(&ep->com, ABORTING); 2511 connect_reply_upcall(ep, -ETIMEDOUT); 2512 break; 2513 case MPA_REQ_WAIT: 2514 __state_set(&ep->com, ABORTING); 2515 break; 2516 case CLOSING: 2517 case MORIBUND: 2518 if (ep->com.cm_id && ep->com.qp) { 2519 attrs.next_state = C4IW_QP_STATE_ERROR; 2520 c4iw_modify_qp(ep->com.qp->rhp, 2521 ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, 2522 &attrs, 1); 2523 } 2524 __state_set(&ep->com, ABORTING); 2525 break; 2526 default: 2527 printk(KERN_ERR "%s unexpected state ep %p tid %u state %u\n", 2528 __func__, ep, ep->hwtid, ep->com.state); 2529 WARN_ON(1); 2530 abort = 0; 2531 } 2532 mutex_unlock(&ep->com.mutex); 2533 if (abort) 2534 abort_connection(ep, NULL, GFP_KERNEL); 2535 c4iw_put_ep(&ep->com); 2536 } 2537 2538 static void process_timedout_eps(void) 2539 { 2540 struct c4iw_ep *ep; 2541 2542 spin_lock_irq(&timeout_lock); 2543 while (!list_empty(&timeout_list)) { 2544 struct list_head *tmp; 2545 2546 tmp = timeout_list.next; 2547 list_del(tmp); 2548 spin_unlock_irq(&timeout_lock); 2549 ep = list_entry(tmp, struct c4iw_ep, entry); 2550 process_timeout(ep); 2551 spin_lock_irq(&timeout_lock); 2552 } 2553 spin_unlock_irq(&timeout_lock); 2554 } 2555 2556 static void process_work(struct work_struct *work) 2557 { 2558 struct sk_buff *skb = NULL; 2559 struct c4iw_dev *dev; 2560 struct cpl_act_establish *rpl; 2561 unsigned int opcode; 2562 int ret; 2563 2564 while ((skb = skb_dequeue(&rxq))) { 2565 rpl = cplhdr(skb); 2566 dev = *((struct c4iw_dev **) (skb->cb + sizeof(void *))); 2567 opcode = rpl->ot.opcode; 2568 2569 BUG_ON(!work_handlers[opcode]); 2570 ret = work_handlers[opcode](dev, skb); 2571 if (!ret) 2572 kfree_skb(skb); 2573 } 2574 process_timedout_eps(); 2575 } 2576 2577 static DECLARE_WORK(skb_work, process_work); 2578 2579 static void ep_timeout(unsigned long arg) 2580 { 2581 struct c4iw_ep *ep = (struct c4iw_ep *)arg; 2582 2583 spin_lock(&timeout_lock); 2584 list_add_tail(&ep->entry, &timeout_list); 2585 spin_unlock(&timeout_lock); 2586 queue_work(workq, &skb_work); 2587 } 2588 2589 /* 2590 * All the CM events are handled on a work queue to have a safe context. 2591 */ 2592 static int sched(struct c4iw_dev *dev, struct sk_buff *skb) 2593 { 2594 2595 /* 2596 * Save dev in the skb->cb area. 2597 */ 2598 *((struct c4iw_dev **) (skb->cb + sizeof(void *))) = dev; 2599 2600 /* 2601 * Queue the skb and schedule the worker thread. 2602 */ 2603 skb_queue_tail(&rxq, skb); 2604 queue_work(workq, &skb_work); 2605 return 0; 2606 } 2607 2608 static int set_tcb_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 2609 { 2610 struct cpl_set_tcb_rpl *rpl = cplhdr(skb); 2611 2612 if (rpl->status != CPL_ERR_NONE) { 2613 printk(KERN_ERR MOD "Unexpected SET_TCB_RPL status %u " 2614 "for tid %u\n", rpl->status, GET_TID(rpl)); 2615 } 2616 kfree_skb(skb); 2617 return 0; 2618 } 2619 2620 static int fw6_msg(struct c4iw_dev *dev, struct sk_buff *skb) 2621 { 2622 struct cpl_fw6_msg *rpl = cplhdr(skb); 2623 struct c4iw_wr_wait *wr_waitp; 2624 int ret; 2625 2626 PDBG("%s type %u\n", __func__, rpl->type); 2627 2628 switch (rpl->type) { 2629 case 1: 2630 ret = (int)((be64_to_cpu(rpl->data[0]) >> 8) & 0xff); 2631 wr_waitp = (struct c4iw_wr_wait *)(__force unsigned long) rpl->data[1]; 2632 PDBG("%s wr_waitp %p ret %u\n", __func__, wr_waitp, ret); 2633 if (wr_waitp) 2634 c4iw_wake_up(wr_waitp, ret ? -ret : 0); 2635 kfree_skb(skb); 2636 break; 2637 case 2: 2638 sched(dev, skb); 2639 break; 2640 default: 2641 printk(KERN_ERR MOD "%s unexpected fw6 msg type %u\n", __func__, 2642 rpl->type); 2643 kfree_skb(skb); 2644 break; 2645 } 2646 return 0; 2647 } 2648 2649 static int peer_abort_intr(struct c4iw_dev *dev, struct sk_buff *skb) 2650 { 2651 struct cpl_abort_req_rss *req = cplhdr(skb); 2652 struct c4iw_ep *ep; 2653 struct tid_info *t = dev->rdev.lldi.tids; 2654 unsigned int tid = GET_TID(req); 2655 2656 ep = lookup_tid(t, tid); 2657 if (is_neg_adv_abort(req->status)) { 2658 PDBG("%s neg_adv_abort ep %p tid %u\n", __func__, ep, 2659 ep->hwtid); 2660 kfree_skb(skb); 2661 return 0; 2662 } 2663 PDBG("%s ep %p tid %u state %u\n", __func__, ep, ep->hwtid, 2664 ep->com.state); 2665 2666 /* 2667 * Wake up any threads in rdma_init() or rdma_fini(). 2668 * However, this is not needed if com state is just 2669 * MPA_REQ_SENT 2670 */ 2671 if (ep->com.state != MPA_REQ_SENT) 2672 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 2673 sched(dev, skb); 2674 return 0; 2675 } 2676 2677 /* 2678 * Most upcalls from the T4 Core go to sched() to 2679 * schedule the processing on a work queue. 2680 */ 2681 c4iw_handler_func c4iw_handlers[NUM_CPL_CMDS] = { 2682 [CPL_ACT_ESTABLISH] = sched, 2683 [CPL_ACT_OPEN_RPL] = sched, 2684 [CPL_RX_DATA] = sched, 2685 [CPL_ABORT_RPL_RSS] = sched, 2686 [CPL_ABORT_RPL] = sched, 2687 [CPL_PASS_OPEN_RPL] = sched, 2688 [CPL_CLOSE_LISTSRV_RPL] = sched, 2689 [CPL_PASS_ACCEPT_REQ] = sched, 2690 [CPL_PASS_ESTABLISH] = sched, 2691 [CPL_PEER_CLOSE] = sched, 2692 [CPL_CLOSE_CON_RPL] = sched, 2693 [CPL_ABORT_REQ_RSS] = peer_abort_intr, 2694 [CPL_RDMA_TERMINATE] = sched, 2695 [CPL_FW4_ACK] = sched, 2696 [CPL_SET_TCB_RPL] = set_tcb_rpl, 2697 [CPL_FW6_MSG] = fw6_msg 2698 }; 2699 2700 int __init c4iw_cm_init(void) 2701 { 2702 spin_lock_init(&timeout_lock); 2703 skb_queue_head_init(&rxq); 2704 2705 workq = create_singlethread_workqueue("iw_cxgb4"); 2706 if (!workq) 2707 return -ENOMEM; 2708 2709 return 0; 2710 } 2711 2712 void __exit c4iw_cm_term(void) 2713 { 2714 WARN_ON(!list_empty(&timeout_list)); 2715 flush_workqueue(workq); 2716 destroy_workqueue(workq); 2717 } 2718