1 /*- 2 * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 3 * 4 * Copyright (c) 2015 - 2026 Intel Corporation 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenFabrics.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #include "irdma_main.h" 36 37 static void irdma_cm_post_event(struct irdma_cm_event *event); 38 static void irdma_disconnect_worker(struct work_struct *work); 39 40 /** 41 * irdma_cm_node_cmp_state - Compare the state of a CM node 42 * @cm_node: Pointer to the CM node structure 43 * @state: The state to compare against 44 * 45 * This function checks if the current state of the given CM node matches 46 * the specified state. 47 * 48 * Return: true if the states match, false otherwise. 49 */ 50 static bool 51 irdma_cm_node_cmp_state(struct irdma_cm_node *cm_node, 52 enum irdma_cm_node_state state) 53 { 54 55 return cm_node->state == state; 56 } 57 58 /** 59 * irdma_cm_node_set_state - Set the state of a CM node 60 * @cm_node: Pointer to the CM node whose state is to be updated 61 * @state: The new state to set for the CM node 62 * 63 * This function updates the state of the specified CM node to the 64 * provided state and returns the previous state of the CM node. 65 * 66 * Return: The previous state of the CM node. 67 */ 68 static enum irdma_cm_node_state 69 irdma_cm_node_set_state(struct irdma_cm_node *cm_node, 70 enum irdma_cm_node_state state) 71 { 72 enum irdma_cm_node_state old_state; 73 74 old_state = cm_node->state; 75 cm_node->state = state; 76 return old_state; 77 } 78 79 /** 80 * irdma_rem_ref_sqbuf - put back puda buffer if refcount is 0 81 * @vsi: The VSI structure of the device 82 * @buf: puda buffer to free 83 */ 84 static int 85 irdma_rem_ref_sqbuf(struct irdma_sc_vsi *vsi, struct irdma_puda_buf *buf) 86 { 87 struct irdma_puda_rsrc *ilq = vsi->ilq; 88 struct irdma_cm_node *cm_node = buf->scratch; 89 struct irdma_cm_core *cm_core; 90 91 if (!atomic_dec_and_test(&buf->pb_refcount)) 92 return 0; 93 94 irdma_puda_ret_bufpool(ilq, buf); 95 96 if (cm_node) { 97 buf->scratch = NULL; 98 cm_core = cm_node->cm_core; 99 cm_core->cm_free_ah(cm_node); 100 } 101 102 return 1; 103 } 104 105 /** 106 * irdma_cm_ilq_cmpl_handler - callback function when ILQ completes a send 107 * @vsi: The VSI structure of the device 108 * @bufp: puda buffer structure from sent packet 109 */ 110 void 111 irdma_cm_ilq_cmpl_handler(struct irdma_sc_vsi *vsi, void *bufp) 112 { 113 struct irdma_puda_buf *buf = bufp; 114 115 irdma_rem_ref_sqbuf(vsi, buf); 116 } 117 118 /** 119 * irdma_cm_send_buf - Sends a buffer using the PUDA ILQ 120 * @ilq: Pointer to the PUDA (Protocol Unit Data Agent) resource structure 121 * @buf: Pointer to the PUDA buffer to be sent 122 * 123 * This function is responsible for transmitting a buffer through the 124 * specified PUDA resource. It is typically used in the context of 125 * managing RDMA connections and their associated data transfers. 126 * 127 * Return: 0 on success, or a negative error code on failure. 128 */ 129 static int 130 irdma_cm_send_buf( 131 struct irdma_puda_rsrc *ilq, 132 struct irdma_puda_buf *buf 133 ) 134 { 135 int ret; 136 137 if (!atomic_inc_not_zero(&buf->pb_refcount)) 138 pr_err("irdma: puda buffer refcnt increase from zero\n"); 139 140 ret = irdma_puda_send_buf(ilq, buf); 141 if (ret) 142 irdma_rem_ref_sqbuf(ilq->vsi, buf); 143 144 return ret; 145 } 146 147 /** 148 * irdma_record_ird_ord - Record IRD/ORD passed in 149 * @cm_node: connection's node 150 * @conn_ird: connection IRD 151 * @conn_ord: connection ORD 152 */ 153 static void 154 irdma_record_ird_ord(struct irdma_cm_node *cm_node, u32 conn_ird, 155 u32 conn_ord) 156 { 157 if (conn_ird > cm_node->dev->hw_attrs.max_hw_ird) 158 conn_ird = cm_node->dev->hw_attrs.max_hw_ird; 159 160 if (conn_ord > cm_node->dev->hw_attrs.max_hw_ord) 161 conn_ord = cm_node->dev->hw_attrs.max_hw_ord; 162 else if (!conn_ord && cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO) 163 conn_ord = 1; 164 cm_node->ird_size = conn_ird; 165 cm_node->ord_size = conn_ord; 166 } 167 168 /** 169 * irdma_copy_ip_ntohl - copy IP address from network to host 170 * @dst: IP address in host order 171 * @src: IP address in network order (big endian) 172 */ 173 void 174 irdma_copy_ip_ntohl(u32 *dst, __be32 *src) 175 { 176 *dst++ = ntohl(*src++); 177 *dst++ = ntohl(*src++); 178 *dst++ = ntohl(*src++); 179 *dst = ntohl(*src); 180 } 181 182 /** 183 * irdma_copy_ip_htonl - copy IP address from host to network order 184 * @dst: IP address in network order (big endian) 185 * @src: IP address in host order 186 */ 187 void 188 irdma_copy_ip_htonl(__be32 *dst, u32 *src) 189 { 190 *dst++ = htonl(*src++); 191 *dst++ = htonl(*src++); 192 *dst++ = htonl(*src++); 193 *dst = htonl(*src); 194 } 195 196 /** 197 * irdma_get_addr_info 198 * @cm_node: contains ip/tcp info 199 * @cm_info: to get a copy of the cm_node ip/tcp info 200 */ 201 static void 202 irdma_get_addr_info(struct irdma_cm_node *cm_node, 203 struct irdma_cm_info *cm_info) 204 { 205 memset(cm_info, 0, sizeof(*cm_info)); 206 cm_info->ipv4 = cm_node->ipv4; 207 cm_info->vlan_id = cm_node->vlan_id; 208 memcpy(cm_info->loc_addr, cm_node->loc_addr, sizeof(cm_info->loc_addr)); 209 memcpy(cm_info->rem_addr, cm_node->rem_addr, sizeof(cm_info->rem_addr)); 210 cm_info->loc_port = cm_node->loc_port; 211 cm_info->rem_port = cm_node->rem_port; 212 } 213 214 /** 215 * irdma_fill_sockaddr4 - fill in addr info for IPv4 connection 216 * @cm_node: connection's node 217 * @event: upper layer's cm event 218 */ 219 static inline void 220 irdma_fill_sockaddr4(struct irdma_cm_node *cm_node, 221 struct iw_cm_event *event) 222 { 223 struct sockaddr_in *laddr = (struct sockaddr_in *)&event->local_addr; 224 struct sockaddr_in *raddr = (struct sockaddr_in *)&event->remote_addr; 225 226 laddr->sin_family = AF_INET; 227 raddr->sin_family = AF_INET; 228 229 laddr->sin_port = htons(cm_node->loc_port); 230 raddr->sin_port = htons(cm_node->rem_port); 231 232 laddr->sin_addr.s_addr = htonl(cm_node->loc_addr[0]); 233 raddr->sin_addr.s_addr = htonl(cm_node->rem_addr[0]); 234 } 235 236 /** 237 * irdma_fill_sockaddr6 - fill in addr info for IPv6 connection 238 * @cm_node: connection's node 239 * @event: upper layer's cm event 240 */ 241 static inline void 242 irdma_fill_sockaddr6(struct irdma_cm_node *cm_node, 243 struct iw_cm_event *event) 244 { 245 struct sockaddr_in6 *laddr6 = (struct sockaddr_in6 *)&event->local_addr; 246 struct sockaddr_in6 *raddr6 = (struct sockaddr_in6 *)&event->remote_addr; 247 248 laddr6->sin6_family = AF_INET6; 249 raddr6->sin6_family = AF_INET6; 250 251 laddr6->sin6_port = htons(cm_node->loc_port); 252 raddr6->sin6_port = htons(cm_node->rem_port); 253 254 irdma_copy_ip_htonl(laddr6->sin6_addr.__u6_addr.__u6_addr32, 255 cm_node->loc_addr); 256 irdma_copy_ip_htonl(raddr6->sin6_addr.__u6_addr.__u6_addr32, 257 cm_node->rem_addr); 258 } 259 260 /** 261 * irdma_get_cmevent_info - for cm event upcall 262 * @cm_node: connection's node 263 * @cm_id: upper layers cm struct for the event 264 * @event: upper layer's cm event 265 */ 266 static inline void 267 irdma_get_cmevent_info(struct irdma_cm_node *cm_node, 268 struct iw_cm_id *cm_id, 269 struct iw_cm_event *event) 270 { 271 memcpy(&event->local_addr, &cm_id->m_local_addr, 272 sizeof(event->local_addr)); 273 memcpy(&event->remote_addr, &cm_id->m_remote_addr, 274 sizeof(event->remote_addr)); 275 if (cm_node) { 276 event->private_data = cm_node->pdata_buf; 277 event->private_data_len = (u8)cm_node->pdata.size; 278 event->ird = cm_node->ird_size; 279 event->ord = cm_node->ord_size; 280 } 281 } 282 283 /** 284 * irdma_send_cm_event - upcall cm's event handler 285 * @cm_node: connection's node 286 * @cm_id: upper layer's cm info struct 287 * @type: Event type to indicate 288 * @status: status for the event type 289 */ 290 static int 291 irdma_send_cm_event(struct irdma_cm_node *cm_node, 292 struct iw_cm_id *cm_id, 293 enum iw_cm_event_type type, int status) 294 { 295 struct iw_cm_event event = {0}; 296 297 event.event = type; 298 event.status = status; 299 300 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 301 "cm_node %p cm_id=%p state=%d accel=%d event_type=%d status=%d\n", 302 cm_node, cm_id, cm_node->accelerated, cm_node->state, type, 303 status); 304 305 switch (type) { 306 case IW_CM_EVENT_CONNECT_REQUEST: 307 if (cm_node->ipv4) 308 irdma_fill_sockaddr4(cm_node, &event); 309 else 310 irdma_fill_sockaddr6(cm_node, &event); 311 event.provider_data = cm_node; 312 event.private_data = cm_node->pdata_buf; 313 event.private_data_len = (u8)cm_node->pdata.size; 314 event.ird = cm_node->ird_size; 315 break; 316 case IW_CM_EVENT_CONNECT_REPLY: 317 irdma_get_cmevent_info(cm_node, cm_id, &event); 318 break; 319 case IW_CM_EVENT_ESTABLISHED: 320 event.ird = cm_node->ird_size; 321 event.ord = cm_node->ord_size; 322 break; 323 case IW_CM_EVENT_DISCONNECT: 324 case IW_CM_EVENT_CLOSE: 325 /* Wait if we are in RTS but havent issued the iwcm event upcall */ 326 if (!cm_node->accelerated) 327 wait_for_completion(&cm_node->establish_comp); 328 break; 329 default: 330 return -EINVAL; 331 } 332 333 return cm_id->event_handler(cm_id, &event); 334 } 335 336 /** 337 * irdma_timer_list_prep - add connection nodes to a list to perform timer tasks 338 * @cm_core: cm's core 339 * @timer_list: a timer list to which cm_node will be selected 340 */ 341 static void 342 irdma_timer_list_prep(struct irdma_cm_core *cm_core, 343 struct list_head *timer_list) 344 { 345 struct irdma_cm_node *cm_node; 346 int bkt; 347 348 HASH_FOR_EACH_RCU(cm_core->cm_hash_tbl, bkt, cm_node, list) { 349 if ((cm_node->close_entry || cm_node->send_entry) && 350 irdma_add_ref_cmnode(cm_node)) 351 list_add(&cm_node->timer_entry, timer_list); 352 } 353 } 354 355 /** 356 * irdma_create_event - create cm event 357 * @cm_node: connection's node 358 * @type: Event type to generate 359 */ 360 static struct irdma_cm_event * 361 irdma_create_event(struct irdma_cm_node *cm_node, 362 enum irdma_cm_event_type type) 363 { 364 struct irdma_cm_event *event; 365 366 if (!cm_node->cm_id) 367 return NULL; 368 369 event = kzalloc(sizeof(*event), GFP_ATOMIC); 370 371 if (!event) 372 return NULL; 373 374 event->type = type; 375 event->cm_node = cm_node; 376 memcpy(event->cm_info.rem_addr, cm_node->rem_addr, 377 sizeof(event->cm_info.rem_addr)); 378 memcpy(event->cm_info.loc_addr, cm_node->loc_addr, 379 sizeof(event->cm_info.loc_addr)); 380 event->cm_info.rem_port = cm_node->rem_port; 381 event->cm_info.loc_port = cm_node->loc_port; 382 event->cm_info.cm_id = cm_node->cm_id; 383 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 384 "node=%p event=%p type=%u dst=%x src=%x\n", cm_node, event, 385 type, event->cm_info.loc_addr[0], 386 event->cm_info.rem_addr[0]); 387 irdma_cm_post_event(event); 388 389 return event; 390 } 391 392 /** 393 * irdma_free_retrans_entry - free send entry 394 * @cm_node: connection's node 395 */ 396 static void 397 irdma_free_retrans_entry(struct irdma_cm_node *cm_node) 398 { 399 struct irdma_timer_entry *send_entry; 400 401 send_entry = cm_node->send_entry; 402 cm_node->send_entry = NULL; 403 if (!send_entry) 404 return; 405 406 irdma_rem_ref_sqbuf(&cm_node->iwdev->vsi, send_entry->sqbuf); 407 kfree(send_entry); 408 irdma_rem_ref_cmnode(cm_node); 409 } 410 411 /** 412 * irdma_cleanup_retrans_entry - free send entry with lock 413 * @cm_node: connection's node 414 */ 415 static void 416 irdma_cleanup_retrans_entry(struct irdma_cm_node *cm_node) 417 { 418 unsigned long flags; 419 420 spin_lock_irqsave(&cm_node->retrans_list_lock, flags); 421 irdma_free_retrans_entry(cm_node); 422 spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags); 423 } 424 425 /** 426 * irdma_form_ah_cm_frame - get a free packet and build frame with address handle 427 * @cm_node: connection's node ionfo to use in frame 428 * @options: pointer to options info 429 * @hdr: pointer mpa header 430 * @pdata: pointer to private data 431 * @flags: indicates FIN or ACK 432 */ 433 static struct irdma_puda_buf * 434 irdma_form_ah_cm_frame(struct irdma_cm_node *cm_node, 435 struct irdma_kmem_info *options, 436 struct irdma_kmem_info *hdr, 437 struct irdma_mpa_priv_info *pdata, 438 u8 flags) 439 { 440 struct irdma_puda_buf *sqbuf; 441 struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi; 442 u8 *buf; 443 struct tcphdr *tcph; 444 u16 pktsize; 445 u32 opts_len = 0; 446 u32 pd_len = 0; 447 u32 hdr_len = 0; 448 449 if (!cm_node->ah || !cm_node->ah->ah_info.ah_valid) { 450 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "AH invalid\n"); 451 return NULL; 452 } 453 454 sqbuf = irdma_puda_get_bufpool(vsi->ilq); 455 if (!sqbuf) { 456 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "SQ buf NULL\n"); 457 return NULL; 458 } 459 460 sqbuf->ah_id = cm_node->ah->ah_info.ah_idx; 461 sqbuf->ah = cm_node->ah; 462 buf = sqbuf->mem.va; 463 if (options) 464 opts_len = (u32)options->size; 465 466 if (hdr) 467 hdr_len = hdr->size; 468 469 if (pdata) 470 pd_len = pdata->size; 471 472 pktsize = sizeof(*tcph) + opts_len + hdr_len + pd_len; 473 474 memset(buf, 0, pktsize); 475 476 sqbuf->totallen = pktsize; 477 sqbuf->tcphlen = sizeof(*tcph) + opts_len; 478 sqbuf->scratch = cm_node; 479 480 tcph = (struct tcphdr *)buf; 481 buf += sizeof(*tcph); 482 483 tcph->th_sport = htons(cm_node->loc_port); 484 tcph->th_dport = htons(cm_node->rem_port); 485 tcph->th_seq = htonl(cm_node->tcp_cntxt.loc_seq_num); 486 487 if (flags & SET_ACK) { 488 cm_node->tcp_cntxt.loc_ack_num = cm_node->tcp_cntxt.rcv_nxt; 489 tcph->th_ack = htonl(cm_node->tcp_cntxt.loc_ack_num); 490 tcp_set_flags(tcph, tcp_get_flags(tcph) | TH_ACK); 491 } else { 492 tcph->th_ack = 0; 493 } 494 495 if (flags & SET_SYN) { 496 cm_node->tcp_cntxt.loc_seq_num++; 497 tcp_set_flags(tcph, tcp_get_flags(tcph) | TH_SYN); 498 } else { 499 cm_node->tcp_cntxt.loc_seq_num += hdr_len + pd_len; 500 } 501 502 if (flags & SET_FIN) { 503 cm_node->tcp_cntxt.loc_seq_num++; 504 tcp_set_flags(tcph, tcp_get_flags(tcph) | TH_FIN); 505 } 506 507 if (flags & SET_RST) 508 tcp_set_flags(tcph, tcp_get_flags(tcph) | TH_RST); 509 510 tcph->th_off = (u16)((sizeof(*tcph) + opts_len + 3) >> 2); 511 sqbuf->tcphlen = tcph->th_off << 2; 512 tcph->th_win = htons(cm_node->tcp_cntxt.rcv_wnd); 513 tcph->th_urp = 0; 514 515 if (opts_len) { 516 memcpy(buf, options->addr, opts_len); 517 buf += opts_len; 518 } 519 520 if (hdr_len) { 521 memcpy(buf, hdr->addr, hdr_len); 522 buf += hdr_len; 523 } 524 525 if (pdata && pdata->addr) 526 memcpy(buf, pdata->addr, pdata->size); 527 528 atomic_set(&sqbuf->pb_refcount, 1); 529 530 irdma_debug_buf(vsi->dev, IRDMA_DEBUG_ILQ, "TRANSMIT ILQ BUFFER", 531 sqbuf->mem.va, sqbuf->totallen); 532 533 return sqbuf; 534 } 535 536 /** 537 * irdma_form_uda_cm_frame - get a free packet and build frame full tcpip packet 538 * @cm_node: connection's node ionfo to use in frame 539 * @options: pointer to options info 540 * @hdr: pointer mpa header 541 * @pdata: pointer to private data 542 * @flags: indicates FIN or ACK 543 */ 544 static struct irdma_puda_buf * 545 irdma_form_uda_cm_frame(struct irdma_cm_node *cm_node, 546 struct irdma_kmem_info *options, 547 struct irdma_kmem_info *hdr, 548 struct irdma_mpa_priv_info *pdata, 549 u8 flags) 550 { 551 struct irdma_puda_buf *sqbuf; 552 struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi; 553 u8 *buf; 554 555 struct tcphdr *tcph; 556 struct ip *iph; 557 struct ip6_hdr *ip6h; 558 struct ether_header *ethh; 559 u16 pktsize; 560 u16 eth_hlen = ETH_HLEN; 561 u32 opts_len = 0; 562 u32 pd_len = 0; 563 u32 hdr_len = 0; 564 565 u16 vtag; 566 567 sqbuf = irdma_puda_get_bufpool(vsi->ilq); 568 if (!sqbuf) 569 return NULL; 570 571 buf = sqbuf->mem.va; 572 573 if (options) 574 opts_len = (u32)options->size; 575 576 if (hdr) 577 hdr_len = hdr->size; 578 579 if (pdata) 580 pd_len = pdata->size; 581 582 if (cm_node->vlan_id < VLAN_N_VID) 583 eth_hlen += 4; 584 585 if (cm_node->ipv4) 586 pktsize = sizeof(*iph) + sizeof(*tcph); 587 else 588 pktsize = sizeof(*ip6h) + sizeof(*tcph); 589 pktsize += opts_len + hdr_len + pd_len; 590 591 memset(buf, 0, eth_hlen + pktsize); 592 593 sqbuf->totallen = pktsize + eth_hlen; 594 sqbuf->maclen = eth_hlen; 595 sqbuf->tcphlen = sizeof(*tcph) + opts_len; 596 sqbuf->scratch = cm_node; 597 598 ethh = (struct ether_header *)buf; 599 buf += eth_hlen; 600 601 if (cm_node->do_lpb) 602 sqbuf->do_lpb = true; 603 604 if (cm_node->ipv4) { 605 sqbuf->ipv4 = true; 606 607 iph = (struct ip *)buf; 608 buf += sizeof(*iph); 609 tcph = (struct tcphdr *)buf; 610 buf += sizeof(*tcph); 611 612 ether_addr_copy(ethh->ether_dhost, cm_node->rem_mac); 613 ether_addr_copy(ethh->ether_shost, cm_node->loc_mac); 614 if (cm_node->vlan_id < VLAN_N_VID) { 615 ((struct ether_vlan_header *)ethh)->evl_proto = 616 htons(ETH_P_8021Q); 617 vtag = (cm_node->user_pri << VLAN_PRIO_SHIFT) | 618 cm_node->vlan_id; 619 ((struct ether_vlan_header *)ethh)->evl_tag = htons(vtag); 620 621 ((struct ether_vlan_header *)ethh)->evl_encap_proto = 622 htons(ETH_P_IP); 623 } else { 624 ethh->ether_type = htons(ETH_P_IP); 625 } 626 627 iph->ip_v = IPVERSION; 628 iph->ip_hl = 5; /* 5 * 4Byte words, IP headr len */ 629 iph->ip_tos = cm_node->tos; 630 iph->ip_len = htons(pktsize); 631 iph->ip_id = htons(++cm_node->tcp_cntxt.loc_id); 632 633 iph->ip_off = htons(0x4000); 634 iph->ip_ttl = 0x40; 635 iph->ip_p = IPPROTO_TCP; 636 iph->ip_src.s_addr = htonl(cm_node->loc_addr[0]); 637 iph->ip_dst.s_addr = htonl(cm_node->rem_addr[0]); 638 } else { 639 sqbuf->ipv4 = false; 640 ip6h = (struct ip6_hdr *)buf; 641 buf += sizeof(*ip6h); 642 tcph = (struct tcphdr *)buf; 643 buf += sizeof(*tcph); 644 645 ether_addr_copy(ethh->ether_dhost, cm_node->rem_mac); 646 ether_addr_copy(ethh->ether_shost, cm_node->loc_mac); 647 if (cm_node->vlan_id < VLAN_N_VID) { 648 ((struct ether_vlan_header *)ethh)->evl_proto = 649 htons(ETH_P_8021Q); 650 vtag = (cm_node->user_pri << VLAN_PRIO_SHIFT) | 651 cm_node->vlan_id; 652 ((struct ether_vlan_header *)ethh)->evl_tag = htons(vtag); 653 ((struct ether_vlan_header *)ethh)->evl_encap_proto = 654 htons(ETH_P_IPV6); 655 } else { 656 ethh->ether_type = htons(ETH_P_IPV6); 657 } 658 ip6h->ip6_vfc = 6 << 4; 659 ip6h->ip6_vfc |= cm_node->tos >> 4; 660 ip6h->ip6_flow = cm_node->tos << 20; 661 ip6h->ip6_plen = htons(pktsize - sizeof(*ip6h)); 662 ip6h->ip6_nxt = 6; 663 ip6h->ip6_hops = 128; 664 irdma_copy_ip_htonl(ip6h->ip6_src.__u6_addr.__u6_addr32, 665 cm_node->loc_addr); 666 irdma_copy_ip_htonl(ip6h->ip6_dst.__u6_addr.__u6_addr32, 667 cm_node->rem_addr); 668 } 669 670 tcph->th_sport = htons(cm_node->loc_port); 671 tcph->th_dport = htons(cm_node->rem_port); 672 tcph->th_seq = htonl(cm_node->tcp_cntxt.loc_seq_num); 673 674 if (flags & SET_ACK) { 675 cm_node->tcp_cntxt.loc_ack_num = cm_node->tcp_cntxt.rcv_nxt; 676 tcph->th_ack = htonl(cm_node->tcp_cntxt.loc_ack_num); 677 tcp_set_flags(tcph, tcp_get_flags(tcph) | TH_ACK); 678 } else { 679 tcph->th_ack = 0; 680 } 681 682 if (flags & SET_SYN) { 683 cm_node->tcp_cntxt.loc_seq_num++; 684 tcp_set_flags(tcph, tcp_get_flags(tcph) | TH_SYN); 685 } else { 686 cm_node->tcp_cntxt.loc_seq_num += hdr_len + pd_len; 687 } 688 689 if (flags & SET_FIN) { 690 cm_node->tcp_cntxt.loc_seq_num++; 691 tcp_set_flags(tcph, tcp_get_flags(tcph) | TH_FIN); 692 } 693 694 if (flags & SET_RST) 695 tcp_set_flags(tcph, tcp_get_flags(tcph) | TH_RST); 696 697 tcph->th_off = (u16)((sizeof(*tcph) + opts_len + 3) >> 2); 698 sqbuf->tcphlen = tcph->th_off << 2; 699 tcph->th_win = htons(cm_node->tcp_cntxt.rcv_wnd); 700 tcph->th_urp = 0; 701 702 if (opts_len) { 703 memcpy(buf, options->addr, opts_len); 704 buf += opts_len; 705 } 706 707 if (hdr_len) { 708 memcpy(buf, hdr->addr, hdr_len); 709 buf += hdr_len; 710 } 711 712 if (pdata && pdata->addr) 713 memcpy(buf, pdata->addr, pdata->size); 714 715 atomic_set(&sqbuf->pb_refcount, 1); 716 717 irdma_debug_buf(vsi->dev, IRDMA_DEBUG_ILQ, "TRANSMIT ILQ BUFFER", 718 sqbuf->mem.va, sqbuf->totallen); 719 720 return sqbuf; 721 } 722 723 /** 724 * irdma_send_reset - Send RST packet 725 * @cm_node: connection's node 726 */ 727 int 728 irdma_send_reset(struct irdma_cm_node *cm_node) 729 { 730 struct irdma_puda_buf *sqbuf; 731 int flags = SET_RST | SET_ACK; 732 733 sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, NULL, NULL, 734 flags); 735 if (!sqbuf) 736 return -ENOMEM; 737 738 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 739 "caller: %pS cm_node %p cm_id=%p accel=%d state=%d rem_port=0x%04x, loc_port=0x%04x rem_addr=%x loc_addr=%x\n", 740 __builtin_return_address(0), cm_node, cm_node->cm_id, 741 cm_node->accelerated, cm_node->state, cm_node->rem_port, 742 cm_node->loc_port, cm_node->rem_addr[0], 743 cm_node->loc_addr[0]); 744 745 return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 0, 746 1); 747 } 748 749 /** 750 * irdma_active_open_err - send event for active side cm error 751 * @cm_node: connection's node 752 * @reset: Flag to send reset or not 753 */ 754 static void 755 irdma_active_open_err(struct irdma_cm_node *cm_node, bool reset) 756 { 757 irdma_cleanup_retrans_entry(cm_node); 758 cm_node->cm_core->stats_connect_errs++; 759 if (reset) { 760 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 761 "cm_node=%p state=%d\n", cm_node, cm_node->state); 762 irdma_add_ref_cmnode(cm_node); 763 if (irdma_send_reset(cm_node)) 764 irdma_rem_ref_cmnode(cm_node); 765 } 766 767 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_CLOSED); 768 irdma_create_event(cm_node, IRDMA_CM_EVENT_ABORTED); 769 } 770 771 /** 772 * irdma_passive_open_err - handle passive side cm error 773 * @cm_node: connection's node 774 * @reset: send reset or just free cm_node 775 */ 776 static void 777 irdma_passive_open_err(struct irdma_cm_node *cm_node, bool reset) 778 { 779 irdma_cleanup_retrans_entry(cm_node); 780 cm_node->cm_core->stats_passive_errs++; 781 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_CLOSED); 782 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 783 "cm_node=%p state=%d\n", cm_node, cm_node->state); 784 if (reset) 785 irdma_send_reset(cm_node); 786 else 787 irdma_rem_ref_cmnode(cm_node); 788 } 789 790 /** 791 * irdma_event_connect_error - to create connect error event 792 * @event: cm information for connect event 793 */ 794 static void 795 irdma_event_connect_error(struct irdma_cm_event *event) 796 { 797 struct irdma_qp *iwqp; 798 struct iw_cm_id *cm_id; 799 800 cm_id = event->cm_node->cm_id; 801 if (!cm_id) 802 return; 803 804 iwqp = cm_id->provider_data; 805 806 if (!iwqp || !iwqp->iwdev) 807 return; 808 809 iwqp->cm_id = NULL; 810 cm_id->provider_data = NULL; 811 irdma_send_cm_event(event->cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY, 812 -ECONNRESET); 813 irdma_rem_ref_cmnode(event->cm_node); 814 } 815 816 /** 817 * irdma_process_options - process options from TCP header 818 * @cm_node: connection's node 819 * @optionsloc: point to start of options 820 * @optionsize: size of all options 821 * @syn_pkt: flag if syn packet 822 */ 823 static int 824 irdma_process_options(struct irdma_cm_node *cm_node, u8 *optionsloc, 825 u32 optionsize, u32 syn_pkt) 826 { 827 u32 tmp; 828 u32 offset = 0; 829 union all_known_options *all_options; 830 char got_mss_option = 0; 831 832 while (offset < optionsize) { 833 all_options = (union all_known_options *)(optionsloc + offset); 834 switch (all_options->base.optionnum) { 835 case OPTION_NUM_EOL: 836 offset = optionsize; 837 break; 838 case OPTION_NUM_NONE: 839 offset += 1; 840 continue; 841 case OPTION_NUM_MSS: 842 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 843 "MSS Length: %d Offset: %d Size: %d\n", 844 all_options->mss.len, offset, optionsize); 845 got_mss_option = 1; 846 if (all_options->mss.len != 4) 847 return -EINVAL; 848 tmp = ntohs(all_options->mss.mss); 849 if ((cm_node->ipv4 && 850 (tmp + IRDMA_MTU_TO_MSS_IPV4) < IRDMA_MIN_MTU_IPV4) || 851 (!cm_node->ipv4 && 852 (tmp + IRDMA_MTU_TO_MSS_IPV6) < IRDMA_MIN_MTU_IPV6)) 853 return -EINVAL; 854 if (tmp < cm_node->tcp_cntxt.mss) 855 cm_node->tcp_cntxt.mss = tmp; 856 break; 857 case OPTION_NUM_WINDOW_SCALE: 858 cm_node->tcp_cntxt.snd_wscale = 859 all_options->windowscale.shiftcount; 860 break; 861 default: 862 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 863 "Unsupported TCP Option: %x\n", 864 all_options->base.optionnum); 865 break; 866 } 867 offset += all_options->base.len; 868 } 869 if (!got_mss_option && syn_pkt) 870 cm_node->tcp_cntxt.mss = IRDMA_CM_DEFAULT_MSS; 871 872 return 0; 873 } 874 875 /** 876 * irdma_handle_tcp_options - setup TCP context info after parsing TCP options 877 * @cm_node: connection's node 878 * @tcph: pointer tcp header 879 * @optionsize: size of options rcvd 880 * @passive: active or passive flag 881 */ 882 static int 883 irdma_handle_tcp_options(struct irdma_cm_node *cm_node, 884 struct tcphdr *tcph, int optionsize, 885 int passive) 886 { 887 u8 *optionsloc = (u8 *)&tcph[1]; 888 int ret; 889 890 if (optionsize) { 891 ret = irdma_process_options(cm_node, optionsloc, optionsize, 892 (u32)tcp_get_flags(tcph) & TH_SYN); 893 if (ret) { 894 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 895 "Node %p, Sending Reset\n", cm_node); 896 if (passive) 897 irdma_passive_open_err(cm_node, true); 898 else 899 irdma_active_open_err(cm_node, true); 900 return ret; 901 } 902 } 903 904 cm_node->tcp_cntxt.snd_wnd = ntohs(tcph->th_win) 905 << cm_node->tcp_cntxt.snd_wscale; 906 907 if (cm_node->tcp_cntxt.snd_wnd > cm_node->tcp_cntxt.max_snd_wnd) 908 cm_node->tcp_cntxt.max_snd_wnd = cm_node->tcp_cntxt.snd_wnd; 909 910 return 0; 911 } 912 913 /** 914 * irdma_build_mpa_v1 - build a MPA V1 frame 915 * @cm_node: connection's node 916 * @start_addr: address where to build frame 917 * @mpa_key: to do read0 or write0 918 */ 919 static void 920 irdma_build_mpa_v1(struct irdma_cm_node *cm_node, void *start_addr, 921 u8 mpa_key) 922 { 923 struct ietf_mpa_v1 *mpa_frame = start_addr; 924 925 switch (mpa_key) { 926 case MPA_KEY_REQUEST: 927 memcpy(mpa_frame->key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE); 928 break; 929 case MPA_KEY_REPLY: 930 memcpy(mpa_frame->key, IEFT_MPA_KEY_REP, IETF_MPA_KEY_SIZE); 931 break; 932 default: 933 break; 934 } 935 mpa_frame->flags = IETF_MPA_FLAGS_CRC; 936 mpa_frame->rev = cm_node->mpa_frame_rev; 937 mpa_frame->priv_data_len = htons(cm_node->pdata.size); 938 } 939 940 /** 941 * irdma_build_mpa_v2 - build a MPA V2 frame 942 * @cm_node: connection's node 943 * @start_addr: buffer start address 944 * @mpa_key: to do read0 or write0 945 */ 946 static void 947 irdma_build_mpa_v2(struct irdma_cm_node *cm_node, void *start_addr, 948 u8 mpa_key) 949 { 950 struct ietf_mpa_v2 *mpa_frame = start_addr; 951 struct ietf_rtr_msg *rtr_msg = &mpa_frame->rtr_msg; 952 u16 ctrl_ird, ctrl_ord; 953 954 /* initialize the upper 5 bytes of the frame */ 955 irdma_build_mpa_v1(cm_node, start_addr, mpa_key); 956 mpa_frame->flags |= IETF_MPA_V2_FLAG; 957 if (cm_node->iwdev->iw_ooo) { 958 mpa_frame->flags |= IETF_MPA_FLAGS_MARKERS; 959 cm_node->rcv_mark_en = true; 960 } 961 mpa_frame->priv_data_len = cpu_to_be16(be16_to_cpu(mpa_frame->priv_data_len) + 962 IETF_RTR_MSG_SIZE); 963 964 /* initialize RTR msg */ 965 if (cm_node->mpav2_ird_ord == IETF_NO_IRD_ORD) { 966 ctrl_ird = IETF_NO_IRD_ORD; 967 ctrl_ord = IETF_NO_IRD_ORD; 968 } else { 969 ctrl_ird = (cm_node->ird_size > IETF_NO_IRD_ORD) ? 970 IETF_NO_IRD_ORD : 971 cm_node->ird_size; 972 ctrl_ord = (cm_node->ord_size > IETF_NO_IRD_ORD) ? 973 IETF_NO_IRD_ORD : 974 cm_node->ord_size; 975 } 976 ctrl_ird |= IETF_PEER_TO_PEER; 977 978 switch (mpa_key) { 979 case MPA_KEY_REQUEST: 980 ctrl_ord |= IETF_RDMA0_WRITE; 981 ctrl_ord |= IETF_RDMA0_READ; 982 break; 983 case MPA_KEY_REPLY: 984 switch (cm_node->send_rdma0_op) { 985 case SEND_RDMA_WRITE_ZERO: 986 ctrl_ord |= IETF_RDMA0_WRITE; 987 break; 988 case SEND_RDMA_READ_ZERO: 989 ctrl_ord |= IETF_RDMA0_READ; 990 break; 991 } 992 break; 993 default: 994 break; 995 } 996 rtr_msg->ctrl_ird = htons(ctrl_ird); 997 rtr_msg->ctrl_ord = htons(ctrl_ord); 998 } 999 1000 /** 1001 * irdma_cm_build_mpa_frame - build mpa frame for mpa version 1 or version 2 1002 * @cm_node: connection's node 1003 * @mpa: mpa: data buffer 1004 * @mpa_key: to do read0 or write0 1005 */ 1006 static int 1007 irdma_cm_build_mpa_frame(struct irdma_cm_node *cm_node, 1008 struct irdma_kmem_info *mpa, u8 mpa_key) 1009 { 1010 int hdr_len = 0; 1011 1012 switch (cm_node->mpa_frame_rev) { 1013 case IETF_MPA_V1: 1014 hdr_len = sizeof(struct ietf_mpa_v1); 1015 irdma_build_mpa_v1(cm_node, mpa->addr, mpa_key); 1016 break; 1017 case IETF_MPA_V2: 1018 hdr_len = sizeof(struct ietf_mpa_v2); 1019 irdma_build_mpa_v2(cm_node, mpa->addr, mpa_key); 1020 break; 1021 default: 1022 break; 1023 } 1024 1025 return hdr_len; 1026 } 1027 1028 /** 1029 * irdma_send_mpa_request - active node send mpa request to passive node 1030 * @cm_node: connection's node 1031 */ 1032 static int 1033 irdma_send_mpa_request(struct irdma_cm_node *cm_node) 1034 { 1035 struct irdma_puda_buf *sqbuf; 1036 1037 cm_node->mpa_hdr.addr = &cm_node->mpa_v2_frame; 1038 cm_node->mpa_hdr.size = irdma_cm_build_mpa_frame(cm_node, 1039 &cm_node->mpa_hdr, 1040 MPA_KEY_REQUEST); 1041 if (!cm_node->mpa_hdr.size) { 1042 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 1043 "mpa size = %d\n", cm_node->mpa_hdr.size); 1044 return -EINVAL; 1045 } 1046 1047 sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, 1048 &cm_node->mpa_hdr, 1049 &cm_node->pdata, SET_ACK); 1050 if (!sqbuf) 1051 return -ENOMEM; 1052 1053 return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1, 1054 0); 1055 } 1056 1057 /** 1058 * irdma_send_mpa_reject - 1059 * @cm_node: connection's node 1060 * @pdata: reject data for connection 1061 * @plen: length of reject data 1062 */ 1063 static int 1064 irdma_send_mpa_reject(struct irdma_cm_node *cm_node, 1065 const void *pdata, u8 plen) 1066 { 1067 struct irdma_puda_buf *sqbuf; 1068 struct irdma_mpa_priv_info priv_info; 1069 1070 cm_node->mpa_hdr.addr = &cm_node->mpa_v2_frame; 1071 cm_node->mpa_hdr.size = irdma_cm_build_mpa_frame(cm_node, 1072 &cm_node->mpa_hdr, 1073 MPA_KEY_REPLY); 1074 1075 cm_node->mpa_v2_frame.flags |= IETF_MPA_FLAGS_REJECT; 1076 priv_info.addr = pdata; 1077 priv_info.size = plen; 1078 1079 sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, 1080 &cm_node->mpa_hdr, &priv_info, 1081 SET_ACK | SET_FIN); 1082 if (!sqbuf) 1083 return -ENOMEM; 1084 1085 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_FIN_WAIT1); 1086 1087 return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1, 1088 0); 1089 } 1090 1091 /** 1092 * irdma_negotiate_mpa_v2_ird_ord - negotiate MPAv2 IRD/ORD 1093 * @cm_node: connection's node 1094 * @buf: Data pointer 1095 */ 1096 static int 1097 irdma_negotiate_mpa_v2_ird_ord(struct irdma_cm_node *cm_node, 1098 u8 *buf) 1099 { 1100 struct ietf_mpa_v2 *mpa_v2_frame; 1101 struct ietf_rtr_msg *rtr_msg; 1102 u16 ird_size; 1103 u16 ord_size; 1104 u16 ctrl_ord; 1105 u16 ctrl_ird; 1106 1107 mpa_v2_frame = (struct ietf_mpa_v2 *)buf; 1108 rtr_msg = &mpa_v2_frame->rtr_msg; 1109 1110 /* parse rtr message */ 1111 ctrl_ord = ntohs(rtr_msg->ctrl_ord); 1112 ctrl_ird = ntohs(rtr_msg->ctrl_ird); 1113 ird_size = ctrl_ird & IETF_NO_IRD_ORD; 1114 ord_size = ctrl_ord & IETF_NO_IRD_ORD; 1115 1116 if (!(ctrl_ird & IETF_PEER_TO_PEER)) 1117 return -EOPNOTSUPP; 1118 1119 if (ird_size == IETF_NO_IRD_ORD || ord_size == IETF_NO_IRD_ORD) { 1120 cm_node->mpav2_ird_ord = IETF_NO_IRD_ORD; 1121 goto negotiate_done; 1122 } 1123 1124 if (!irdma_cm_node_cmp_state(cm_node, IRDMA_CM_STATE_MPAREQ_SENT)) { 1125 /* responder */ 1126 if (!ord_size && (ctrl_ord & IETF_RDMA0_READ)) 1127 cm_node->ird_size = 1; 1128 if (cm_node->ord_size > ird_size) 1129 cm_node->ord_size = ird_size; 1130 } else { 1131 /* initiator */ 1132 if (!ird_size && (ctrl_ord & IETF_RDMA0_READ)) 1133 /* Remote peer doesn't support RDMA0_READ */ 1134 return -EOPNOTSUPP; 1135 1136 if (cm_node->ord_size > ird_size) 1137 cm_node->ord_size = ird_size; 1138 1139 if (cm_node->ird_size < ord_size) 1140 /* no resources available */ 1141 return -EINVAL; 1142 } 1143 1144 negotiate_done: 1145 if (ctrl_ord & IETF_RDMA0_READ) 1146 cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO; 1147 else if (ctrl_ord & IETF_RDMA0_WRITE) 1148 cm_node->send_rdma0_op = SEND_RDMA_WRITE_ZERO; 1149 else 1150 /* Not supported RDMA0 operation */ 1151 return -EOPNOTSUPP; 1152 1153 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 1154 "MPAV2 Negotiated ORD: %d, IRD: %d\n", cm_node->ord_size, 1155 cm_node->ird_size); 1156 return 0; 1157 } 1158 1159 /** 1160 * irdma_parse_mpa - process an IETF MPA frame 1161 * @cm_node: connection's node 1162 * @buf: Data pointer 1163 * @type: to return accept or reject 1164 * @len: Len of mpa buffer 1165 */ 1166 static int 1167 irdma_parse_mpa(struct irdma_cm_node *cm_node, u8 *buf, u32 *type, 1168 u32 len) 1169 { 1170 struct ietf_mpa_v1 *mpa_frame; 1171 int mpa_hdr_len, priv_data_len, ret; 1172 1173 *type = IRDMA_MPA_REQUEST_ACCEPT; 1174 1175 if (len < sizeof(*mpa_frame)) { 1176 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 1177 "ietf buffer small (%x)\n", len); 1178 return -EINVAL; 1179 } 1180 1181 mpa_frame = (struct ietf_mpa_v1 *)buf; 1182 mpa_hdr_len = sizeof(*mpa_frame); 1183 priv_data_len = ntohs(mpa_frame->priv_data_len); 1184 1185 if (priv_data_len > IETF_MAX_PRIV_DATA_LEN) { 1186 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 1187 "private_data too big %d\n", priv_data_len); 1188 return -EOVERFLOW; 1189 } 1190 1191 if (mpa_frame->rev != IETF_MPA_V1 && mpa_frame->rev != IETF_MPA_V2) { 1192 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 1193 "unsupported mpa rev = %d\n", mpa_frame->rev); 1194 return -EINVAL; 1195 } 1196 1197 if (mpa_frame->rev > cm_node->mpa_frame_rev) { 1198 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 1199 "rev %d\n", mpa_frame->rev); 1200 return -EINVAL; 1201 } 1202 1203 cm_node->mpa_frame_rev = mpa_frame->rev; 1204 if (!irdma_cm_node_cmp_state(cm_node, IRDMA_CM_STATE_MPAREQ_SENT)) { 1205 if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REQ, 1206 IETF_MPA_KEY_SIZE)) { 1207 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 1208 "Unexpected MPA Key received\n"); 1209 return -EINVAL; 1210 } 1211 } else { 1212 if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REP, 1213 IETF_MPA_KEY_SIZE)) { 1214 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 1215 "Unexpected MPA Key received\n"); 1216 return -EINVAL; 1217 } 1218 } 1219 1220 if (priv_data_len + mpa_hdr_len > len) { 1221 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 1222 "ietf buffer len(%x + %x != %x)\n", priv_data_len, 1223 mpa_hdr_len, len); 1224 return -EOVERFLOW; 1225 } 1226 1227 if (len > IRDMA_MAX_CM_BUF) { 1228 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 1229 "ietf buffer large len = %d\n", len); 1230 return -EOVERFLOW; 1231 } 1232 1233 switch (mpa_frame->rev) { 1234 case IETF_MPA_V2: 1235 mpa_hdr_len += IETF_RTR_MSG_SIZE; 1236 ret = irdma_negotiate_mpa_v2_ird_ord(cm_node, buf); 1237 if (ret) 1238 return ret; 1239 break; 1240 case IETF_MPA_V1: 1241 default: 1242 break; 1243 } 1244 1245 memcpy(cm_node->pdata_buf, buf + mpa_hdr_len, priv_data_len); 1246 cm_node->pdata.size = priv_data_len; 1247 1248 if (mpa_frame->flags & IETF_MPA_FLAGS_REJECT) 1249 *type = IRDMA_MPA_REQUEST_REJECT; 1250 1251 if (mpa_frame->flags & IETF_MPA_FLAGS_MARKERS) 1252 cm_node->snd_mark_en = true; 1253 1254 return 0; 1255 } 1256 1257 /** 1258 * irdma_schedule_cm_timer 1259 * @cm_node: connection's node 1260 * @sqbuf: buffer to send 1261 * @type: if it is send or close 1262 * @send_retrans: if rexmits to be done 1263 * @close_when_complete: is cm_node to be removed 1264 * 1265 * note - cm_node needs to be protected before calling this. Encase in: 1266 * irdma_rem_ref_cmnode(cm_core, cm_node); 1267 * irdma_schedule_cm_timer(...) 1268 * atomic_inc(&cm_node->refcnt); 1269 */ 1270 int 1271 irdma_schedule_cm_timer(struct irdma_cm_node *cm_node, 1272 struct irdma_puda_buf *sqbuf, 1273 enum irdma_timer_type type, int send_retrans, 1274 int close_when_complete) 1275 { 1276 struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi; 1277 struct irdma_cm_core *cm_core = cm_node->cm_core; 1278 struct irdma_timer_entry *new_send; 1279 u32 was_timer_set; 1280 unsigned long flags; 1281 1282 new_send = kzalloc(sizeof(*new_send), GFP_ATOMIC); 1283 if (!new_send) { 1284 if (type != IRDMA_TIMER_TYPE_CLOSE) 1285 irdma_rem_ref_sqbuf(vsi, sqbuf); 1286 return -ENOMEM; 1287 } 1288 1289 new_send->retrycount = IRDMA_DEFAULT_RETRYS; 1290 new_send->retranscount = IRDMA_DEFAULT_RETRANS; 1291 new_send->sqbuf = sqbuf; 1292 new_send->timetosend = jiffies; 1293 new_send->type = type; 1294 new_send->send_retrans = send_retrans; 1295 new_send->close_when_complete = close_when_complete; 1296 1297 if (type == IRDMA_TIMER_TYPE_CLOSE) { 1298 new_send->timetosend += (HZ / 10); 1299 if (cm_node->close_entry) { 1300 irdma_rem_ref_sqbuf(vsi, sqbuf); 1301 kfree(new_send); 1302 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 1303 "already close entry\n"); 1304 return -EINVAL; 1305 } 1306 1307 cm_node->close_entry = new_send; 1308 } else { /* type == IRDMA_TIMER_TYPE_SEND */ 1309 spin_lock_irqsave(&cm_node->retrans_list_lock, flags); 1310 if (cm_node->send_entry) { 1311 spin_unlock_irqrestore(&cm_node->retrans_list_lock, 1312 flags); 1313 irdma_rem_ref_sqbuf(vsi, sqbuf); 1314 kfree(new_send); 1315 1316 return -EINVAL; 1317 } 1318 cm_node->send_entry = new_send; 1319 irdma_add_ref_cmnode(cm_node); 1320 spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags); 1321 new_send->timetosend = jiffies + IRDMA_RETRY_TIMEOUT; 1322 1323 if (sqbuf->ah) 1324 atomic_inc(&sqbuf->ah->ah_info.ah_refcnt); 1325 1326 if (irdma_cm_send_buf(vsi->ilq, new_send->sqbuf)) 1327 cm_core->cm_free_ah(cm_node); 1328 1329 if (!send_retrans) { 1330 irdma_cleanup_retrans_entry(cm_node); 1331 if (close_when_complete) 1332 irdma_rem_ref_cmnode(cm_node); 1333 return 0; 1334 } 1335 } 1336 1337 spin_lock_irqsave(&cm_core->ht_lock, flags); 1338 was_timer_set = timer_pending(&cm_core->tcp_timer); 1339 1340 if (!was_timer_set) { 1341 cm_core->tcp_timer.expires = new_send->timetosend; 1342 add_timer(&cm_core->tcp_timer); 1343 } 1344 spin_unlock_irqrestore(&cm_core->ht_lock, flags); 1345 1346 return 0; 1347 } 1348 1349 /** 1350 * irdma_retrans_expired - Could not rexmit the packet 1351 * @cm_node: connection's node 1352 */ 1353 static void 1354 irdma_retrans_expired(struct irdma_cm_node *cm_node) 1355 { 1356 enum irdma_cm_node_state state; 1357 1358 state = irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_CLOSED); 1359 switch (state) { 1360 case IRDMA_CM_STATE_SYN_RCVD: 1361 case IRDMA_CM_STATE_CLOSING: 1362 irdma_rem_ref_cmnode(cm_node); 1363 break; 1364 case IRDMA_CM_STATE_FIN_WAIT1: 1365 case IRDMA_CM_STATE_LAST_ACK: 1366 irdma_send_reset(cm_node); 1367 break; 1368 default: 1369 irdma_add_ref_cmnode(cm_node); 1370 if (irdma_send_reset(cm_node)) 1371 irdma_rem_ref_cmnode(cm_node); 1372 irdma_create_event(cm_node, IRDMA_CM_EVENT_ABORTED); 1373 break; 1374 } 1375 } 1376 1377 /** 1378 * irdma_handle_close_entry - for handling retry/timeouts 1379 * @cm_node: connection's node 1380 * @rem_node: flag for remove cm_node 1381 */ 1382 static void 1383 irdma_handle_close_entry(struct irdma_cm_node *cm_node, 1384 u32 rem_node) 1385 { 1386 struct irdma_timer_entry *close_entry = cm_node->close_entry; 1387 struct irdma_qp *iwqp; 1388 unsigned long flags; 1389 1390 if (!close_entry) 1391 return; 1392 iwqp = (struct irdma_qp *)close_entry->sqbuf; 1393 if (iwqp) { 1394 spin_lock_irqsave(&iwqp->lock, flags); 1395 if (iwqp->cm_id) { 1396 iwqp->hw_tcp_state = IRDMA_TCP_STATE_CLOSED; 1397 iwqp->hw_iwarp_state = IRDMA_QP_STATE_ERROR; 1398 iwqp->last_aeq = IRDMA_AE_RESET_SENT; 1399 iwqp->ibqp_state = IB_QPS_ERR; 1400 spin_unlock_irqrestore(&iwqp->lock, flags); 1401 irdma_cm_disconn(iwqp); 1402 } else { 1403 spin_unlock_irqrestore(&iwqp->lock, flags); 1404 } 1405 } else if (rem_node) { 1406 /* TIME_WAIT state */ 1407 irdma_rem_ref_cmnode(cm_node); 1408 } 1409 1410 kfree(close_entry); 1411 cm_node->close_entry = NULL; 1412 } 1413 1414 /** 1415 * irdma_cm_timer_tick - system's timer expired callback 1416 * @t: Pointer to timer_list 1417 */ 1418 static void 1419 irdma_cm_timer_tick(struct timer_list *t) 1420 { 1421 unsigned long nexttimeout = jiffies + IRDMA_LONG_TIME; 1422 struct irdma_cm_node *cm_node; 1423 struct irdma_timer_entry *send_entry, *close_entry; 1424 struct list_head *list_core_temp; 1425 struct list_head *list_node; 1426 struct irdma_cm_core *cm_core = timer_container_of(cm_core, t, tcp_timer); 1427 struct irdma_sc_vsi *vsi; 1428 u32 settimer = 0; 1429 unsigned long timetosend; 1430 unsigned long flags; 1431 struct list_head timer_list; 1432 1433 INIT_LIST_HEAD(&timer_list); 1434 1435 rcu_read_lock(); 1436 irdma_timer_list_prep(cm_core, &timer_list); 1437 rcu_read_unlock(); 1438 1439 list_for_each_safe(list_node, list_core_temp, &timer_list) { 1440 cm_node = container_of(list_node, struct irdma_cm_node, 1441 timer_entry); 1442 close_entry = cm_node->close_entry; 1443 1444 if (close_entry) { 1445 if (time_after(close_entry->timetosend, jiffies)) { 1446 if (nexttimeout > close_entry->timetosend || 1447 !settimer) { 1448 nexttimeout = close_entry->timetosend; 1449 settimer = 1; 1450 } 1451 } else { 1452 irdma_handle_close_entry(cm_node, 1); 1453 } 1454 } 1455 1456 spin_lock_irqsave(&cm_node->retrans_list_lock, flags); 1457 1458 send_entry = cm_node->send_entry; 1459 if (!send_entry) 1460 goto done; 1461 if (time_after(send_entry->timetosend, jiffies)) { 1462 if (!irdma_cm_node_cmp_state(cm_node, IRDMA_CM_STATE_OFFLOADED)) { 1463 if (nexttimeout > send_entry->timetosend || 1464 !settimer) { 1465 nexttimeout = send_entry->timetosend; 1466 settimer = 1; 1467 } 1468 } else { 1469 irdma_free_retrans_entry(cm_node); 1470 } 1471 goto done; 1472 } 1473 1474 if (irdma_cm_node_cmp_state(cm_node, IRDMA_CM_STATE_OFFLOADED) || 1475 irdma_cm_node_cmp_state(cm_node, IRDMA_CM_STATE_CLOSED)) { 1476 irdma_free_retrans_entry(cm_node); 1477 goto done; 1478 } 1479 1480 if (!send_entry->retranscount || !send_entry->retrycount) { 1481 irdma_free_retrans_entry(cm_node); 1482 1483 spin_unlock_irqrestore(&cm_node->retrans_list_lock, 1484 flags); 1485 irdma_retrans_expired(cm_node); 1486 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_CLOSED); 1487 spin_lock_irqsave(&cm_node->retrans_list_lock, flags); 1488 goto done; 1489 } 1490 spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags); 1491 1492 vsi = &cm_node->iwdev->vsi; 1493 if (!cm_node->ack_rcvd) { 1494 if (send_entry->sqbuf->ah) 1495 atomic_inc(&send_entry->sqbuf->ah->ah_info.ah_refcnt); 1496 if (irdma_cm_send_buf(vsi->ilq, send_entry->sqbuf)) 1497 cm_core->cm_free_ah(cm_node); 1498 1499 cm_node->cm_core->stats_pkt_retrans++; 1500 } 1501 1502 spin_lock_irqsave(&cm_node->retrans_list_lock, flags); 1503 if (send_entry->send_retrans) { 1504 send_entry->retranscount--; 1505 timetosend = IRDMA_RETRY_TIMEOUT << 1506 min(IRDMA_DEFAULT_RETRANS - 1507 send_entry->retranscount, (u32)4); 1508 send_entry->timetosend = jiffies + 1509 min(timetosend, IRDMA_MAX_TIMEOUT); 1510 if (nexttimeout > send_entry->timetosend || !settimer) { 1511 nexttimeout = send_entry->timetosend; 1512 settimer = 1; 1513 } 1514 } else { 1515 int close_when_complete; 1516 1517 close_when_complete = send_entry->close_when_complete; 1518 irdma_free_retrans_entry(cm_node); 1519 if (close_when_complete) 1520 irdma_rem_ref_cmnode(cm_node); 1521 } 1522 done: 1523 spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags); 1524 irdma_rem_ref_cmnode(cm_node); 1525 } 1526 1527 if (settimer) { 1528 spin_lock_irqsave(&cm_core->ht_lock, flags); 1529 if (!timer_pending(&cm_core->tcp_timer)) { 1530 cm_core->tcp_timer.expires = nexttimeout; 1531 add_timer(&cm_core->tcp_timer); 1532 } 1533 spin_unlock_irqrestore(&cm_core->ht_lock, flags); 1534 } 1535 } 1536 1537 /** 1538 * irdma_send_syn - send SYN packet 1539 * @cm_node: connection's node 1540 * @sendack: flag to set ACK bit or not 1541 */ 1542 int 1543 irdma_send_syn(struct irdma_cm_node *cm_node, u32 sendack) 1544 { 1545 struct irdma_puda_buf *sqbuf; 1546 int flags = SET_SYN; 1547 char optionsbuf[sizeof(struct option_mss) + 1548 sizeof(struct option_windowscale) + 1549 sizeof(struct option_base) + TCP_OPTIONS_PADDING]; 1550 struct irdma_kmem_info opts; 1551 int optionssize = 0; 1552 1553 /* Sending MSS option */ 1554 union all_known_options *options; 1555 1556 opts.addr = optionsbuf; 1557 if (!cm_node) 1558 return -EINVAL; 1559 1560 options = (union all_known_options *)&optionsbuf[optionssize]; 1561 options->mss.optionnum = OPTION_NUM_MSS; 1562 options->mss.len = sizeof(struct option_mss); 1563 options->mss.mss = htons(cm_node->tcp_cntxt.mss); 1564 optionssize += sizeof(struct option_mss); 1565 1566 options = (union all_known_options *)&optionsbuf[optionssize]; 1567 options->windowscale.optionnum = OPTION_NUM_WINDOW_SCALE; 1568 options->windowscale.len = sizeof(struct option_windowscale); 1569 options->windowscale.shiftcount = cm_node->tcp_cntxt.rcv_wscale; 1570 optionssize += sizeof(struct option_windowscale); 1571 options = (union all_known_options *)&optionsbuf[optionssize]; 1572 options->eol = OPTION_NUM_EOL; 1573 optionssize += 1; 1574 1575 if (sendack) 1576 flags |= SET_ACK; 1577 1578 opts.size = optionssize; 1579 1580 sqbuf = cm_node->cm_core->form_cm_frame(cm_node, &opts, NULL, NULL, 1581 flags); 1582 if (!sqbuf) 1583 return -ENOMEM; 1584 1585 return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1, 1586 0); 1587 } 1588 1589 /** 1590 * irdma_send_ack - Send ACK packet 1591 * @cm_node: connection's node 1592 */ 1593 void 1594 irdma_send_ack(struct irdma_cm_node *cm_node) 1595 { 1596 struct irdma_puda_buf *sqbuf; 1597 struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi; 1598 1599 sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, NULL, NULL, 1600 SET_ACK); 1601 if (sqbuf) { 1602 if (sqbuf->ah) 1603 atomic_inc(&sqbuf->ah->ah_info.ah_refcnt); 1604 1605 if (irdma_cm_send_buf(vsi->ilq, sqbuf)) 1606 cm_node->cm_core->cm_free_ah(cm_node); 1607 1608 irdma_rem_ref_sqbuf(vsi, sqbuf); 1609 } 1610 } 1611 1612 /** 1613 * irdma_send_fin - Send FIN pkt 1614 * @cm_node: connection's node 1615 */ 1616 static int 1617 irdma_send_fin(struct irdma_cm_node *cm_node) 1618 { 1619 struct irdma_puda_buf *sqbuf; 1620 1621 sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, NULL, NULL, 1622 SET_ACK | SET_FIN); 1623 if (!sqbuf) 1624 return -ENOMEM; 1625 1626 return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1, 1627 0); 1628 } 1629 1630 /** 1631 * irdma_find_listener - find a cm node listening on this addr-port pair 1632 * @cm_core: cm's core 1633 * @dst_addr: listener ip addr 1634 * @ipv4: flag indicating IPv4 when true 1635 * @dst_port: listener tcp port num 1636 * @vlan_id: virtual LAN ID 1637 * @listener_state: state to match with listen node's 1638 */ 1639 static struct irdma_cm_listener * 1640 irdma_find_listener(struct irdma_cm_core *cm_core, u32 *dst_addr, bool ipv4, u16 dst_port, 1641 u16 vlan_id, enum irdma_cm_listener_state listener_state) 1642 { 1643 struct irdma_cm_listener *listen_node; 1644 static const u32 ip_zero[4] = {0, 0, 0, 0}; 1645 u32 listen_addr[4]; 1646 u16 listen_port; 1647 unsigned long flags; 1648 1649 /* walk list and find cm_node associated with this session ID */ 1650 spin_lock_irqsave(&cm_core->listen_list_lock, flags); 1651 list_for_each_entry(listen_node, &cm_core->listen_list, list) { 1652 memcpy(listen_addr, listen_node->loc_addr, sizeof(listen_addr)); 1653 listen_port = listen_node->loc_port; 1654 if (listen_node->ipv4 != ipv4 || listen_port != dst_port || 1655 !(listener_state & listen_node->listener_state)) 1656 continue; 1657 /* compare node pair, return node handle if a match */ 1658 if (!memcmp(listen_addr, ip_zero, sizeof(listen_addr)) || 1659 (!memcmp(listen_addr, dst_addr, sizeof(listen_addr)) && 1660 vlan_id == listen_node->vlan_id)) { 1661 atomic_inc(&listen_node->refcnt); 1662 spin_unlock_irqrestore(&cm_core->listen_list_lock, 1663 flags); 1664 return listen_node; 1665 } 1666 } 1667 spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); 1668 1669 return NULL; 1670 } 1671 1672 /** 1673 * irdma_del_multiple_qhash - Remove qhash and child listens 1674 * @iwdev: iWarp device 1675 * @cm_info: CM info for parent listen node 1676 * @cm_parent_listen_node: The parent listen node 1677 */ 1678 static int 1679 irdma_del_multiple_qhash(struct irdma_device *iwdev, 1680 struct irdma_cm_info *cm_info, 1681 struct irdma_cm_listener *cm_parent_listen_node) 1682 { 1683 struct irdma_cm_listener *child_listen_node; 1684 struct list_head *pos, *tpos; 1685 unsigned long flags; 1686 int ret = -EINVAL; 1687 1688 spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags); 1689 list_for_each_safe(pos, tpos, 1690 &cm_parent_listen_node->child_listen_list) { 1691 child_listen_node = list_entry(pos, struct irdma_cm_listener, 1692 child_listen_list); 1693 if (child_listen_node->ipv4) 1694 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 1695 "removing child listen for IP=%x, port=%d, vlan=%d\n", 1696 child_listen_node->loc_addr[0], 1697 child_listen_node->loc_port, 1698 child_listen_node->vlan_id); 1699 else 1700 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 1701 "removing child listen for IP=%x:%x:%x:%x, port=%d, vlan=%d\n", 1702 IRDMA_PRINT_IP6(child_listen_node->loc_addr), 1703 child_listen_node->loc_port, 1704 child_listen_node->vlan_id); 1705 list_del(pos); 1706 memcpy(cm_info->loc_addr, child_listen_node->loc_addr, 1707 sizeof(cm_info->loc_addr)); 1708 cm_info->vlan_id = child_listen_node->vlan_id; 1709 if (child_listen_node->qhash_set) { 1710 ret = irdma_manage_qhash(iwdev, cm_info, 1711 IRDMA_QHASH_TYPE_TCP_SYN, 1712 IRDMA_QHASH_MANAGE_TYPE_DELETE, 1713 NULL, false); 1714 child_listen_node->qhash_set = false; 1715 } else { 1716 ret = 0; 1717 } 1718 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 1719 "Child listen node freed = %p\n", 1720 child_listen_node); 1721 kfree(child_listen_node); 1722 cm_parent_listen_node->cm_core->stats_listen_nodes_destroyed++; 1723 } 1724 spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock, flags); 1725 1726 return ret; 1727 } 1728 1729 static u8 irdma_iw_get_vlan_prio(u32 *loc_addr, u8 prio, bool ipv4) 1730 { 1731 return prio; 1732 } 1733 1734 /** 1735 * irdma_get_vlan_mac_ipv6 - Get the vlan and mac for an IPv6 1736 * address 1737 * @addr: local IPv6 address 1738 * @vlan_id: vlan id for the given IPv6 address 1739 * @mac: mac address for the given IPv6 address 1740 * 1741 * Returns the net_device of the IPv6 address and also sets the 1742 * vlan id and mac for that address. 1743 */ 1744 void 1745 irdma_get_vlan_mac_ipv6(struct iw_cm_id *cm_id, u32 *addr, u16 *vlan_id, u8 *mac) 1746 { 1747 if_t ip_dev = NULL; 1748 struct in6_addr laddr6; 1749 struct vnet *vnet = &init_net; 1750 struct ifaddr *ifa; 1751 u16 scope_id = 0; 1752 1753 irdma_copy_ip_htonl(laddr6.__u6_addr.__u6_addr32, addr); 1754 if (vlan_id) 1755 *vlan_id = 0xFFFF; /* Match rdma_vlan_dev_vlan_id() */ 1756 if (mac) 1757 eth_zero_addr(mac); 1758 1759 if (IN6_IS_SCOPE_LINKLOCAL(&laddr6) || 1760 IN6_IS_ADDR_MC_INTFACELOCAL(&laddr6)) 1761 scope_id = ntohs(laddr6.__u6_addr.__u6_addr16[1]); 1762 1763 #ifdef VIMAGE 1764 vnet = irdma_cmid_to_vnet(cm_id); 1765 #endif 1766 ip_dev = ip6_ifp_find(vnet, laddr6, scope_id); 1767 if (ip_dev) { 1768 if (vlan_id) 1769 *vlan_id = rdma_vlan_dev_vlan_id(ip_dev); 1770 ifa = if_getifaddr(ip_dev); 1771 if (ifa && ifa->ifa_addr && mac) 1772 ether_addr_copy(mac, if_getlladdr(ip_dev)); 1773 } 1774 } 1775 1776 /** 1777 * irdma_get_vlan_ipv4 - Returns the vlan_id for IPv4 address 1778 * @addr: local IPv4 address 1779 */ 1780 u16 1781 irdma_get_vlan_ipv4(struct iw_cm_id *cm_id, u32 *addr) 1782 { 1783 u16 vlan_id = 0xFFFF; 1784 #ifdef INET 1785 if_t netdev; 1786 struct vnet *vnet = &init_net; 1787 1788 #ifdef VIMAGE 1789 vnet = irdma_cmid_to_vnet(cm_id); 1790 #endif 1791 netdev = ip_ifp_find(vnet, htonl(addr[0])); 1792 if (netdev) { 1793 vlan_id = rdma_vlan_dev_vlan_id(netdev); 1794 dev_put(netdev); 1795 } 1796 #endif 1797 1798 return vlan_id; 1799 } 1800 1801 /** 1802 * irdma_add_mqh_ifa_cb - Adds multiple qhashes for IPv4/IPv6 1803 * @arg: Calback argument structure from irdma_add_mqh 1804 * @ifa: Current address to compute against 1805 * @count: Current cumulative output of all callbacks in this iteration 1806 * 1807 * Adds a qhash and a child listen node for a single IPv4/IPv6 address 1808 * on the adapter and adds the associated qhash filter 1809 */ 1810 static u_int 1811 irdma_add_mqh_ifa_cb(void *arg, struct ifaddr *ifa, u_int count) 1812 { 1813 struct irdma_add_mqh_cbs *cbs = arg; 1814 struct irdma_cm_listener *child_listen_node; 1815 struct irdma_cm_info *cm_info = cbs->cm_info; 1816 struct irdma_device *iwdev = cbs->iwdev; 1817 struct irdma_cm_listener *cm_parent_listen_node = cbs->cm_listen_node; 1818 if_t ip_dev = ifa->ifa_ifp; 1819 unsigned long flags; 1820 int ret; 1821 1822 if (count) 1823 return 0; 1824 1825 child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_ATOMIC); 1826 if (!child_listen_node) { 1827 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 1828 "listener memory allocation\n"); 1829 return -ENOMEM; 1830 } 1831 1832 memcpy(child_listen_node, cm_parent_listen_node, 1833 sizeof(*child_listen_node)); 1834 cm_info->vlan_id = rdma_vlan_dev_vlan_id(ip_dev); 1835 child_listen_node->vlan_id = cm_info->vlan_id; 1836 if (cm_info->ipv4) { 1837 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 1838 "Allocating child CM Listener forIP=%x, vlan_id=%d, MAC=%x:%x:%x:%x:%x:%x\n", 1839 ((struct sockaddr_in *)&ifa->ifa_addr)->sin_addr.s_addr, 1840 rdma_vlan_dev_vlan_id(ip_dev), 1841 if_getlladdr(ip_dev)[0], if_getlladdr(ip_dev)[1], 1842 if_getlladdr(ip_dev)[2], if_getlladdr(ip_dev)[3], 1843 if_getlladdr(ip_dev)[4], if_getlladdr(ip_dev)[5]); 1844 child_listen_node->loc_addr[0] = 1845 ntohl(((struct sockaddr_in *)ifa->ifa_addr)->sin_addr.s_addr); 1846 } else { 1847 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 1848 "IP=%x:%x:%x:%x, vlan_id=%d, MAC=%x:%x:%x:%x:%x:%x\n", 1849 IRDMA_PRINT_IP6(&((struct sockaddr_in6 *)ifa->ifa_addr)->sin6_addr), 1850 rdma_vlan_dev_vlan_id(ip_dev), 1851 if_getlladdr(ip_dev)[0], if_getlladdr(ip_dev)[1], 1852 if_getlladdr(ip_dev)[2], if_getlladdr(ip_dev)[3], 1853 if_getlladdr(ip_dev)[4], if_getlladdr(ip_dev)[5]); 1854 irdma_copy_ip_ntohl(child_listen_node->loc_addr, 1855 ((struct sockaddr_in6 *)ifa->ifa_addr)->sin6_addr.__u6_addr.__u6_addr32); 1856 } 1857 memcpy(cm_info->loc_addr, child_listen_node->loc_addr, 1858 sizeof(cm_info->loc_addr)); 1859 if (!iwdev->vsi.dscp_mode) 1860 cm_info->user_pri = 1861 irdma_iw_get_vlan_prio(child_listen_node->loc_addr, 1862 cm_info->user_pri, 1863 cm_info->ipv4); 1864 ret = irdma_add_qhash_wait_no_lock(iwdev, cm_info); 1865 if (ret) { 1866 kfree(child_listen_node); 1867 return ret; 1868 } 1869 1870 child_listen_node->qhash_set = true; 1871 spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags); 1872 list_add(&child_listen_node->child_listen_list, 1873 &cm_parent_listen_node->child_listen_list); 1874 spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock, flags); 1875 cm_parent_listen_node->cm_core->stats_listen_nodes_created++; 1876 1877 return 0; 1878 } 1879 1880 /** 1881 * irdma_add_mqh - Adds multiple qhashes 1882 * @iwdev: iWarp device 1883 * @cm_info: CM info for parent listen node 1884 * @cm_listen_node: The parent listen node 1885 */ 1886 static int 1887 irdma_add_mqh(struct irdma_device *iwdev, 1888 struct irdma_cm_info *cm_info, 1889 struct irdma_cm_listener *cm_listen_node) 1890 { 1891 struct epoch_tracker et; 1892 struct irdma_add_mqh_cbs cbs; 1893 struct if_iter iter; 1894 if_t ifp; 1895 int err = -ENOENT; 1896 1897 cbs.iwdev = iwdev; 1898 cbs.cm_info = cm_info; 1899 cbs.cm_listen_node = cm_listen_node; 1900 1901 VNET_ITERATOR_DECL(vnet_iter); 1902 1903 VNET_LIST_RLOCK(); 1904 NET_EPOCH_ENTER(et); 1905 VNET_FOREACH(vnet_iter) { 1906 CURVNET_SET_QUIET(vnet_iter); 1907 for (ifp = if_iter_start(&iter); ifp != NULL; ifp = if_iter_next(&iter)) { 1908 if (!(if_getflags(ifp) & IFF_UP)) 1909 continue; 1910 1911 if (((rdma_vlan_dev_vlan_id(ifp) >= VLAN_N_VID) || 1912 (rdma_vlan_dev_real_dev(ifp) != iwdev->netdev)) && 1913 ifp != iwdev->netdev) 1914 continue; 1915 1916 if_addr_rlock(ifp); 1917 if (cm_info->ipv4) 1918 err = if_foreach_addr_type(ifp, AF_INET, irdma_add_mqh_ifa_cb, &cbs); 1919 else 1920 err = if_foreach_addr_type(ifp, AF_INET6, irdma_add_mqh_ifa_cb, &cbs); 1921 if_addr_runlock(ifp); 1922 } 1923 if_iter_finish(&iter); 1924 CURVNET_RESTORE(); 1925 } 1926 NET_EPOCH_EXIT(et); 1927 VNET_LIST_RUNLOCK(); 1928 1929 return err; 1930 } 1931 1932 /** 1933 * irdma_reset_list_prep - add connection nodes slated for reset to list 1934 * @cm_core: cm's core 1935 * @listener: pointer to listener node 1936 * @reset_list: a list to which cm_node will be selected 1937 */ 1938 static void 1939 irdma_reset_list_prep(struct irdma_cm_core *cm_core, 1940 struct irdma_cm_listener *listener, 1941 struct list_head *reset_list) 1942 { 1943 struct irdma_cm_node *cm_node; 1944 int bkt; 1945 1946 HASH_FOR_EACH_RCU(cm_core->cm_hash_tbl, bkt, cm_node, list) { 1947 if (cm_node->listener == listener && 1948 !cm_node->accelerated && 1949 irdma_add_ref_cmnode(cm_node)) 1950 list_add(&cm_node->reset_entry, reset_list); 1951 } 1952 } 1953 1954 /** 1955 * irdma_dec_refcnt_listen - delete listener and associated cm nodes 1956 * @cm_core: cm's core 1957 * @listener: pointer to listener node 1958 * @free_hanging_nodes: to free associated cm_nodes 1959 * @apbvt_del: flag to delete the apbvt 1960 */ 1961 static int 1962 irdma_dec_refcnt_listen(struct irdma_cm_core *cm_core, 1963 struct irdma_cm_listener *listener, 1964 int free_hanging_nodes, bool apbvt_del) 1965 { 1966 struct list_head *list_pos; 1967 struct list_head *list_temp; 1968 struct irdma_cm_node *cm_node; 1969 struct list_head reset_list; 1970 struct irdma_cm_info nfo; 1971 enum irdma_cm_node_state old_state; 1972 unsigned long flags; 1973 int err; 1974 1975 /* free non-accelerated child nodes for this listener */ 1976 INIT_LIST_HEAD(&reset_list); 1977 if (free_hanging_nodes) { 1978 rcu_read_lock(); 1979 irdma_reset_list_prep(cm_core, listener, &reset_list); 1980 rcu_read_unlock(); 1981 } 1982 1983 list_for_each_safe(list_pos, list_temp, &reset_list) { 1984 cm_node = container_of(list_pos, struct irdma_cm_node, 1985 reset_entry); 1986 if (cm_node->state >= IRDMA_CM_STATE_FIN_WAIT1) { 1987 irdma_rem_ref_cmnode(cm_node); 1988 continue; 1989 } 1990 1991 irdma_cleanup_retrans_entry(cm_node); 1992 err = irdma_send_reset(cm_node); 1993 if (err) { 1994 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_CLOSED); 1995 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 1996 "send reset failed\n"); 1997 } else { 1998 old_state = irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_LISTENER_DESTROYED); 1999 if (old_state != IRDMA_CM_STATE_MPAREQ_RCVD) 2000 irdma_rem_ref_cmnode(cm_node); 2001 } 2002 } 2003 2004 if (atomic_dec_and_test(&listener->refcnt)) { 2005 spin_lock_irqsave(&cm_core->listen_list_lock, flags); 2006 list_del(&listener->list); 2007 spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); 2008 2009 if (apbvt_del) 2010 irdma_del_apbvt(listener->iwdev, 2011 listener->apbvt_entry); 2012 memcpy(nfo.loc_addr, listener->loc_addr, sizeof(nfo.loc_addr)); 2013 nfo.loc_port = listener->loc_port; 2014 nfo.ipv4 = listener->ipv4; 2015 nfo.vlan_id = listener->vlan_id; 2016 nfo.user_pri = listener->user_pri; 2017 nfo.qh_qpid = listener->iwdev->vsi.ilq->qp_id; 2018 2019 if (!list_empty(&listener->child_listen_list)) { 2020 irdma_del_multiple_qhash(listener->iwdev, &nfo, 2021 listener); 2022 } else { 2023 if (listener->qhash_set) 2024 irdma_manage_qhash(listener->iwdev, 2025 &nfo, 2026 IRDMA_QHASH_TYPE_TCP_SYN, 2027 IRDMA_QHASH_MANAGE_TYPE_DELETE, 2028 NULL, false); 2029 } 2030 2031 cm_core->stats_listen_destroyed++; 2032 cm_core->stats_listen_nodes_destroyed++; 2033 irdma_debug(&listener->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 2034 "loc_port=0x%04x loc_addr=%x cm_listen_node=%p cm_id=%p qhash_set=%d vlan_id=%d apbvt_del=%d\n", 2035 listener->loc_port, listener->loc_addr[0], listener, 2036 listener->cm_id, listener->qhash_set, 2037 listener->vlan_id, apbvt_del); 2038 kfree(listener); 2039 listener = NULL; 2040 return 0; 2041 } 2042 2043 return -EINVAL; 2044 } 2045 2046 /** 2047 * irdma_cm_del_listen - delete a listener 2048 * @cm_core: cm's core 2049 * @listener: passive connection's listener 2050 * @apbvt_del: flag to delete apbvt 2051 */ 2052 static int 2053 irdma_cm_del_listen(struct irdma_cm_core *cm_core, 2054 struct irdma_cm_listener *listener, 2055 bool apbvt_del) 2056 { 2057 listener->listener_state = IRDMA_CM_LISTENER_PASSIVE_STATE; 2058 listener->cm_id = NULL; 2059 2060 return irdma_dec_refcnt_listen(cm_core, listener, 1, apbvt_del); 2061 } 2062 2063 /** 2064 * irdma_find_node - find a cm node that matches the reference cm node 2065 * @cm_core: cm's core 2066 * @rem_port: remote tcp port num 2067 * @rem_addr: remote ip addr 2068 * @loc_port: local tcp port num 2069 * @loc_addr: local ip addr 2070 * @vlan_id: local VLAN ID 2071 */ 2072 struct irdma_cm_node * 2073 irdma_find_node(struct irdma_cm_core *cm_core, 2074 u16 rem_port, u32 *rem_addr, u16 loc_port, 2075 u32 *loc_addr, u16 vlan_id) 2076 { 2077 struct irdma_cm_node *cm_node; 2078 u32 key = (rem_port << 16) | loc_port; 2079 2080 rcu_read_lock(); 2081 HASH_FOR_EACH_POSSIBLE_RCU(cm_core->cm_hash_tbl, cm_node, list, key) { 2082 if (cm_node->vlan_id == vlan_id && 2083 cm_node->loc_port == loc_port && cm_node->rem_port == rem_port && 2084 !memcmp(cm_node->loc_addr, loc_addr, sizeof(cm_node->loc_addr)) && 2085 !memcmp(cm_node->rem_addr, rem_addr, sizeof(cm_node->rem_addr))) { 2086 if (!irdma_add_ref_cmnode(cm_node)) 2087 goto exit; 2088 rcu_read_unlock(); 2089 return cm_node; 2090 } 2091 } 2092 2093 exit: 2094 rcu_read_unlock(); 2095 2096 /* no owner node */ 2097 return NULL; 2098 } 2099 2100 /** 2101 * irdma_add_hte_node - add a cm node to the hash table 2102 * @cm_core: cm's core 2103 * @cm_node: connection's node 2104 */ 2105 static void 2106 irdma_add_hte_node(struct irdma_cm_core *cm_core, 2107 struct irdma_cm_node *cm_node) 2108 { 2109 unsigned long flags; 2110 u32 key = (cm_node->rem_port << 16) | cm_node->loc_port; 2111 2112 spin_lock_irqsave(&cm_core->ht_lock, flags); 2113 HASH_ADD_RCU(cm_core->cm_hash_tbl, &cm_node->list, key); 2114 spin_unlock_irqrestore(&cm_core->ht_lock, flags); 2115 } 2116 2117 /** 2118 * irdma_ipv4_is_lpb - check if loopback 2119 * @loc_addr: local addr to compare 2120 * @rem_addr: remote address 2121 */ 2122 bool 2123 irdma_ipv4_is_lpb(u32 loc_addr, u32 rem_addr) 2124 { 2125 return ipv4_is_loopback(htonl(rem_addr)) || (loc_addr == rem_addr); 2126 } 2127 2128 /** 2129 * irdma_ipv6_is_lpb - check if loopback 2130 * @loc_addr: local addr to compare 2131 * @rem_addr: remote address 2132 */ 2133 bool 2134 irdma_ipv6_is_lpb(u32 *loc_addr, u32 *rem_addr) 2135 { 2136 struct in6_addr raddr6; 2137 2138 irdma_copy_ip_htonl(raddr6.__u6_addr.__u6_addr32, rem_addr); 2139 2140 return !memcmp(loc_addr, rem_addr, 16) || ipv6_addr_loopback(&raddr6); 2141 } 2142 2143 /** 2144 * irdma_cm_create_ah - create a cm address handle 2145 * @cm_node: The connection manager node to create AH for 2146 * @wait: Provides option to wait for ah creation or not 2147 */ 2148 static int 2149 irdma_cm_create_ah(struct irdma_cm_node *cm_node, bool wait) 2150 { 2151 struct irdma_ah_info ah_info = {0}; 2152 struct irdma_device *iwdev = cm_node->iwdev; 2153 #ifdef VIMAGE 2154 struct vnet *vnet = irdma_cmid_to_vnet(cm_node->cm_id); 2155 #endif 2156 2157 ether_addr_copy(ah_info.mac_addr, if_getlladdr(iwdev->netdev)); 2158 2159 ah_info.hop_ttl = 0x40; 2160 ah_info.tc_tos = cm_node->tos; 2161 ah_info.vsi = &iwdev->vsi; 2162 2163 if (cm_node->ipv4) { 2164 ah_info.ipv4_valid = true; 2165 ah_info.dest_ip_addr[0] = cm_node->rem_addr[0]; 2166 ah_info.src_ip_addr[0] = cm_node->loc_addr[0]; 2167 CURVNET_SET_QUIET(vnet); 2168 ah_info.do_lpbk = irdma_ipv4_is_lpb(ah_info.src_ip_addr[0], 2169 ah_info.dest_ip_addr[0]); 2170 CURVNET_RESTORE(); 2171 } else { 2172 memcpy(ah_info.dest_ip_addr, cm_node->rem_addr, 2173 sizeof(ah_info.dest_ip_addr)); 2174 memcpy(ah_info.src_ip_addr, cm_node->loc_addr, 2175 sizeof(ah_info.src_ip_addr)); 2176 ah_info.do_lpbk = irdma_ipv6_is_lpb(ah_info.src_ip_addr, 2177 ah_info.dest_ip_addr); 2178 } 2179 2180 ah_info.vlan_tag = cm_node->vlan_id; 2181 if (cm_node->vlan_id < VLAN_N_VID) { 2182 ah_info.insert_vlan_tag = 1; 2183 ah_info.vlan_tag |= cm_node->user_pri << VLAN_PRIO_SHIFT; 2184 } 2185 2186 ah_info.dst_arpindex = 2187 irdma_arp_table(iwdev->rf, ah_info.dest_ip_addr, 2188 NULL, IRDMA_ARP_RESOLVE); 2189 2190 if (irdma_puda_create_ah(&iwdev->rf->sc_dev, &ah_info, wait, 2191 IRDMA_PUDA_RSRC_TYPE_ILQ, cm_node, 2192 &cm_node->ah)) 2193 return -ENOMEM; 2194 2195 atomic_set(&cm_node->ah->ah_info.ah_refcnt, 1); 2196 2197 return 0; 2198 } 2199 2200 /** 2201 * irdma_cm_free_ah_worker - async free a cm address handle 2202 * @work: pointer to ah structure 2203 */ 2204 static void 2205 irdma_cm_free_ah_worker(struct work_struct *work) 2206 { 2207 struct irdma_sc_ah *ah = container_of(work, struct irdma_sc_ah, ah_free_work); 2208 2209 irdma_puda_free_ah(ah->dev, ah); 2210 } 2211 2212 /** 2213 * irdma_cm_free_ah - free a cm address handle 2214 * @cm_node: The connection manager node to create AH for 2215 */ 2216 static void 2217 irdma_cm_free_ah(struct irdma_cm_node *cm_node) 2218 { 2219 struct irdma_device *iwdev = cm_node->iwdev; 2220 2221 if (cm_node->ah) { 2222 if (!atomic_dec_and_test(&cm_node->ah->ah_info.ah_refcnt)) 2223 return; 2224 2225 INIT_WORK(&cm_node->ah->ah_free_work, irdma_cm_free_ah_worker); 2226 queue_work(iwdev->cleanup_wq, &cm_node->ah->ah_free_work); 2227 cm_node->ah = NULL; 2228 } 2229 } 2230 2231 /** 2232 * irdma_make_cm_node - create a new instance of a cm node 2233 * @cm_core: cm's core 2234 * @iwdev: iwarp device structure 2235 * @cm_info: quad info for connection 2236 * @listener: passive connection's listener 2237 */ 2238 static struct irdma_cm_node * 2239 irdma_make_cm_node(struct irdma_cm_core *cm_core, struct irdma_device *iwdev, 2240 struct irdma_cm_info *cm_info, 2241 struct irdma_cm_listener *listener) 2242 { 2243 struct irdma_cm_node *cm_node; 2244 int arpindex; 2245 if_t netdev = iwdev->netdev; 2246 int ret; 2247 2248 /* create an hte and cm_node for this instance */ 2249 cm_node = kzalloc(sizeof(*cm_node), GFP_ATOMIC); 2250 if (!cm_node) 2251 return ERR_PTR(-ENOMEM); 2252 2253 /* set our node specific transport info */ 2254 cm_node->ipv4 = cm_info->ipv4; 2255 cm_node->vlan_id = cm_info->vlan_id; 2256 if (cm_node->vlan_id >= VLAN_N_VID && iwdev->dcb_vlan_mode) 2257 cm_node->vlan_id = 0; 2258 cm_node->tos = cm_info->tos; 2259 cm_node->user_pri = cm_info->user_pri; 2260 if (listener) { 2261 if (listener->tos != cm_info->tos) 2262 irdma_dev_warn(&iwdev->ibdev, 2263 "application TOS[%d] and remote client TOS[%d] mismatch\n", 2264 listener->tos, cm_info->tos); 2265 if (iwdev->vsi.dscp_mode) { 2266 cm_node->user_pri = listener->user_pri; 2267 } else { 2268 cm_node->tos = max(listener->tos, cm_info->tos); 2269 cm_node->user_pri = rt_tos2priority(cm_node->tos); 2270 cm_node->user_pri = 2271 irdma_iw_get_vlan_prio(cm_info->loc_addr, 2272 cm_node->user_pri, 2273 cm_info->ipv4); 2274 } 2275 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DCB, 2276 "listener: TOS:[%d] UP:[%d]\n", cm_node->tos, 2277 cm_node->user_pri); 2278 } 2279 memcpy(cm_node->loc_addr, cm_info->loc_addr, sizeof(cm_node->loc_addr)); 2280 memcpy(cm_node->rem_addr, cm_info->rem_addr, sizeof(cm_node->rem_addr)); 2281 cm_node->loc_port = cm_info->loc_port; 2282 cm_node->rem_port = cm_info->rem_port; 2283 2284 cm_node->mpa_frame_rev = IRDMA_CM_DEFAULT_MPA_VER; 2285 cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO; 2286 cm_node->iwdev = iwdev; 2287 cm_node->dev = &iwdev->rf->sc_dev; 2288 2289 cm_node->ird_size = cm_node->dev->hw_attrs.max_hw_ird; 2290 cm_node->ord_size = cm_node->dev->hw_attrs.max_hw_ord; 2291 2292 cm_node->listener = listener; 2293 cm_node->cm_id = cm_info->cm_id; 2294 ether_addr_copy(cm_node->loc_mac, if_getlladdr(netdev)); 2295 spin_lock_init(&cm_node->retrans_list_lock); 2296 cm_node->ack_rcvd = false; 2297 2298 init_completion(&cm_node->establish_comp); 2299 atomic_set(&cm_node->refcnt, 1); 2300 /* associate our parent CM core */ 2301 cm_node->cm_core = cm_core; 2302 cm_node->tcp_cntxt.loc_id = IRDMA_CM_DEFAULT_LOCAL_ID; 2303 cm_node->tcp_cntxt.rcv_wscale = iwdev->rcv_wscale; 2304 cm_node->tcp_cntxt.rcv_wnd = iwdev->rcv_wnd >> cm_node->tcp_cntxt.rcv_wscale; 2305 kc_set_loc_seq_num_mss(cm_node); 2306 2307 arpindex = irdma_resolve_neigh_lpb_chk(iwdev, cm_node, cm_info); 2308 if (arpindex < 0) { 2309 ret = -EINVAL; 2310 goto err; 2311 } 2312 2313 ether_addr_copy(cm_node->rem_mac, iwdev->rf->arp_table[arpindex].mac_addr); 2314 irdma_add_hte_node(cm_core, cm_node); 2315 cm_core->stats_nodes_created++; 2316 return cm_node; 2317 2318 err: 2319 kfree(cm_node); 2320 2321 return ERR_PTR(ret); 2322 } 2323 2324 static void 2325 irdma_destroy_connection(struct irdma_cm_node *cm_node) 2326 { 2327 struct irdma_cm_core *cm_core = cm_node->cm_core; 2328 struct irdma_qp *iwqp; 2329 struct irdma_cm_info nfo; 2330 2331 /* if the node is destroyed before connection was accelerated */ 2332 if (!cm_node->accelerated && cm_node->accept_pend) { 2333 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 2334 "node destroyed before established\n"); 2335 atomic_dec(&cm_node->listener->pend_accepts_cnt); 2336 } 2337 2338 if (cm_node->send_entry) 2339 irdma_cleanup_retrans_entry(cm_node); 2340 if (cm_node->close_entry) 2341 irdma_handle_close_entry(cm_node, 0); 2342 if (cm_node->listener) { 2343 irdma_dec_refcnt_listen(cm_core, cm_node->listener, 0, true); 2344 } else { 2345 if (cm_node->apbvt_set) { 2346 irdma_del_apbvt(cm_node->iwdev, cm_node->apbvt_entry); 2347 cm_node->apbvt_set = 0; 2348 } 2349 irdma_get_addr_info(cm_node, &nfo); 2350 if (cm_node->qhash_set) { 2351 nfo.qh_qpid = cm_node->iwdev->vsi.ilq->qp_id; 2352 irdma_manage_qhash(cm_node->iwdev, &nfo, 2353 IRDMA_QHASH_TYPE_TCP_ESTABLISHED, 2354 IRDMA_QHASH_MANAGE_TYPE_DELETE, NULL, 2355 false); 2356 cm_node->qhash_set = 0; 2357 } 2358 } 2359 2360 iwqp = cm_node->iwqp; 2361 if (iwqp) { 2362 cm_node->cm_id->rem_ref(cm_node->cm_id); 2363 cm_node->cm_id = NULL; 2364 iwqp->cm_id = NULL; 2365 irdma_qp_rem_ref(&iwqp->ibqp); 2366 cm_node->iwqp = NULL; 2367 } else if (cm_node->qhash_set) { 2368 irdma_get_addr_info(cm_node, &nfo); 2369 nfo.qh_qpid = cm_node->iwdev->vsi.ilq->qp_id; 2370 irdma_manage_qhash(cm_node->iwdev, &nfo, 2371 IRDMA_QHASH_TYPE_TCP_ESTABLISHED, 2372 IRDMA_QHASH_MANAGE_TYPE_DELETE, NULL, false); 2373 cm_node->qhash_set = 0; 2374 } 2375 2376 cm_core->cm_free_ah(cm_node); 2377 } 2378 2379 /** 2380 * irdma_add_ref_cmnode - add reference to an instance of a cm node 2381 * @cm_node: connection's node 2382 */ 2383 bool 2384 irdma_add_ref_cmnode(struct irdma_cm_node *cm_node) 2385 { 2386 if (atomic_inc_not_zero(&cm_node->refcnt)) 2387 return true; 2388 2389 /* 2390 * Trying to add refcount to a cmnode being destroyed. 2391 */ 2392 2393 return false; 2394 } 2395 2396 /** 2397 * irdma_rem_ref_cmnode - destroy an instance of a cm node 2398 * @cm_node: connection's node 2399 */ 2400 void 2401 irdma_rem_ref_cmnode(struct irdma_cm_node *cm_node) 2402 { 2403 struct irdma_cm_core *cm_core = cm_node->cm_core; 2404 unsigned long flags; 2405 2406 spin_lock_irqsave(&cm_core->ht_lock, flags); 2407 2408 if (!atomic_dec_and_test(&cm_node->refcnt)) { 2409 spin_unlock_irqrestore(&cm_core->ht_lock, flags); 2410 return; 2411 } 2412 if (cm_node->iwqp) { 2413 cm_node->iwqp->cm_node = NULL; 2414 cm_node->iwqp->cm_id = NULL; 2415 } 2416 HASH_DEL_RCU(cm_core->cm_hash_tbl, &cm_node->list); 2417 cm_node->cm_core->stats_nodes_destroyed++; 2418 2419 spin_unlock_irqrestore(&cm_core->ht_lock, flags); 2420 2421 irdma_destroy_connection(cm_node); 2422 2423 kfree_rcu(cm_node, rcu_head); 2424 } 2425 2426 /** 2427 * irdma_handle_fin_pkt - FIN packet received 2428 * @cm_node: connection's node 2429 */ 2430 static void 2431 irdma_handle_fin_pkt(struct irdma_cm_node *cm_node) 2432 { 2433 switch (cm_node->state) { 2434 case IRDMA_CM_STATE_SYN_RCVD: 2435 case IRDMA_CM_STATE_SYN_SENT: 2436 case IRDMA_CM_STATE_ESTABLISHED: 2437 case IRDMA_CM_STATE_MPAREJ_RCVD: 2438 cm_node->tcp_cntxt.rcv_nxt++; 2439 irdma_cleanup_retrans_entry(cm_node); 2440 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_LAST_ACK); 2441 irdma_send_fin(cm_node); 2442 break; 2443 case IRDMA_CM_STATE_MPAREQ_SENT: 2444 irdma_create_event(cm_node, IRDMA_CM_EVENT_ABORTED); 2445 cm_node->tcp_cntxt.rcv_nxt++; 2446 irdma_cleanup_retrans_entry(cm_node); 2447 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_CLOSED); 2448 irdma_add_ref_cmnode(cm_node); 2449 if (irdma_send_reset(cm_node)) 2450 irdma_rem_ref_cmnode(cm_node); 2451 2452 break; 2453 case IRDMA_CM_STATE_FIN_WAIT1: 2454 cm_node->tcp_cntxt.rcv_nxt++; 2455 irdma_cleanup_retrans_entry(cm_node); 2456 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_CLOSING); 2457 irdma_send_ack(cm_node); 2458 /* 2459 * Wait for ACK as this is simultaneous close. After we receive ACK, do not send anything. Just rm the 2460 * node. 2461 */ 2462 break; 2463 case IRDMA_CM_STATE_FIN_WAIT2: 2464 cm_node->tcp_cntxt.rcv_nxt++; 2465 irdma_cleanup_retrans_entry(cm_node); 2466 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_TIME_WAIT); 2467 irdma_send_ack(cm_node); 2468 irdma_schedule_cm_timer(cm_node, NULL, IRDMA_TIMER_TYPE_CLOSE, 2469 1, 0); 2470 break; 2471 case IRDMA_CM_STATE_TIME_WAIT: 2472 cm_node->tcp_cntxt.rcv_nxt++; 2473 irdma_cleanup_retrans_entry(cm_node); 2474 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_CLOSED); 2475 irdma_rem_ref_cmnode(cm_node); 2476 break; 2477 case IRDMA_CM_STATE_OFFLOADED: 2478 default: 2479 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 2480 "bad state node state = %d\n", cm_node->state); 2481 break; 2482 } 2483 } 2484 2485 /** 2486 * irdma_handle_rst_pkt - process received RST packet 2487 * @cm_node: connection's node 2488 * @rbuf: receive buffer 2489 */ 2490 static void 2491 irdma_handle_rst_pkt(struct irdma_cm_node *cm_node, 2492 struct irdma_puda_buf *rbuf) 2493 { 2494 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 2495 "caller: %pS cm_node=%p state=%d rem_port=0x%04x loc_port=0x%04x rem_addr=%x loc_addr=%x\n", 2496 __builtin_return_address(0), cm_node, cm_node->state, 2497 cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr[0], 2498 cm_node->loc_addr[0]); 2499 2500 irdma_cleanup_retrans_entry(cm_node); 2501 switch (cm_node->state) { 2502 case IRDMA_CM_STATE_SYN_SENT: 2503 case IRDMA_CM_STATE_MPAREQ_SENT: 2504 switch (cm_node->mpa_frame_rev) { 2505 case IETF_MPA_V2: 2506 /* Drop down to MPA_V1 */ 2507 cm_node->mpa_frame_rev = IETF_MPA_V1; 2508 /* send a syn and goto syn sent state */ 2509 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_SYN_SENT); 2510 if (irdma_send_syn(cm_node, 0)) 2511 irdma_active_open_err(cm_node, false); 2512 break; 2513 case IETF_MPA_V1: 2514 default: 2515 irdma_active_open_err(cm_node, false); 2516 break; 2517 } 2518 break; 2519 case IRDMA_CM_STATE_MPAREQ_RCVD: 2520 atomic_inc(&cm_node->passive_state); 2521 break; 2522 case IRDMA_CM_STATE_ESTABLISHED: 2523 case IRDMA_CM_STATE_SYN_RCVD: 2524 case IRDMA_CM_STATE_LISTENING: 2525 irdma_passive_open_err(cm_node, false); 2526 break; 2527 case IRDMA_CM_STATE_OFFLOADED: 2528 irdma_active_open_err(cm_node, false); 2529 break; 2530 case IRDMA_CM_STATE_CLOSED: 2531 break; 2532 case IRDMA_CM_STATE_FIN_WAIT2: 2533 case IRDMA_CM_STATE_FIN_WAIT1: 2534 case IRDMA_CM_STATE_LAST_ACK: 2535 case IRDMA_CM_STATE_TIME_WAIT: 2536 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_CLOSED); 2537 irdma_rem_ref_cmnode(cm_node); 2538 break; 2539 default: 2540 break; 2541 } 2542 } 2543 2544 /** 2545 * irdma_handle_rcv_mpa - Process a recv'd mpa buffer 2546 * @cm_node: connection's node 2547 * @rbuf: receive buffer 2548 */ 2549 static void 2550 irdma_handle_rcv_mpa(struct irdma_cm_node *cm_node, 2551 struct irdma_puda_buf *rbuf) 2552 { 2553 int err; 2554 int datasize = rbuf->datalen; 2555 u8 *dataloc = rbuf->data; 2556 2557 enum irdma_cm_event_type type = IRDMA_CM_EVENT_UNKNOWN; 2558 u32 res_type; 2559 2560 err = irdma_parse_mpa(cm_node, dataloc, &res_type, datasize); 2561 if (err) { 2562 if (irdma_cm_node_cmp_state(cm_node, IRDMA_CM_STATE_MPAREQ_SENT)) 2563 irdma_active_open_err(cm_node, true); 2564 else 2565 irdma_passive_open_err(cm_node, true); 2566 return; 2567 } 2568 2569 switch (cm_node->state) { 2570 case IRDMA_CM_STATE_ESTABLISHED: 2571 if (res_type == IRDMA_MPA_REQUEST_REJECT) 2572 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 2573 "state for reject\n"); 2574 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_MPAREQ_RCVD); 2575 type = IRDMA_CM_EVENT_MPA_REQ; 2576 irdma_send_ack(cm_node); /* ACK received MPA request */ 2577 atomic_set(&cm_node->passive_state, 2578 IRDMA_PASSIVE_STATE_INDICATED); 2579 break; 2580 case IRDMA_CM_STATE_MPAREQ_SENT: 2581 irdma_cleanup_retrans_entry(cm_node); 2582 if (res_type == IRDMA_MPA_REQUEST_REJECT) { 2583 type = IRDMA_CM_EVENT_MPA_REJECT; 2584 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_MPAREJ_RCVD); 2585 } else { 2586 type = IRDMA_CM_EVENT_CONNECTED; 2587 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_OFFLOADED); 2588 } 2589 irdma_send_ack(cm_node); 2590 break; 2591 default: 2592 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 2593 "wrong cm_node state=%d\n", cm_node->state); 2594 break; 2595 } 2596 irdma_create_event(cm_node, type); 2597 } 2598 2599 /** 2600 * irdma_check_syn - Check for error on received syn ack 2601 * @cm_node: connection's node 2602 * @tcph: pointer tcp header 2603 */ 2604 static int 2605 irdma_check_syn(struct irdma_cm_node *cm_node, struct tcphdr *tcph) 2606 { 2607 if (ntohl(tcph->th_ack) != cm_node->tcp_cntxt.loc_seq_num) { 2608 irdma_active_open_err(cm_node, true); 2609 return 1; 2610 } 2611 2612 return 0; 2613 } 2614 2615 /** 2616 * irdma_check_seq - check seq numbers if OK 2617 * @cm_node: connection's node 2618 * @tcph: pointer tcp header 2619 */ 2620 static int 2621 irdma_check_seq(struct irdma_cm_node *cm_node, struct tcphdr *tcph) 2622 { 2623 u32 seq; 2624 u32 ack_seq; 2625 u32 loc_seq_num = cm_node->tcp_cntxt.loc_seq_num; 2626 u32 rcv_nxt = cm_node->tcp_cntxt.rcv_nxt; 2627 u32 rcv_wnd; 2628 int err = 0; 2629 2630 seq = ntohl(tcph->th_seq); 2631 ack_seq = ntohl(tcph->th_ack); 2632 rcv_wnd = cm_node->tcp_cntxt.rcv_wnd; 2633 if (ack_seq != loc_seq_num || 2634 !between(seq, rcv_nxt, (rcv_nxt + rcv_wnd))) 2635 err = -1; 2636 if (err) 2637 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 2638 "seq number err\n"); 2639 2640 return err; 2641 } 2642 2643 void 2644 irdma_add_conn_est_qh(struct irdma_cm_node *cm_node) 2645 { 2646 struct irdma_cm_info nfo; 2647 2648 irdma_get_addr_info(cm_node, &nfo); 2649 nfo.qh_qpid = cm_node->iwdev->vsi.ilq->qp_id; 2650 irdma_manage_qhash(cm_node->iwdev, &nfo, 2651 IRDMA_QHASH_TYPE_TCP_ESTABLISHED, 2652 IRDMA_QHASH_MANAGE_TYPE_ADD, 2653 cm_node, false); 2654 cm_node->qhash_set = true; 2655 } 2656 2657 /** 2658 * irdma_handle_syn_pkt - is for Passive node 2659 * @cm_node: connection's node 2660 * @rbuf: receive buffer 2661 */ 2662 static void 2663 irdma_handle_syn_pkt(struct irdma_cm_node *cm_node, 2664 struct irdma_puda_buf *rbuf) 2665 { 2666 struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph; 2667 int err; 2668 u32 inc_sequence; 2669 int optionsize; 2670 2671 optionsize = (tcph->th_off << 2) - sizeof(*tcph); 2672 inc_sequence = ntohl(tcph->th_seq); 2673 2674 switch (cm_node->state) { 2675 case IRDMA_CM_STATE_SYN_SENT: 2676 case IRDMA_CM_STATE_MPAREQ_SENT: 2677 /* Rcvd syn on active open connection */ 2678 irdma_active_open_err(cm_node, 1); 2679 break; 2680 case IRDMA_CM_STATE_LISTENING: 2681 /* Passive OPEN */ 2682 if (atomic_read(&cm_node->listener->pend_accepts_cnt) > 2683 cm_node->listener->backlog) { 2684 cm_node->cm_core->stats_backlog_drops++; 2685 irdma_passive_open_err(cm_node, false); 2686 break; 2687 } 2688 err = irdma_handle_tcp_options(cm_node, tcph, optionsize, 1); 2689 if (err) { 2690 irdma_passive_open_err(cm_node, false); 2691 /* drop pkt */ 2692 break; 2693 } 2694 err = cm_node->cm_core->cm_create_ah(cm_node, false); 2695 if (err) { 2696 irdma_passive_open_err(cm_node, false); 2697 /* drop pkt */ 2698 break; 2699 } 2700 cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1; 2701 cm_node->accept_pend = 1; 2702 atomic_inc(&cm_node->listener->pend_accepts_cnt); 2703 2704 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_SYN_RCVD); 2705 break; 2706 case IRDMA_CM_STATE_CLOSED: 2707 irdma_cleanup_retrans_entry(cm_node); 2708 irdma_add_ref_cmnode(cm_node); 2709 if (irdma_send_reset(cm_node)) 2710 irdma_rem_ref_cmnode(cm_node); 2711 break; 2712 case IRDMA_CM_STATE_OFFLOADED: 2713 case IRDMA_CM_STATE_ESTABLISHED: 2714 case IRDMA_CM_STATE_FIN_WAIT1: 2715 case IRDMA_CM_STATE_FIN_WAIT2: 2716 case IRDMA_CM_STATE_MPAREQ_RCVD: 2717 case IRDMA_CM_STATE_LAST_ACK: 2718 case IRDMA_CM_STATE_CLOSING: 2719 case IRDMA_CM_STATE_UNKNOWN: 2720 default: 2721 break; 2722 } 2723 } 2724 2725 /** 2726 * irdma_handle_synack_pkt - Process SYN+ACK packet (active side) 2727 * @cm_node: connection's node 2728 * @rbuf: receive buffer 2729 */ 2730 static void 2731 irdma_handle_synack_pkt(struct irdma_cm_node *cm_node, 2732 struct irdma_puda_buf *rbuf) 2733 { 2734 struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph; 2735 int err; 2736 u32 inc_sequence; 2737 int optionsize; 2738 2739 optionsize = (tcph->th_off << 2) - sizeof(*tcph); 2740 inc_sequence = ntohl(tcph->th_seq); 2741 switch (cm_node->state) { 2742 case IRDMA_CM_STATE_SYN_SENT: 2743 irdma_cleanup_retrans_entry(cm_node); 2744 /* active open */ 2745 if (irdma_check_syn(cm_node, tcph)) { 2746 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 2747 "check syn fail\n"); 2748 return; 2749 } 2750 cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->th_ack); 2751 /* setup options */ 2752 err = irdma_handle_tcp_options(cm_node, tcph, optionsize, 0); 2753 if (err) { 2754 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 2755 "cm_node=%p tcp_options failed\n", cm_node); 2756 break; 2757 } 2758 irdma_cleanup_retrans_entry(cm_node); 2759 cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1; 2760 irdma_send_ack(cm_node); /* ACK for the syn_ack */ 2761 err = irdma_send_mpa_request(cm_node); 2762 if (err) { 2763 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 2764 "cm_node=%p irdma_send_mpa_request failed\n", 2765 cm_node); 2766 break; 2767 } 2768 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_MPAREQ_SENT); 2769 break; 2770 case IRDMA_CM_STATE_MPAREQ_RCVD: 2771 irdma_passive_open_err(cm_node, true); 2772 break; 2773 case IRDMA_CM_STATE_LISTENING: 2774 cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->th_ack); 2775 irdma_cleanup_retrans_entry(cm_node); 2776 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_CLOSED); 2777 irdma_send_reset(cm_node); 2778 break; 2779 case IRDMA_CM_STATE_CLOSED: 2780 cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->th_ack); 2781 irdma_cleanup_retrans_entry(cm_node); 2782 irdma_add_ref_cmnode(cm_node); 2783 if (irdma_send_reset(cm_node)) 2784 irdma_rem_ref_cmnode(cm_node); 2785 break; 2786 case IRDMA_CM_STATE_ESTABLISHED: 2787 case IRDMA_CM_STATE_FIN_WAIT1: 2788 case IRDMA_CM_STATE_FIN_WAIT2: 2789 case IRDMA_CM_STATE_LAST_ACK: 2790 case IRDMA_CM_STATE_OFFLOADED: 2791 case IRDMA_CM_STATE_CLOSING: 2792 case IRDMA_CM_STATE_UNKNOWN: 2793 case IRDMA_CM_STATE_MPAREQ_SENT: 2794 default: 2795 break; 2796 } 2797 } 2798 2799 /** 2800 * irdma_handle_ack_pkt - process packet with ACK 2801 * @cm_node: connection's node 2802 * @rbuf: receive buffer 2803 */ 2804 static int 2805 irdma_handle_ack_pkt(struct irdma_cm_node *cm_node, 2806 struct irdma_puda_buf *rbuf) 2807 { 2808 struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph; 2809 u32 inc_sequence; 2810 int ret; 2811 int optionsize; 2812 u32 datasize = rbuf->datalen; 2813 2814 optionsize = (tcph->th_off << 2) - sizeof(*tcph); 2815 2816 if (irdma_check_seq(cm_node, tcph)) 2817 return -EINVAL; 2818 2819 inc_sequence = ntohl(tcph->th_seq); 2820 switch (cm_node->state) { 2821 case IRDMA_CM_STATE_SYN_RCVD: 2822 irdma_cleanup_retrans_entry(cm_node); 2823 ret = irdma_handle_tcp_options(cm_node, tcph, optionsize, 1); 2824 if (ret) 2825 return ret; 2826 cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->th_ack); 2827 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_ESTABLISHED); 2828 if (datasize) { 2829 cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize; 2830 irdma_handle_rcv_mpa(cm_node, rbuf); 2831 } 2832 break; 2833 case IRDMA_CM_STATE_ESTABLISHED: 2834 irdma_cleanup_retrans_entry(cm_node); 2835 if (datasize) { 2836 cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize; 2837 irdma_handle_rcv_mpa(cm_node, rbuf); 2838 } 2839 break; 2840 case IRDMA_CM_STATE_MPAREQ_SENT: 2841 cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->th_ack); 2842 if (datasize) { 2843 cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize; 2844 cm_node->ack_rcvd = false; 2845 irdma_handle_rcv_mpa(cm_node, rbuf); 2846 } else { 2847 cm_node->ack_rcvd = true; 2848 } 2849 break; 2850 case IRDMA_CM_STATE_LISTENING: 2851 irdma_cleanup_retrans_entry(cm_node); 2852 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_CLOSED); 2853 irdma_send_reset(cm_node); 2854 break; 2855 case IRDMA_CM_STATE_CLOSED: 2856 irdma_cleanup_retrans_entry(cm_node); 2857 irdma_add_ref_cmnode(cm_node); 2858 if (irdma_send_reset(cm_node)) 2859 irdma_rem_ref_cmnode(cm_node); 2860 break; 2861 case IRDMA_CM_STATE_LAST_ACK: 2862 case IRDMA_CM_STATE_CLOSING: 2863 irdma_cleanup_retrans_entry(cm_node); 2864 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_CLOSED); 2865 irdma_rem_ref_cmnode(cm_node); 2866 break; 2867 case IRDMA_CM_STATE_FIN_WAIT1: 2868 irdma_cleanup_retrans_entry(cm_node); 2869 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_FIN_WAIT2); 2870 break; 2871 case IRDMA_CM_STATE_SYN_SENT: 2872 case IRDMA_CM_STATE_FIN_WAIT2: 2873 case IRDMA_CM_STATE_OFFLOADED: 2874 case IRDMA_CM_STATE_MPAREQ_RCVD: 2875 case IRDMA_CM_STATE_UNKNOWN: 2876 default: 2877 irdma_cleanup_retrans_entry(cm_node); 2878 break; 2879 } 2880 2881 return 0; 2882 } 2883 2884 /** 2885 * irdma_process_pkt - process cm packet 2886 * @cm_node: connection's node 2887 * @rbuf: receive buffer 2888 */ 2889 static void 2890 irdma_process_pkt(struct irdma_cm_node *cm_node, 2891 struct irdma_puda_buf *rbuf) 2892 { 2893 enum irdma_tcpip_pkt_type pkt_type = IRDMA_PKT_TYPE_UNKNOWN; 2894 struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph; 2895 u32 fin_set = 0; 2896 int err; 2897 2898 if (tcp_get_flags(tcph) & TH_RST) { 2899 pkt_type = IRDMA_PKT_TYPE_RST; 2900 } else if (tcp_get_flags(tcph) & TH_SYN) { 2901 pkt_type = IRDMA_PKT_TYPE_SYN; 2902 if (tcp_get_flags(tcph) & TH_ACK) 2903 pkt_type = IRDMA_PKT_TYPE_SYNACK; 2904 } else if (tcp_get_flags(tcph) & TH_ACK) { 2905 pkt_type = IRDMA_PKT_TYPE_ACK; 2906 } 2907 if (tcp_get_flags(tcph) & TH_FIN) 2908 fin_set = 1; 2909 2910 switch (pkt_type) { 2911 case IRDMA_PKT_TYPE_SYN: 2912 irdma_handle_syn_pkt(cm_node, rbuf); 2913 break; 2914 case IRDMA_PKT_TYPE_SYNACK: 2915 irdma_handle_synack_pkt(cm_node, rbuf); 2916 break; 2917 case IRDMA_PKT_TYPE_ACK: 2918 err = irdma_handle_ack_pkt(cm_node, rbuf); 2919 if (fin_set && !err) 2920 irdma_handle_fin_pkt(cm_node); 2921 break; 2922 case IRDMA_PKT_TYPE_RST: 2923 irdma_handle_rst_pkt(cm_node, rbuf); 2924 break; 2925 default: 2926 if (fin_set && 2927 (!irdma_check_seq(cm_node, (struct tcphdr *)rbuf->tcph))) 2928 irdma_handle_fin_pkt(cm_node); 2929 break; 2930 } 2931 } 2932 2933 /** 2934 * irdma_make_listen_node - create a listen node with params 2935 * @cm_core: cm's core 2936 * @iwdev: iwarp device structure 2937 * @cm_info: quad info for connection 2938 */ 2939 static struct irdma_cm_listener * 2940 irdma_make_listen_node(struct irdma_cm_core *cm_core, 2941 struct irdma_device *iwdev, 2942 struct irdma_cm_info *cm_info) 2943 { 2944 struct irdma_cm_listener *listener; 2945 unsigned long flags; 2946 2947 /* cannot have multiple matching listeners */ 2948 listener = irdma_find_listener(cm_core, cm_info->loc_addr, cm_info->ipv4, 2949 cm_info->loc_port, cm_info->vlan_id, 2950 IRDMA_CM_LISTENER_EITHER_STATE); 2951 if (listener && 2952 listener->listener_state == IRDMA_CM_LISTENER_ACTIVE_STATE) { 2953 atomic_dec(&listener->refcnt); 2954 return NULL; 2955 } 2956 2957 if (!listener) { 2958 /* 2959 * create a CM listen node 1/2 node to compare incoming traffic to 2960 */ 2961 listener = kzalloc(sizeof(*listener), GFP_KERNEL); 2962 if (!listener) 2963 return NULL; 2964 cm_core->stats_listen_nodes_created++; 2965 memcpy(listener->loc_addr, cm_info->loc_addr, 2966 sizeof(listener->loc_addr)); 2967 listener->loc_port = cm_info->loc_port; 2968 2969 INIT_LIST_HEAD(&listener->child_listen_list); 2970 2971 atomic_set(&listener->refcnt, 1); 2972 } else { 2973 listener->reused_node = 1; 2974 } 2975 2976 listener->cm_id = cm_info->cm_id; 2977 listener->ipv4 = cm_info->ipv4; 2978 listener->vlan_id = cm_info->vlan_id; 2979 atomic_set(&listener->pend_accepts_cnt, 0); 2980 listener->cm_core = cm_core; 2981 listener->iwdev = iwdev; 2982 2983 listener->backlog = cm_info->backlog; 2984 listener->listener_state = IRDMA_CM_LISTENER_ACTIVE_STATE; 2985 2986 if (!listener->reused_node) { 2987 spin_lock_irqsave(&cm_core->listen_list_lock, flags); 2988 list_add(&listener->list, &cm_core->listen_list); 2989 spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); 2990 } 2991 2992 return listener; 2993 } 2994 2995 /** 2996 * irdma_create_cm_node - make a connection node with params 2997 * @cm_core: cm's core 2998 * @iwdev: iwarp device structure 2999 * @conn_param: connection parameters 3000 * @cm_info: quad info for connection 3001 * @caller_cm_node: pointer to cm_node structure to return 3002 */ 3003 static int 3004 irdma_create_cm_node(struct irdma_cm_core *cm_core, 3005 struct irdma_device *iwdev, 3006 struct iw_cm_conn_param *conn_param, 3007 struct irdma_cm_info *cm_info, 3008 struct irdma_cm_node **caller_cm_node) 3009 { 3010 struct irdma_cm_node *cm_node; 3011 u16 private_data_len = conn_param->private_data_len; 3012 const void *private_data = conn_param->private_data; 3013 3014 /* create a CM connection node */ 3015 cm_node = irdma_make_cm_node(cm_core, iwdev, cm_info, NULL); 3016 if (IS_ERR(cm_node)) 3017 return PTR_ERR(cm_node); 3018 3019 /* set our node side to client (active) side */ 3020 cm_node->tcp_cntxt.client = 1; 3021 cm_node->tcp_cntxt.rcv_wscale = IRDMA_CM_DEFAULT_RCV_WND_SCALE; 3022 3023 irdma_record_ird_ord(cm_node, conn_param->ird, conn_param->ord); 3024 3025 cm_node->pdata.size = private_data_len; 3026 cm_node->pdata.addr = cm_node->pdata_buf; 3027 3028 memcpy(cm_node->pdata_buf, private_data, private_data_len); 3029 *caller_cm_node = cm_node; 3030 3031 return 0; 3032 } 3033 3034 /** 3035 * irdma_cm_reject - reject and teardown a connection 3036 * @cm_node: connection's node 3037 * @pdata: ptr to private data for reject 3038 * @plen: size of private data 3039 */ 3040 static int 3041 irdma_cm_reject(struct irdma_cm_node *cm_node, const void *pdata, 3042 u8 plen) 3043 { 3044 int ret; 3045 int passive_state; 3046 3047 if (cm_node->tcp_cntxt.client) 3048 return 0; 3049 3050 irdma_cleanup_retrans_entry(cm_node); 3051 3052 passive_state = atomic_add_return(1, &cm_node->passive_state); 3053 if (passive_state == IRDMA_SEND_RESET_EVENT) { 3054 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_CLOSED); 3055 irdma_rem_ref_cmnode(cm_node); 3056 return 0; 3057 } 3058 3059 if (irdma_cm_node_cmp_state(cm_node, IRDMA_CM_STATE_LISTENER_DESTROYED)) { 3060 irdma_rem_ref_cmnode(cm_node); 3061 return 0; 3062 } 3063 3064 ret = irdma_send_mpa_reject(cm_node, pdata, plen); 3065 if (!ret) 3066 return 0; 3067 3068 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_CLOSED); 3069 if (irdma_send_reset(cm_node)) 3070 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 3071 "send reset failed\n"); 3072 3073 return ret; 3074 } 3075 3076 /** 3077 * irdma_cm_close - close of cm connection 3078 * @cm_node: connection's node 3079 */ 3080 static int 3081 irdma_cm_close(struct irdma_cm_node *cm_node) 3082 { 3083 switch (cm_node->state) { 3084 case IRDMA_CM_STATE_SYN_RCVD: 3085 case IRDMA_CM_STATE_SYN_SENT: 3086 case IRDMA_CM_STATE_ONE_SIDE_ESTABLISHED: 3087 case IRDMA_CM_STATE_ESTABLISHED: 3088 case IRDMA_CM_STATE_ACCEPTING: 3089 case IRDMA_CM_STATE_MPAREQ_SENT: 3090 case IRDMA_CM_STATE_MPAREQ_RCVD: 3091 irdma_cleanup_retrans_entry(cm_node); 3092 irdma_send_reset(cm_node); 3093 break; 3094 case IRDMA_CM_STATE_CLOSE_WAIT: 3095 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_LAST_ACK); 3096 irdma_send_fin(cm_node); 3097 break; 3098 case IRDMA_CM_STATE_FIN_WAIT1: 3099 case IRDMA_CM_STATE_FIN_WAIT2: 3100 case IRDMA_CM_STATE_LAST_ACK: 3101 case IRDMA_CM_STATE_TIME_WAIT: 3102 case IRDMA_CM_STATE_CLOSING: 3103 return -EINVAL; 3104 case IRDMA_CM_STATE_LISTENING: 3105 irdma_cleanup_retrans_entry(cm_node); 3106 irdma_send_reset(cm_node); 3107 break; 3108 case IRDMA_CM_STATE_MPAREJ_RCVD: 3109 case IRDMA_CM_STATE_UNKNOWN: 3110 case IRDMA_CM_STATE_INITED: 3111 case IRDMA_CM_STATE_CLOSED: 3112 case IRDMA_CM_STATE_LISTENER_DESTROYED: 3113 irdma_rem_ref_cmnode(cm_node); 3114 break; 3115 case IRDMA_CM_STATE_OFFLOADED: 3116 if (cm_node->send_entry) 3117 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 3118 "CM send_entry in OFFLOADED state\n"); 3119 irdma_rem_ref_cmnode(cm_node); 3120 break; 3121 } 3122 3123 return 0; 3124 } 3125 3126 /** 3127 * irdma_receive_ilq - recv an ETHERNET packet, and process it 3128 * through CM 3129 * @vsi: VSI structure of dev 3130 * @rbuf: receive buffer 3131 */ 3132 void 3133 irdma_receive_ilq(struct irdma_sc_vsi *vsi, struct irdma_puda_buf *rbuf) 3134 { 3135 struct irdma_cm_node *cm_node; 3136 struct irdma_cm_listener *listener; 3137 struct ip *iph; 3138 struct ip6_hdr *ip6h; 3139 struct tcphdr *tcph; 3140 struct irdma_cm_info cm_info = {0}; 3141 struct irdma_device *iwdev = vsi->back_vsi; 3142 struct irdma_cm_core *cm_core = &iwdev->cm_core; 3143 struct ether_vlan_header *ethh; 3144 u16 vtag; 3145 3146 /* if vlan, then maclen = 18 else 14 */ 3147 iph = (struct ip *)rbuf->iph; 3148 irdma_debug_buf(vsi->dev, IRDMA_DEBUG_ILQ, "RECEIVE ILQ BUFFER", 3149 rbuf->mem.va, rbuf->totallen); 3150 if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 3151 if (rbuf->vlan_valid) { 3152 vtag = rbuf->vlan_id; 3153 cm_info.user_pri = (vtag & EVL_PRI_MASK) >> 3154 VLAN_PRIO_SHIFT; 3155 cm_info.vlan_id = vtag & EVL_VLID_MASK; 3156 } else { 3157 cm_info.vlan_id = 0xFFFF; 3158 } 3159 } else { 3160 ethh = rbuf->mem.va; 3161 3162 if (ethh->evl_proto == htons(ETH_P_8021Q)) { 3163 vtag = ntohs(ethh->evl_tag); 3164 cm_info.user_pri = (vtag & EVL_PRI_MASK) >> 3165 VLAN_PRIO_SHIFT; 3166 cm_info.vlan_id = vtag & EVL_VLID_MASK; 3167 irdma_debug(&cm_core->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 3168 "vlan_id=%d\n", cm_info.vlan_id); 3169 } else { 3170 cm_info.vlan_id = 0xFFFF; 3171 } 3172 } 3173 tcph = (struct tcphdr *)rbuf->tcph; 3174 3175 if (rbuf->ipv4) { 3176 cm_info.loc_addr[0] = ntohl(iph->ip_dst.s_addr); 3177 cm_info.rem_addr[0] = ntohl(iph->ip_src.s_addr); 3178 cm_info.ipv4 = true; 3179 cm_info.tos = iph->ip_tos; 3180 } else { 3181 ip6h = (struct ip6_hdr *)rbuf->iph; 3182 irdma_copy_ip_ntohl(cm_info.loc_addr, 3183 ip6h->ip6_dst.__u6_addr.__u6_addr32); 3184 irdma_copy_ip_ntohl(cm_info.rem_addr, 3185 ip6h->ip6_src.__u6_addr.__u6_addr32); 3186 cm_info.ipv4 = false; 3187 cm_info.tos = (ip6h->ip6_vfc << 4) | ip6h->ip6_flow; 3188 } 3189 cm_info.loc_port = ntohs(tcph->th_dport); 3190 cm_info.rem_port = ntohs(tcph->th_sport); 3191 cm_node = irdma_find_node(cm_core, cm_info.rem_port, cm_info.rem_addr, 3192 cm_info.loc_port, cm_info.loc_addr, cm_info.vlan_id); 3193 3194 if (!cm_node) { 3195 /* 3196 * Only type of packet accepted are for the PASSIVE open (syn only) 3197 */ 3198 if (!(tcp_get_flags(tcph) & TH_SYN) || tcp_get_flags(tcph) & TH_ACK) 3199 return; 3200 3201 listener = irdma_find_listener(cm_core, 3202 cm_info.loc_addr, 3203 cm_info.ipv4, 3204 cm_info.loc_port, 3205 cm_info.vlan_id, 3206 IRDMA_CM_LISTENER_ACTIVE_STATE); 3207 if (!listener) { 3208 cm_info.cm_id = NULL; 3209 irdma_debug(&cm_core->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 3210 "no listener found\n"); 3211 return; 3212 } 3213 3214 cm_info.cm_id = listener->cm_id; 3215 cm_node = irdma_make_cm_node(cm_core, iwdev, &cm_info, 3216 listener); 3217 if (IS_ERR(cm_node)) { 3218 irdma_debug(&cm_core->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 3219 "allocate node failed ret=%ld\n", 3220 PTR_ERR(cm_node)); 3221 atomic_dec(&listener->refcnt); 3222 return; 3223 } 3224 3225 if (!(tcp_get_flags(tcph) & (TH_RST | TH_FIN))) { 3226 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_LISTENING); 3227 } else { 3228 irdma_rem_ref_cmnode(cm_node); 3229 return; 3230 } 3231 3232 irdma_add_ref_cmnode(cm_node); 3233 } else if (irdma_cm_node_cmp_state(cm_node, IRDMA_CM_STATE_OFFLOADED)) { 3234 irdma_rem_ref_cmnode(cm_node); 3235 return; 3236 } 3237 3238 irdma_process_pkt(cm_node, rbuf); 3239 irdma_rem_ref_cmnode(cm_node); 3240 } 3241 3242 static int 3243 irdma_add_qh(struct irdma_cm_node *cm_node, bool active) 3244 { 3245 if (!active) 3246 irdma_add_conn_est_qh(cm_node); 3247 return 0; 3248 } 3249 3250 static void 3251 irdma_cm_free_ah_nop(struct irdma_cm_node *cm_node) 3252 { 3253 } 3254 3255 /** 3256 * irdma_setup_cm_core - setup top level instance of a cm core 3257 * @iwdev: iwarp device structure 3258 * @rdma_ver: HW version 3259 */ 3260 int 3261 irdma_setup_cm_core(struct irdma_device *iwdev, u8 rdma_ver) 3262 { 3263 struct irdma_cm_core *cm_core = &iwdev->cm_core; 3264 3265 cm_core->iwdev = iwdev; 3266 cm_core->dev = &iwdev->rf->sc_dev; 3267 3268 /* Handles CM event work items send to Iwarp core */ 3269 cm_core->event_wq = alloc_ordered_workqueue("iwarp-event-wq", 0); 3270 if (!cm_core->event_wq) 3271 return -ENOMEM; 3272 3273 INIT_LIST_HEAD(&cm_core->listen_list); 3274 3275 timer_setup(&cm_core->tcp_timer, irdma_cm_timer_tick, 0); 3276 3277 spin_lock_init(&cm_core->ht_lock); 3278 spin_lock_init(&cm_core->listen_list_lock); 3279 spin_lock_init(&cm_core->apbvt_lock); 3280 switch (rdma_ver) { 3281 case IRDMA_GEN_1: 3282 cm_core->form_cm_frame = irdma_form_uda_cm_frame; 3283 cm_core->cm_create_ah = irdma_add_qh; 3284 cm_core->cm_free_ah = irdma_cm_free_ah_nop; 3285 break; 3286 case IRDMA_GEN_2: 3287 default: 3288 cm_core->form_cm_frame = irdma_form_ah_cm_frame; 3289 cm_core->cm_create_ah = irdma_cm_create_ah; 3290 cm_core->cm_free_ah = irdma_cm_free_ah; 3291 } 3292 3293 return 0; 3294 } 3295 3296 /** 3297 * irdma_cleanup_cm_core - deallocate a top level instance of a 3298 * cm core 3299 * @cm_core: cm's core 3300 */ 3301 void 3302 irdma_cleanup_cm_core(struct irdma_cm_core *cm_core) 3303 { 3304 if (!cm_core) 3305 return; 3306 3307 del_timer_sync(&cm_core->tcp_timer); 3308 3309 destroy_workqueue(cm_core->event_wq); 3310 cm_core->dev->ws_reset(&cm_core->iwdev->vsi); 3311 } 3312 3313 /** 3314 * irdma_init_tcp_ctx - setup qp context 3315 * @cm_node: connection's node 3316 * @tcp_info: offload info for tcp 3317 * @iwqp: associate qp for the connection 3318 */ 3319 static void 3320 irdma_init_tcp_ctx(struct irdma_cm_node *cm_node, 3321 struct irdma_tcp_offload_info *tcp_info, 3322 struct irdma_qp *iwqp) 3323 { 3324 tcp_info->ipv4 = cm_node->ipv4; 3325 tcp_info->drop_ooo_seg = !iwqp->iwdev->iw_ooo; 3326 tcp_info->wscale = true; 3327 tcp_info->ignore_tcp_opt = true; 3328 tcp_info->ignore_tcp_uns_opt = true; 3329 tcp_info->no_nagle = false; 3330 3331 tcp_info->ttl = IRDMA_DEFAULT_TTL; 3332 tcp_info->rtt_var = IRDMA_DEFAULT_RTT_VAR; 3333 tcp_info->ss_thresh = IRDMA_DEFAULT_SS_THRESH; 3334 tcp_info->rexmit_thresh = IRDMA_DEFAULT_REXMIT_THRESH; 3335 3336 tcp_info->tcp_state = IRDMA_TCP_STATE_ESTABLISHED; 3337 tcp_info->snd_wscale = cm_node->tcp_cntxt.snd_wscale; 3338 tcp_info->rcv_wscale = cm_node->tcp_cntxt.rcv_wscale; 3339 3340 tcp_info->snd_nxt = cm_node->tcp_cntxt.loc_seq_num; 3341 tcp_info->snd_wnd = cm_node->tcp_cntxt.snd_wnd; 3342 tcp_info->rcv_nxt = cm_node->tcp_cntxt.rcv_nxt; 3343 tcp_info->snd_max = cm_node->tcp_cntxt.loc_seq_num; 3344 3345 tcp_info->snd_una = cm_node->tcp_cntxt.loc_seq_num; 3346 tcp_info->cwnd = 2 * cm_node->tcp_cntxt.mss; 3347 tcp_info->snd_wl1 = cm_node->tcp_cntxt.rcv_nxt; 3348 tcp_info->snd_wl2 = cm_node->tcp_cntxt.loc_seq_num; 3349 tcp_info->max_snd_window = cm_node->tcp_cntxt.max_snd_wnd; 3350 tcp_info->rcv_wnd = cm_node->tcp_cntxt.rcv_wnd 3351 << cm_node->tcp_cntxt.rcv_wscale; 3352 3353 tcp_info->flow_label = 0; 3354 tcp_info->snd_mss = (u32)cm_node->tcp_cntxt.mss; 3355 tcp_info->tos = cm_node->tos; 3356 if (cm_node->vlan_id < VLAN_N_VID) { 3357 tcp_info->insert_vlan_tag = true; 3358 tcp_info->vlan_tag = cm_node->vlan_id; 3359 tcp_info->vlan_tag |= cm_node->user_pri << VLAN_PRIO_SHIFT; 3360 } 3361 tcp_info->src_port = cm_node->loc_port; 3362 tcp_info->dst_port = cm_node->rem_port; 3363 tcp_info->arp_idx = (u16)irdma_arp_table(iwqp->iwdev->rf, 3364 cm_node->rem_addr, NULL, 3365 IRDMA_ARP_RESOLVE); 3366 if (cm_node->ipv4) { 3367 tcp_info->dest_ip_addr[3] = cm_node->rem_addr[0]; 3368 tcp_info->local_ipaddr[3] = cm_node->loc_addr[0]; 3369 } else { 3370 memcpy(tcp_info->dest_ip_addr, cm_node->rem_addr, 3371 sizeof(tcp_info->dest_ip_addr)); 3372 memcpy(tcp_info->local_ipaddr, cm_node->loc_addr, 3373 sizeof(tcp_info->local_ipaddr)); 3374 } 3375 } 3376 3377 /** 3378 * irdma_cm_init_tsa_conn - setup qp for RTS 3379 * @iwqp: associate qp for the connection 3380 * @cm_node: connection's node 3381 */ 3382 static void 3383 irdma_cm_init_tsa_conn(struct irdma_qp *iwqp, 3384 struct irdma_cm_node *cm_node) 3385 { 3386 struct irdma_iwarp_offload_info *iwarp_info; 3387 struct irdma_qp_host_ctx_info *ctx_info; 3388 3389 iwarp_info = &iwqp->iwarp_info; 3390 ctx_info = &iwqp->ctx_info; 3391 3392 ctx_info->tcp_info = &iwqp->tcp_info; 3393 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; 3394 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; 3395 3396 iwarp_info->ord_size = cm_node->ord_size; 3397 iwarp_info->ird_size = cm_node->ird_size; 3398 iwarp_info->rd_en = true; 3399 iwarp_info->rdmap_ver = 1; 3400 iwarp_info->ddp_ver = 1; 3401 iwarp_info->pd_id = iwqp->iwpd->sc_pd.pd_id; 3402 3403 ctx_info->tcp_info_valid = true; 3404 ctx_info->iwarp_info_valid = true; 3405 ctx_info->user_pri = cm_node->user_pri; 3406 3407 irdma_init_tcp_ctx(cm_node, &iwqp->tcp_info, iwqp); 3408 if (cm_node->snd_mark_en) { 3409 iwarp_info->snd_mark_en = true; 3410 iwarp_info->snd_mark_offset = (iwqp->tcp_info.snd_nxt & SNDMARKER_SEQNMASK) + 3411 cm_node->lsmm_size; 3412 } 3413 3414 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_OFFLOADED); 3415 iwqp->tcp_info.tcp_state = IRDMA_TCP_STATE_ESTABLISHED; 3416 iwqp->tcp_info.src_mac_addr_idx = iwqp->iwdev->mac_ip_table_idx; 3417 3418 if (cm_node->rcv_mark_en) { 3419 iwarp_info->rcv_mark_en = true; 3420 iwarp_info->align_hdrs = true; 3421 } 3422 3423 irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 3424 3425 /* once tcp_info is set, no need to do it again */ 3426 ctx_info->tcp_info_valid = false; 3427 ctx_info->iwarp_info_valid = false; 3428 } 3429 3430 /** 3431 * irdma_cm_disconn - when a connection is being closed 3432 * @iwqp: associated qp for the connection 3433 */ 3434 void 3435 irdma_cm_disconn(struct irdma_qp *iwqp) 3436 { 3437 struct irdma_device *iwdev = iwqp->iwdev; 3438 struct disconn_work *work; 3439 unsigned long flags; 3440 3441 work = kzalloc(sizeof(*work), GFP_ATOMIC); 3442 if (!work) 3443 return; 3444 3445 spin_lock_irqsave(&iwdev->rf->qptable_lock, flags); 3446 if (!iwdev->rf->qp_table[iwqp->ibqp.qp_num]) { 3447 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); 3448 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 3449 "qp_id %d is already freed\n", iwqp->ibqp.qp_num); 3450 kfree(work); 3451 return; 3452 } 3453 irdma_qp_add_ref(&iwqp->ibqp); 3454 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); 3455 3456 work->iwqp = iwqp; 3457 INIT_WORK(&work->work, irdma_disconnect_worker); 3458 queue_work(iwdev->cleanup_wq, &work->work); 3459 } 3460 3461 /** 3462 * irdma_qp_disconnect - free qp and close cm 3463 * @iwqp: associate qp for the connection 3464 */ 3465 static void 3466 irdma_qp_disconnect(struct irdma_qp *iwqp) 3467 { 3468 struct irdma_device *iwdev = iwqp->iwdev; 3469 3470 iwqp->active_conn = 0; 3471 /* close the CM node down if it is still active */ 3472 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "Call close API\n"); 3473 irdma_cm_close(iwqp->cm_node); 3474 } 3475 3476 static void 3477 dump_qp_ae_info(struct irdma_qp *iwqp) 3478 { 3479 struct irdma_device *iwdev = iwqp->iwdev; 3480 struct irdma_ae_info *ae_info = &iwdev->ae_info; 3481 u16 ae = iwqp->last_aeq; 3482 3483 if (!ae) 3484 return; 3485 3486 /* 3487 * When there is a hard link disconnect reduce prints to avoid slowing down qp cleanup. 3488 */ 3489 if (ae == IRDMA_AE_LLP_TOO_MANY_RETRIES) { 3490 unsigned long flags; 3491 u32 retry_cnt; 3492 3493 spin_lock_irqsave(&ae_info->info_lock, flags); 3494 ae_info->retry_cnt++; 3495 if (time_after(ae_info->retry_delay, jiffies)) { 3496 spin_unlock_irqrestore(&ae_info->info_lock, flags); 3497 return; 3498 } 3499 3500 retry_cnt = ae_info->retry_cnt; 3501 ae_info->retry_cnt = 0; 3502 ae_info->retry_delay = jiffies + 3503 msecs_to_jiffies(IRDMA_RETRY_PRINT_MS); 3504 spin_unlock_irqrestore(&ae_info->info_lock, flags); 3505 3506 irdma_dev_err(&iwdev->ibdev, 3507 "qp async event qp_id = %d, ae = 0x%x (%s), qp_cnt = %d\n", 3508 iwqp->sc_qp.qp_uk.qp_id, ae, irdma_get_ae_desc(ae), 3509 retry_cnt); 3510 3511 return; 3512 } 3513 switch (ae) { 3514 case IRDMA_AE_BAD_CLOSE: 3515 case IRDMA_AE_LLP_CLOSE_COMPLETE: 3516 case IRDMA_AE_LLP_CONNECTION_RESET: 3517 case IRDMA_AE_LLP_FIN_RECEIVED: 3518 case IRDMA_AE_LLP_SYN_RECEIVED: 3519 case IRDMA_AE_LLP_TERMINATE_RECEIVED: 3520 case IRDMA_AE_LLP_DOUBT_REACHABILITY: 3521 case IRDMA_AE_LLP_CONNECTION_ESTABLISHED: 3522 case IRDMA_AE_RESET_SENT: 3523 case IRDMA_AE_TERMINATE_SENT: 3524 case IRDMA_AE_RESET_NOT_SENT: 3525 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_AEQ, 3526 "qp async avent qp_id = %d, ae = 0x%x (%s), src = %d, ae_cnt = %d\n", 3527 iwqp->sc_qp.qp_uk.qp_id, ae, irdma_get_ae_desc(ae), 3528 iwqp->ae_src, atomic_read(&ae_info->ae_cnt)); 3529 break; 3530 default: 3531 irdma_dev_err(&iwdev->ibdev, 3532 "qp async event qp_id = %d, ae = 0x%x (%s), src = %d, ae_cnt = %d\n", 3533 iwqp->sc_qp.qp_uk.qp_id, ae, irdma_get_ae_desc(ae), 3534 iwqp->ae_src, atomic_read(&ae_info->ae_cnt)); 3535 } 3536 } 3537 3538 /** 3539 * irdma_cm_disconn_true - called by worker thread to disconnect qp 3540 * @iwqp: associate qp for the connection 3541 */ 3542 static void 3543 irdma_cm_disconn_true(struct irdma_qp *iwqp) 3544 { 3545 struct iw_cm_id *cm_id; 3546 struct irdma_device *iwdev; 3547 struct irdma_sc_qp *qp = &iwqp->sc_qp; 3548 u16 last_ae; 3549 u8 original_hw_tcp_state; 3550 u8 original_ibqp_state; 3551 int disconn_status = 0; 3552 int issue_disconn = 0; 3553 int issue_close = 0; 3554 int issue_flush = 0; 3555 unsigned long flags; 3556 int err; 3557 3558 iwdev = iwqp->iwdev; 3559 3560 dump_qp_ae_info(iwqp); 3561 spin_lock_irqsave(&iwqp->lock, flags); 3562 3563 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 3564 struct ib_qp_attr attr; 3565 3566 if (atomic_read(&iwqp->flush_issued) || 3567 iwqp->sc_qp.qp_uk.destroy_pending) { 3568 spin_unlock_irqrestore(&iwqp->lock, flags); 3569 return; 3570 } 3571 3572 spin_unlock_irqrestore(&iwqp->lock, flags); 3573 3574 attr.qp_state = IB_QPS_ERR; 3575 irdma_modify_qp_roce(&iwqp->ibqp, &attr, IB_QP_STATE, NULL); 3576 irdma_ib_qp_event(iwqp, qp->event_type); 3577 return; 3578 } 3579 3580 cm_id = iwqp->cm_id; 3581 original_hw_tcp_state = iwqp->hw_tcp_state; 3582 original_ibqp_state = iwqp->ibqp_state; 3583 last_ae = iwqp->last_aeq; 3584 3585 if (qp->term_flags) { 3586 issue_disconn = 1; 3587 issue_close = 1; 3588 iwqp->cm_id = NULL; 3589 irdma_terminate_del_timer(qp); 3590 if (!atomic_read(&iwqp->flush_issued)) 3591 issue_flush = 1; 3592 } else if ((original_hw_tcp_state == IRDMA_TCP_STATE_CLOSE_WAIT) || 3593 ((original_ibqp_state == IB_QPS_RTS) && 3594 (last_ae == IRDMA_AE_LLP_CONNECTION_RESET))) { 3595 issue_disconn = 1; 3596 if (last_ae == IRDMA_AE_LLP_CONNECTION_RESET) 3597 disconn_status = -ECONNRESET; 3598 } 3599 3600 if (original_hw_tcp_state == IRDMA_TCP_STATE_CLOSED || 3601 original_hw_tcp_state == IRDMA_TCP_STATE_TIME_WAIT || 3602 last_ae == IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE || 3603 last_ae == IRDMA_AE_BAD_CLOSE || 3604 last_ae == IRDMA_AE_LLP_CONNECTION_RESET || iwdev->rf->reset || !cm_id) { 3605 issue_close = 1; 3606 iwqp->cm_id = NULL; 3607 qp->term_flags = 0; 3608 if (!atomic_read(&iwqp->flush_issued)) 3609 issue_flush = 1; 3610 } 3611 3612 spin_unlock_irqrestore(&iwqp->lock, flags); 3613 if (issue_flush && !iwqp->sc_qp.qp_uk.destroy_pending) { 3614 irdma_flush_wqes(iwqp, IRDMA_FLUSH_SQ | IRDMA_FLUSH_RQ | 3615 IRDMA_FLUSH_WAIT); 3616 3617 if (qp->term_flags) 3618 irdma_ib_qp_event(iwqp, qp->event_type); 3619 } 3620 3621 if (!cm_id || !cm_id->event_handler) 3622 return; 3623 3624 spin_lock_irqsave(&iwdev->cm_core.ht_lock, flags); 3625 if (!iwqp->cm_node) { 3626 spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags); 3627 return; 3628 } 3629 irdma_add_ref_cmnode(iwqp->cm_node); 3630 3631 spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags); 3632 3633 if (issue_disconn) { 3634 err = irdma_send_cm_event(iwqp->cm_node, cm_id, 3635 IW_CM_EVENT_DISCONNECT, 3636 disconn_status); 3637 if (err) 3638 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 3639 "disconnect event failed: - cm_id = %p\n", 3640 cm_id); 3641 } 3642 if (issue_close) { 3643 cm_id->provider_data = iwqp; 3644 err = irdma_send_cm_event(iwqp->cm_node, cm_id, 3645 IW_CM_EVENT_CLOSE, 0); 3646 if (err) 3647 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 3648 "close event failed: - cm_id = %p\n", 3649 cm_id); 3650 irdma_qp_disconnect(iwqp); 3651 } 3652 irdma_rem_ref_cmnode(iwqp->cm_node); 3653 } 3654 3655 /** 3656 * irdma_disconnect_worker - worker for connection close 3657 * @work: points or disconn structure 3658 */ 3659 static void 3660 irdma_disconnect_worker(struct work_struct *work) 3661 { 3662 struct disconn_work *dwork = container_of(work, struct disconn_work, work); 3663 struct irdma_qp *iwqp = dwork->iwqp; 3664 3665 kfree(dwork); 3666 irdma_cm_disconn_true(iwqp); 3667 irdma_qp_rem_ref(&iwqp->ibqp); 3668 } 3669 3670 /** 3671 * irdma_free_lsmm_rsrc - free lsmm memory and deregister 3672 * @iwqp: associate qp for the connection 3673 */ 3674 void 3675 irdma_free_lsmm_rsrc(struct irdma_qp *iwqp) 3676 { 3677 struct irdma_device *iwdev; 3678 3679 iwdev = iwqp->iwdev; 3680 3681 if (iwqp->ietf_mem.va) { 3682 if (iwqp->lsmm_mr) 3683 iwdev->ibdev.dereg_mr(iwqp->lsmm_mr, NULL); 3684 3685 irdma_free_dma_mem(iwdev->rf->sc_dev.hw, 3686 &iwqp->ietf_mem); 3687 iwqp->ietf_mem.va = NULL; 3688 } 3689 } 3690 3691 /** 3692 * irdma_accept - registered call for connection to be accepted 3693 * @cm_id: cm information for passive connection 3694 * @conn_param: accpet parameters 3695 */ 3696 int 3697 irdma_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 3698 { 3699 struct ib_qp *ibqp; 3700 struct irdma_qp *iwqp; 3701 struct irdma_device *iwdev; 3702 struct irdma_sc_dev *dev; 3703 struct irdma_cm_node *cm_node; 3704 struct ib_qp_attr attr = {0}; 3705 int passive_state; 3706 struct ib_mr *ibmr; 3707 struct irdma_pd *iwpd; 3708 u16 buf_len = 0; 3709 struct irdma_kmem_info accept; 3710 u64 tagged_offset; 3711 int wait_ret; 3712 int ret = 0; 3713 3714 ibqp = irdma_get_qp(cm_id->device, conn_param->qpn); 3715 if (!ibqp) 3716 return -EINVAL; 3717 3718 iwqp = to_iwqp(ibqp); 3719 iwdev = iwqp->iwdev; 3720 dev = &iwdev->rf->sc_dev; 3721 cm_node = cm_id->provider_data; 3722 3723 if (((struct sockaddr_in *)&cm_id->local_addr)->sin_family == AF_INET) { 3724 cm_node->ipv4 = true; 3725 cm_node->vlan_id = irdma_get_vlan_ipv4(cm_id, cm_node->loc_addr); 3726 } else { 3727 cm_node->ipv4 = false; 3728 irdma_get_vlan_mac_ipv6(cm_id, cm_node->loc_addr, &cm_node->vlan_id, 3729 NULL); 3730 } 3731 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "Accept vlan_id=%d\n", 3732 cm_node->vlan_id); 3733 3734 if (cm_node->state == IRDMA_CM_STATE_LISTENER_DESTROYED) { 3735 ret = -EINVAL; 3736 goto error; 3737 } 3738 3739 passive_state = atomic_add_return(1, &cm_node->passive_state); 3740 if (passive_state == IRDMA_SEND_RESET_EVENT) { 3741 ret = -ECONNRESET; 3742 goto error; 3743 } 3744 3745 buf_len = conn_param->private_data_len + IRDMA_MAX_IETF_SIZE; 3746 iwqp->ietf_mem.size = buf_len; 3747 iwqp->ietf_mem.va = irdma_allocate_dma_mem(dev->hw, &iwqp->ietf_mem, 3748 iwqp->ietf_mem.size, 1); 3749 if (!iwqp->ietf_mem.va) { 3750 ret = -ENOMEM; 3751 goto error; 3752 } 3753 3754 cm_node->pdata.size = conn_param->private_data_len; 3755 accept.addr = iwqp->ietf_mem.va; 3756 accept.size = irdma_cm_build_mpa_frame(cm_node, &accept, MPA_KEY_REPLY); 3757 memcpy((u8 *)accept.addr + accept.size, conn_param->private_data, 3758 conn_param->private_data_len); 3759 3760 if (cm_node->dev->ws_add(iwqp->sc_qp.vsi, cm_node->user_pri)) { 3761 ret = -ENOMEM; 3762 goto error; 3763 } 3764 iwqp->sc_qp.user_pri = cm_node->user_pri; 3765 irdma_qp_add_qos(&iwqp->sc_qp); 3766 if (cm_node->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_2) 3767 iwdev->rf->check_fc(&iwdev->vsi, &iwqp->sc_qp); 3768 /* setup our first outgoing iWarp send WQE (the IETF frame response) */ 3769 iwpd = iwqp->iwpd; 3770 tagged_offset = (uintptr_t)iwqp->ietf_mem.va; 3771 ibmr = irdma_reg_phys_mr(&iwpd->ibpd, iwqp->ietf_mem.pa, buf_len, 3772 IB_ACCESS_LOCAL_WRITE, &tagged_offset, false); 3773 if (IS_ERR(ibmr)) { 3774 ret = -ENOMEM; 3775 goto error; 3776 } 3777 3778 ibmr->pd = &iwpd->ibpd; 3779 ibmr->device = iwpd->ibpd.device; 3780 iwqp->lsmm_mr = ibmr; 3781 if (iwqp->page) 3782 iwqp->sc_qp.qp_uk.sq_base = kmap_local_page(iwqp->page); 3783 3784 cm_node->lsmm_size = accept.size + conn_param->private_data_len; 3785 irdma_sc_send_lsmm(&iwqp->sc_qp, iwqp->ietf_mem.va, cm_node->lsmm_size, 3786 ibmr->lkey); 3787 3788 if (iwqp->page) 3789 kunmap_local(iwqp->sc_qp.qp_uk.sq_base); 3790 3791 iwqp->cm_id = cm_id; 3792 cm_node->cm_id = cm_id; 3793 3794 cm_id->provider_data = iwqp; 3795 iwqp->active_conn = 0; 3796 iwqp->cm_node = cm_node; 3797 cm_node->iwqp = iwqp; 3798 irdma_cm_init_tsa_conn(iwqp, cm_node); 3799 irdma_qp_add_ref(&iwqp->ibqp); 3800 cm_id->add_ref(cm_id); 3801 3802 attr.qp_state = IB_QPS_RTS; 3803 cm_node->qhash_set = false; 3804 cm_node->cm_core->cm_free_ah(cm_node); 3805 3806 irdma_modify_qp(&iwqp->ibqp, &attr, IB_QP_STATE, NULL); 3807 if (dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE) { 3808 wait_ret = wait_event_interruptible_timeout(iwqp->waitq, 3809 iwqp->rts_ae_rcvd, 3810 IRDMA_MAX_TIMEOUT); 3811 if (!wait_ret) { 3812 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 3813 "Slow Connection: cm_node=%p, loc_port=%d, rem_port=%d, cm_id=%p\n", 3814 cm_node, cm_node->loc_port, 3815 cm_node->rem_port, cm_node->cm_id); 3816 ret = -ECONNRESET; 3817 goto error; 3818 } 3819 } 3820 3821 irdma_send_cm_event(cm_node, cm_id, IW_CM_EVENT_ESTABLISHED, 0); 3822 cm_node->accelerated = true; 3823 complete(&cm_node->establish_comp); 3824 3825 if (cm_node->accept_pend) { 3826 atomic_dec(&cm_node->listener->pend_accepts_cnt); 3827 cm_node->accept_pend = 0; 3828 } 3829 3830 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 3831 "rem_port=0x%04x, loc_port=0x%04x rem_addr=%x loc_addr=%x cm_node=%p cm_id=%p qp_id=%d\n\n", 3832 cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr[0], 3833 cm_node->loc_addr[0], cm_node, cm_id, ibqp->qp_num); 3834 cm_node->cm_core->stats_accepts++; 3835 3836 return 0; 3837 error: 3838 irdma_free_lsmm_rsrc(iwqp); 3839 irdma_rem_ref_cmnode(cm_node); 3840 3841 return ret; 3842 } 3843 3844 /** 3845 * irdma_reject - registered call for connection to be rejected 3846 * @cm_id: cm information for passive connection 3847 * @pdata: private data to be sent 3848 * @pdata_len: private data length 3849 */ 3850 int 3851 irdma_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) 3852 { 3853 struct irdma_device *iwdev; 3854 struct irdma_cm_node *cm_node; 3855 3856 cm_node = cm_id->provider_data; 3857 cm_node->pdata.size = pdata_len; 3858 3859 iwdev = to_iwdev(cm_id->device); 3860 if (!iwdev) 3861 return -EINVAL; 3862 3863 cm_node->cm_core->stats_rejects++; 3864 3865 if (pdata_len + sizeof(struct ietf_mpa_v2) > IRDMA_MAX_CM_BUF) 3866 return -EINVAL; 3867 3868 return irdma_cm_reject(cm_node, pdata, pdata_len); 3869 } 3870 3871 /** 3872 * irdma_connect - registered call for connection to be established 3873 * @cm_id: cm information for passive connection 3874 * @conn_param: Information about the connection 3875 */ 3876 int 3877 irdma_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 3878 { 3879 struct ib_qp *ibqp; 3880 struct irdma_qp *iwqp; 3881 struct irdma_device *iwdev; 3882 struct irdma_cm_node *cm_node; 3883 struct irdma_cm_info cm_info; 3884 struct sockaddr_in *laddr; 3885 struct sockaddr_in *raddr; 3886 struct sockaddr_in6 *laddr6; 3887 struct sockaddr_in6 *raddr6; 3888 int ret = 0; 3889 3890 ibqp = irdma_get_qp(cm_id->device, conn_param->qpn); 3891 if (!ibqp) 3892 return -EINVAL; 3893 iwqp = to_iwqp(ibqp); 3894 if (!iwqp) 3895 return -EINVAL; 3896 iwdev = iwqp->iwdev; 3897 if (!iwdev) 3898 return -EINVAL; 3899 3900 laddr = (struct sockaddr_in *)&cm_id->m_local_addr; 3901 raddr = (struct sockaddr_in *)&cm_id->m_remote_addr; 3902 laddr6 = (struct sockaddr_in6 *)&cm_id->m_local_addr; 3903 raddr6 = (struct sockaddr_in6 *)&cm_id->m_remote_addr; 3904 3905 if (!(laddr->sin_port) || !(raddr->sin_port)) 3906 return -EINVAL; 3907 3908 iwqp->active_conn = 1; 3909 iwqp->cm_id = NULL; 3910 cm_id->provider_data = iwqp; 3911 3912 /* set up the connection params for the node */ 3913 if (cm_id->remote_addr.ss_family == AF_INET) { 3914 if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV4) 3915 return -EINVAL; 3916 3917 cm_info.ipv4 = true; 3918 memset(cm_info.loc_addr, 0, sizeof(cm_info.loc_addr)); 3919 memset(cm_info.rem_addr, 0, sizeof(cm_info.rem_addr)); 3920 cm_info.loc_addr[0] = ntohl(laddr->sin_addr.s_addr); 3921 cm_info.rem_addr[0] = ntohl(raddr->sin_addr.s_addr); 3922 cm_info.loc_port = ntohs(laddr->sin_port); 3923 cm_info.rem_port = ntohs(raddr->sin_port); 3924 cm_info.vlan_id = irdma_get_vlan_ipv4(cm_id, cm_info.loc_addr); 3925 } else { 3926 if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV6) 3927 return -EINVAL; 3928 3929 cm_info.ipv4 = false; 3930 irdma_copy_ip_ntohl(cm_info.loc_addr, 3931 laddr6->sin6_addr.__u6_addr.__u6_addr32); 3932 irdma_copy_ip_ntohl(cm_info.rem_addr, 3933 raddr6->sin6_addr.__u6_addr.__u6_addr32); 3934 cm_info.loc_port = ntohs(laddr6->sin6_port); 3935 cm_info.rem_port = ntohs(raddr6->sin6_port); 3936 irdma_get_vlan_mac_ipv6(cm_id, cm_info.loc_addr, &cm_info.vlan_id, NULL); 3937 } 3938 cm_info.cm_id = cm_id; 3939 cm_info.qh_qpid = iwdev->vsi.ilq->qp_id; 3940 cm_info.tos = cm_id->tos; 3941 if (iwdev->vsi.dscp_mode) { 3942 cm_info.user_pri = 3943 iwqp->sc_qp.vsi->dscp_map[irdma_tos2dscp(cm_info.tos)]; 3944 } else { 3945 cm_info.user_pri = rt_tos2priority(cm_id->tos); 3946 cm_info.user_pri = 3947 irdma_iw_get_vlan_prio(cm_info.loc_addr, 3948 cm_info.user_pri, 3949 cm_info.ipv4); 3950 } 3951 3952 if (iwqp->sc_qp.dev->ws_add(iwqp->sc_qp.vsi, cm_info.user_pri)) 3953 return -ENOMEM; 3954 iwqp->sc_qp.user_pri = cm_info.user_pri; 3955 irdma_qp_add_qos(&iwqp->sc_qp); 3956 if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_2) 3957 iwdev->rf->check_fc(&iwdev->vsi, &iwqp->sc_qp); 3958 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DCB, "TOS:[%d] UP:[%d]\n", 3959 cm_id->tos, cm_info.user_pri); 3960 3961 ret = irdma_create_cm_node(&iwdev->cm_core, iwdev, conn_param, &cm_info, 3962 &cm_node); 3963 if (ret) 3964 return ret; 3965 ret = cm_node->cm_core->cm_create_ah(cm_node, true); 3966 if (ret) 3967 goto err; 3968 if (irdma_manage_qhash(iwdev, &cm_info, 3969 IRDMA_QHASH_TYPE_TCP_ESTABLISHED, 3970 IRDMA_QHASH_MANAGE_TYPE_ADD, NULL, true)) { 3971 ret = -EINVAL; 3972 goto err; 3973 } 3974 cm_node->qhash_set = true; 3975 3976 cm_node->apbvt_entry = irdma_add_apbvt(iwdev, cm_info.loc_port); 3977 if (!cm_node->apbvt_entry) { 3978 ret = -EINVAL; 3979 goto err; 3980 } 3981 3982 cm_node->apbvt_set = true; 3983 iwqp->cm_node = cm_node; 3984 cm_node->iwqp = iwqp; 3985 iwqp->cm_id = cm_id; 3986 irdma_qp_add_ref(&iwqp->ibqp); 3987 cm_id->add_ref(cm_id); 3988 3989 if (!irdma_cm_node_cmp_state(cm_node, IRDMA_CM_STATE_OFFLOADED)) { 3990 irdma_cm_node_set_state(cm_node, IRDMA_CM_STATE_SYN_SENT); 3991 ret = irdma_send_syn(cm_node, 0); 3992 if (ret) 3993 goto err; 3994 } 3995 3996 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 3997 "rem_port=0x%04x, loc_port=0x%04x rem_addr=%x loc_addr=%x cm_node=%p cm_id=%p qp_id = %d\n\n", 3998 cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr[0], 3999 cm_node->loc_addr[0], cm_node, cm_id, ibqp->qp_num); 4000 4001 return 0; 4002 4003 err: 4004 if (cm_info.ipv4) 4005 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 4006 "connect() FAILED: dest addr=%x", 4007 cm_info.rem_addr[0]); 4008 else 4009 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 4010 "connect() FAILED: dest addr=%x:%x:%x:%x", 4011 IRDMA_PRINT_IP6(cm_info.rem_addr)); 4012 irdma_rem_ref_cmnode(cm_node); 4013 iwdev->cm_core.stats_connect_errs++; 4014 4015 return ret; 4016 } 4017 4018 /** 4019 * irdma_create_listen - registered call creating listener 4020 * @cm_id: cm information for passive connection 4021 * @backlog: to max accept pending count 4022 */ 4023 int 4024 irdma_create_listen(struct iw_cm_id *cm_id, int backlog) 4025 { 4026 struct irdma_device *iwdev; 4027 struct irdma_cm_listener *cm_listen_node; 4028 struct irdma_cm_info cm_info = {0}; 4029 struct sockaddr_in *laddr; 4030 struct sockaddr_in6 *laddr6; 4031 bool wildcard = false; 4032 int err; 4033 4034 iwdev = to_iwdev(cm_id->device); 4035 if (!iwdev) 4036 return -EINVAL; 4037 4038 laddr = (struct sockaddr_in *)&cm_id->m_local_addr; 4039 laddr6 = (struct sockaddr_in6 *)&cm_id->m_local_addr; 4040 cm_info.qh_qpid = iwdev->vsi.ilq->qp_id; 4041 4042 if (laddr->sin_family == AF_INET) { 4043 if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV4) 4044 return -EINVAL; 4045 4046 cm_info.ipv4 = true; 4047 cm_info.loc_addr[0] = ntohl(laddr->sin_addr.s_addr); 4048 cm_info.loc_port = ntohs(laddr->sin_port); 4049 4050 if (laddr->sin_addr.s_addr != htonl(INADDR_ANY)) { 4051 cm_info.vlan_id = irdma_get_vlan_ipv4(cm_id, cm_info.loc_addr); 4052 } else { 4053 cm_info.vlan_id = 0xFFFF; 4054 wildcard = true; 4055 } 4056 } else { 4057 if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV6) 4058 return -EINVAL; 4059 4060 cm_info.ipv4 = false; 4061 irdma_copy_ip_ntohl(cm_info.loc_addr, 4062 laddr6->sin6_addr.__u6_addr.__u6_addr32); 4063 cm_info.loc_port = ntohs(laddr6->sin6_port); 4064 if (!IN6_IS_ADDR_UNSPECIFIED(&laddr6->sin6_addr)) { 4065 irdma_get_vlan_mac_ipv6(cm_id, cm_info.loc_addr, 4066 &cm_info.vlan_id, NULL); 4067 } else { 4068 cm_info.vlan_id = 0xFFFF; 4069 wildcard = true; 4070 } 4071 } 4072 4073 if (cm_info.vlan_id >= VLAN_N_VID && iwdev->dcb_vlan_mode) 4074 cm_info.vlan_id = 0; 4075 cm_info.backlog = backlog; 4076 cm_info.cm_id = cm_id; 4077 4078 cm_listen_node = irdma_make_listen_node(&iwdev->cm_core, iwdev, 4079 &cm_info); 4080 if (!cm_listen_node) { 4081 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 4082 "cm_listen_node == NULL\n"); 4083 return -ENOMEM; 4084 } 4085 4086 cm_id->provider_data = cm_listen_node; 4087 4088 cm_listen_node->tos = cm_id->tos; 4089 if (iwdev->vsi.dscp_mode) 4090 cm_listen_node->user_pri = 4091 iwdev->vsi.dscp_map[irdma_tos2dscp(cm_id->tos)]; 4092 else 4093 cm_listen_node->user_pri = rt_tos2priority(cm_id->tos); 4094 cm_info.user_pri = cm_listen_node->user_pri; 4095 if (!cm_listen_node->reused_node) { 4096 if (wildcard) { 4097 err = irdma_add_mqh(iwdev, &cm_info, cm_listen_node); 4098 if (err) 4099 goto error; 4100 } else { 4101 if (!iwdev->vsi.dscp_mode) 4102 cm_info.user_pri = cm_listen_node->user_pri = 4103 irdma_iw_get_vlan_prio(cm_info.loc_addr, 4104 cm_info.user_pri, 4105 cm_info.ipv4); 4106 err = irdma_manage_qhash(iwdev, &cm_info, 4107 IRDMA_QHASH_TYPE_TCP_SYN, 4108 IRDMA_QHASH_MANAGE_TYPE_ADD, 4109 NULL, true); 4110 if (err) 4111 goto error; 4112 4113 cm_listen_node->qhash_set = true; 4114 } 4115 4116 cm_listen_node->apbvt_entry = irdma_add_apbvt(iwdev, 4117 cm_info.loc_port); 4118 if (!cm_listen_node->apbvt_entry) 4119 goto error; 4120 } 4121 cm_id->add_ref(cm_id); 4122 cm_listen_node->cm_core->stats_listen_created++; 4123 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 4124 "loc_port=0x%04x loc_addr=%x cm_listen_node=%p cm_id=%p qhash_set=%d vlan_id=%d\n", 4125 cm_listen_node->loc_port, cm_listen_node->loc_addr[0], 4126 cm_listen_node, cm_listen_node->cm_id, 4127 cm_listen_node->qhash_set, cm_listen_node->vlan_id); 4128 4129 return 0; 4130 4131 error: 4132 4133 irdma_cm_del_listen(&iwdev->cm_core, cm_listen_node, false); 4134 4135 return -EINVAL; 4136 } 4137 4138 /** 4139 * irdma_destroy_listen - registered call to destroy listener 4140 * @cm_id: cm information for passive connection 4141 */ 4142 int 4143 irdma_destroy_listen(struct iw_cm_id *cm_id) 4144 { 4145 struct irdma_device *iwdev; 4146 4147 iwdev = to_iwdev(cm_id->device); 4148 if (cm_id->provider_data) 4149 irdma_cm_del_listen(&iwdev->cm_core, cm_id->provider_data, 4150 true); 4151 else 4152 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 4153 "cm_id->provider_data was NULL\n"); 4154 4155 cm_id->rem_ref(cm_id); 4156 4157 return 0; 4158 } 4159 4160 /** 4161 * irdma_iw_teardown_list_prep - add conn nodes slated for tear 4162 * down to list 4163 * @cm_core: cm's core 4164 * @teardown_list: a list to which cm_node will be selected 4165 * @ipaddr: pointer to ip address 4166 * @nfo: pointer to cm_info structure instance 4167 * @disconnect_all: flag indicating disconnect all QPs 4168 */ 4169 static void 4170 irdma_iw_teardown_list_prep(struct irdma_cm_core *cm_core, 4171 struct list_head *teardown_list, 4172 u32 *ipaddr, 4173 struct irdma_cm_info *nfo, 4174 bool disconnect_all) 4175 { 4176 struct irdma_cm_node *cm_node; 4177 int bkt; 4178 4179 HASH_FOR_EACH_RCU(cm_core->cm_hash_tbl, bkt, cm_node, list) { 4180 if ((disconnect_all || 4181 (nfo->vlan_id == cm_node->vlan_id && 4182 !memcmp(cm_node->loc_addr, ipaddr, nfo->ipv4 ? 4 : 16))) && 4183 irdma_add_ref_cmnode(cm_node)) 4184 list_add(&cm_node->teardown_entry, teardown_list); 4185 } 4186 } 4187 4188 static inline bool 4189 irdma_ip_vlan_match(u32 *ip1, u16 vlan_id1, 4190 bool check_vlan, u32 *ip2, 4191 u16 vlan_id2, bool ipv4) 4192 { 4193 return (!check_vlan || vlan_id1 == vlan_id2) && 4194 !memcmp(ip1, ip2, ipv4 ? 4 : 16); 4195 } 4196 4197 /** 4198 * irdma_roce_teardown_list_prep - add conn nodes slated for 4199 * tear down to list 4200 * @iwdev: RDMA device 4201 * @teardown_list: a list to which cm_node will be selected 4202 * @ipaddr: pointer to ip address 4203 * @nfo: pointer to cm_info structure instance 4204 * @disconnect_all: flag indicating disconnect all QPs 4205 */ 4206 static void 4207 irdma_roce_teardown_list_prep(struct irdma_device *iwdev, 4208 struct list_head *teardown_list, 4209 u32 *ipaddr, 4210 struct irdma_cm_info *nfo, 4211 bool disconnect_all) 4212 { 4213 struct irdma_sc_vsi *vsi = &iwdev->vsi; 4214 struct irdma_sc_qp *sc_qp; 4215 struct list_head *list_node; 4216 struct irdma_qp *qp; 4217 unsigned long flags; 4218 int i; 4219 4220 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) { 4221 mutex_lock(&vsi->qos[i].qos_mutex); 4222 list_for_each(list_node, &vsi->qos[i].qplist) { 4223 u32 qp_ip[4]; 4224 4225 sc_qp = container_of(list_node, struct irdma_sc_qp, 4226 list); 4227 if (sc_qp->qp_uk.qp_type != IRDMA_QP_TYPE_ROCE_RC) 4228 continue; 4229 4230 qp = sc_qp->qp_uk.back_qp; 4231 if (!disconnect_all) { 4232 if (nfo->ipv4) 4233 qp_ip[0] = qp->udp_info.local_ipaddr[3]; 4234 else 4235 memcpy(qp_ip, 4236 &qp->udp_info.local_ipaddr[0], 4237 sizeof(qp_ip)); 4238 } 4239 4240 if (disconnect_all || 4241 irdma_ip_vlan_match(qp_ip, 4242 qp->udp_info.vlan_tag & EVL_VLID_MASK, 4243 qp->udp_info.insert_vlan_tag, 4244 ipaddr, nfo->vlan_id, nfo->ipv4)) { 4245 spin_lock_irqsave(&iwdev->rf->qptable_lock, flags); 4246 if (iwdev->rf->qp_table[sc_qp->qp_uk.qp_id]) { 4247 irdma_qp_add_ref(&qp->ibqp); 4248 list_add(&qp->teardown_entry, teardown_list); 4249 } 4250 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); 4251 } 4252 } 4253 mutex_unlock(&vsi->qos[i].qos_mutex); 4254 } 4255 } 4256 4257 /** 4258 * irdma_cm_event_connected - handle connected active node 4259 * @event: the info for cm_node of connection 4260 */ 4261 static void 4262 irdma_cm_event_connected(struct irdma_cm_event *event) 4263 { 4264 struct irdma_qp *iwqp; 4265 struct irdma_device *iwdev; 4266 struct irdma_cm_node *cm_node; 4267 struct irdma_sc_dev *dev; 4268 struct ib_qp_attr attr = {0}; 4269 struct iw_cm_id *cm_id; 4270 int status; 4271 bool read0; 4272 int wait_ret = 0; 4273 4274 cm_node = event->cm_node; 4275 cm_id = cm_node->cm_id; 4276 iwqp = cm_id->provider_data; 4277 iwdev = iwqp->iwdev; 4278 dev = &iwdev->rf->sc_dev; 4279 if (iwqp->sc_qp.qp_uk.destroy_pending) { 4280 status = -ETIMEDOUT; 4281 goto error; 4282 } 4283 4284 irdma_cm_init_tsa_conn(iwqp, cm_node); 4285 read0 = (cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO); 4286 if (iwqp->page) 4287 iwqp->sc_qp.qp_uk.sq_base = kmap_local_page(iwqp->page); 4288 irdma_sc_send_rtt(&iwqp->sc_qp, read0); 4289 if (iwqp->page) 4290 kunmap_local(iwqp->sc_qp.qp_uk.sq_base); 4291 4292 attr.qp_state = IB_QPS_RTS; 4293 cm_node->qhash_set = false; 4294 irdma_modify_qp(&iwqp->ibqp, &attr, IB_QP_STATE, NULL); 4295 if (dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE) { 4296 wait_ret = wait_event_interruptible_timeout(iwqp->waitq, 4297 iwqp->rts_ae_rcvd, 4298 IRDMA_MAX_TIMEOUT); 4299 if (!wait_ret) 4300 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 4301 "Slow Connection: cm_node=%p, loc_port=%d, rem_port=%d, cm_id=%p\n", 4302 cm_node, cm_node->loc_port, 4303 cm_node->rem_port, cm_node->cm_id); 4304 } 4305 4306 irdma_send_cm_event(cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY, 0); 4307 cm_node->accelerated = true; 4308 complete(&cm_node->establish_comp); 4309 cm_node->cm_core->cm_free_ah(cm_node); 4310 return; 4311 4312 error: 4313 iwqp->cm_id = NULL; 4314 cm_id->provider_data = NULL; 4315 irdma_send_cm_event(event->cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY, 4316 status); 4317 irdma_rem_ref_cmnode(event->cm_node); 4318 } 4319 4320 /** 4321 * irdma_cm_event_reset - handle reset 4322 * @event: the info for cm_node of connection 4323 */ 4324 static void 4325 irdma_cm_event_reset(struct irdma_cm_event *event) 4326 { 4327 struct irdma_cm_node *cm_node = event->cm_node; 4328 struct iw_cm_id *cm_id = cm_node->cm_id; 4329 struct irdma_qp *iwqp; 4330 4331 if (!cm_id) 4332 return; 4333 4334 iwqp = cm_id->provider_data; 4335 if (!iwqp) 4336 return; 4337 4338 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 4339 "reset event %p - cm_id = %p\n", event->cm_node, cm_id); 4340 iwqp->cm_id = NULL; 4341 4342 irdma_send_cm_event(cm_node, cm_node->cm_id, IW_CM_EVENT_DISCONNECT, 4343 -ECONNRESET); 4344 irdma_send_cm_event(cm_node, cm_node->cm_id, IW_CM_EVENT_CLOSE, 0); 4345 } 4346 4347 /** 4348 * irdma_cm_event_handler - send event to cm upper layer 4349 * @work: pointer of cm event info. 4350 */ 4351 static void 4352 irdma_cm_event_handler(struct work_struct *work) 4353 { 4354 struct irdma_cm_event *event = container_of(work, struct irdma_cm_event, event_work); 4355 struct irdma_cm_node *cm_node; 4356 4357 if (!event || !event->cm_node || !event->cm_node->cm_core) 4358 return; 4359 4360 cm_node = event->cm_node; 4361 4362 switch (event->type) { 4363 case IRDMA_CM_EVENT_MPA_REQ: 4364 irdma_send_cm_event(cm_node, cm_node->cm_id, 4365 IW_CM_EVENT_CONNECT_REQUEST, 0); 4366 break; 4367 case IRDMA_CM_EVENT_RESET: 4368 irdma_cm_event_reset(event); 4369 break; 4370 case IRDMA_CM_EVENT_CONNECTED: 4371 if (!event->cm_node->cm_id || 4372 !irdma_cm_node_cmp_state(cm_node, IRDMA_CM_STATE_OFFLOADED)) 4373 break; 4374 irdma_cm_event_connected(event); 4375 break; 4376 case IRDMA_CM_EVENT_MPA_REJECT: 4377 if (!event->cm_node->cm_id || 4378 irdma_cm_node_cmp_state(cm_node, IRDMA_CM_STATE_OFFLOADED)) 4379 break; 4380 irdma_send_cm_event(cm_node, cm_node->cm_id, 4381 IW_CM_EVENT_CONNECT_REPLY, -ECONNREFUSED); 4382 break; 4383 case IRDMA_CM_EVENT_ABORTED: 4384 if (!event->cm_node->cm_id || 4385 irdma_cm_node_cmp_state(cm_node, IRDMA_CM_STATE_OFFLOADED)) 4386 break; 4387 irdma_event_connect_error(event); 4388 break; 4389 default: 4390 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 4391 "bad event type = %d\n", event->type); 4392 break; 4393 } 4394 4395 irdma_rem_ref_cmnode(cm_node); 4396 kfree(event); 4397 } 4398 4399 /** 4400 * irdma_cm_post_event - queue event request for worker thread 4401 * @event: cm node's info for up event call 4402 */ 4403 static void 4404 irdma_cm_post_event(struct irdma_cm_event *event) 4405 { 4406 irdma_add_ref_cmnode(event->cm_node); 4407 INIT_WORK(&event->event_work, irdma_cm_event_handler); 4408 queue_work(event->cm_node->cm_core->event_wq, &event->event_work); 4409 } 4410 4411 /** 4412 * irdma_cm_teardown_connections - teardown QPs 4413 * @iwdev: device pointer 4414 * @ipaddr: Pointer to IPv4 or IPv6 address 4415 * @nfo: Connection info 4416 * @disconnect_all: flag indicating disconnect all QPs 4417 * 4418 * teardown QPs where source or destination addr matches ip addr 4419 */ 4420 static void __unused 4421 irdma_cm_teardown_connections(struct irdma_device *iwdev, 4422 u32 *ipaddr, 4423 struct irdma_cm_info *nfo, 4424 bool disconnect_all) 4425 { 4426 struct irdma_cm_core *cm_core = &iwdev->cm_core; 4427 struct list_head *list_core_temp; 4428 struct list_head *list_node; 4429 struct irdma_cm_node *cm_node; 4430 struct list_head teardown_list; 4431 struct ib_qp_attr attr; 4432 struct irdma_qp *qp; 4433 4434 INIT_LIST_HEAD(&teardown_list); 4435 4436 rcu_read_lock(); 4437 irdma_iw_teardown_list_prep(cm_core, &teardown_list, ipaddr, nfo, disconnect_all); 4438 rcu_read_unlock(); 4439 4440 attr.qp_state = IB_QPS_ERR; 4441 list_for_each_safe(list_node, list_core_temp, &teardown_list) { 4442 cm_node = container_of(list_node, struct irdma_cm_node, 4443 teardown_entry); 4444 irdma_modify_qp(&cm_node->iwqp->ibqp, &attr, IB_QP_STATE, NULL); 4445 if (iwdev->rf->reset) 4446 irdma_cm_disconn(cm_node->iwqp); 4447 irdma_rem_ref_cmnode(cm_node); 4448 } 4449 4450 if (!rdma_protocol_roce(&iwdev->ibdev, 1)) 4451 return; 4452 4453 INIT_LIST_HEAD(&teardown_list); 4454 irdma_roce_teardown_list_prep(iwdev, &teardown_list, ipaddr, nfo, disconnect_all); 4455 4456 list_for_each_safe(list_node, list_core_temp, &teardown_list) { 4457 qp = container_of(list_node, struct irdma_qp, teardown_entry); 4458 irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL); 4459 irdma_ib_qp_event(qp, IRDMA_QP_EVENT_CATASTROPHIC); 4460 irdma_qp_rem_ref(&qp->ibqp); 4461 } 4462 } 4463