1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2015 - 2021 Intel Corporation */ 3 #include "main.h" 4 5 /** 6 * irdma_arp_table -manage arp table 7 * @rf: RDMA PCI function 8 * @ip_addr: ip address for device 9 * @ipv4: IPv4 flag 10 * @mac_addr: mac address ptr 11 * @action: modify, delete or add 12 */ 13 int irdma_arp_table(struct irdma_pci_f *rf, u32 *ip_addr, bool ipv4, 14 const u8 *mac_addr, u32 action) 15 { 16 unsigned long flags; 17 int arp_index; 18 u32 ip[4] = {}; 19 20 if (ipv4) 21 ip[0] = *ip_addr; 22 else 23 memcpy(ip, ip_addr, sizeof(ip)); 24 25 spin_lock_irqsave(&rf->arp_lock, flags); 26 for (arp_index = 0; (u32)arp_index < rf->arp_table_size; arp_index++) { 27 if (!memcmp(rf->arp_table[arp_index].ip_addr, ip, sizeof(ip))) 28 break; 29 } 30 31 switch (action) { 32 case IRDMA_ARP_ADD: 33 if (arp_index != rf->arp_table_size) { 34 arp_index = -1; 35 break; 36 } 37 38 arp_index = 0; 39 if (irdma_alloc_rsrc(rf, rf->allocated_arps, rf->arp_table_size, 40 (u32 *)&arp_index, &rf->next_arp_index)) { 41 arp_index = -1; 42 break; 43 } 44 45 memcpy(rf->arp_table[arp_index].ip_addr, ip, 46 sizeof(rf->arp_table[arp_index].ip_addr)); 47 ether_addr_copy(rf->arp_table[arp_index].mac_addr, mac_addr); 48 break; 49 case IRDMA_ARP_RESOLVE: 50 if (arp_index == rf->arp_table_size) 51 arp_index = -1; 52 break; 53 case IRDMA_ARP_DELETE: 54 if (arp_index == rf->arp_table_size) { 55 arp_index = -1; 56 break; 57 } 58 59 memset(rf->arp_table[arp_index].ip_addr, 0, 60 sizeof(rf->arp_table[arp_index].ip_addr)); 61 eth_zero_addr(rf->arp_table[arp_index].mac_addr); 62 irdma_free_rsrc(rf, rf->allocated_arps, arp_index); 63 break; 64 default: 65 arp_index = -1; 66 break; 67 } 68 69 spin_unlock_irqrestore(&rf->arp_lock, flags); 70 return arp_index; 71 } 72 73 /** 74 * irdma_add_arp - add a new arp entry if needed 75 * @rf: RDMA function 76 * @ip: IP address 77 * @ipv4: IPv4 flag 78 * @mac: MAC address 79 */ 80 int irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, bool ipv4, const u8 *mac) 81 { 82 int arpidx; 83 84 arpidx = irdma_arp_table(rf, &ip[0], ipv4, NULL, IRDMA_ARP_RESOLVE); 85 if (arpidx >= 0) { 86 if (ether_addr_equal(rf->arp_table[arpidx].mac_addr, mac)) 87 return arpidx; 88 89 irdma_manage_arp_cache(rf, rf->arp_table[arpidx].mac_addr, ip, 90 ipv4, IRDMA_ARP_DELETE); 91 } 92 93 irdma_manage_arp_cache(rf, mac, ip, ipv4, IRDMA_ARP_ADD); 94 95 return irdma_arp_table(rf, ip, ipv4, NULL, IRDMA_ARP_RESOLVE); 96 } 97 98 /** 99 * wr32 - write 32 bits to hw register 100 * @hw: hardware information including registers 101 * @reg: register offset 102 * @val: value to write to register 103 */ 104 inline void wr32(struct irdma_hw *hw, u32 reg, u32 val) 105 { 106 writel(val, hw->hw_addr + reg); 107 } 108 109 /** 110 * rd32 - read a 32 bit hw register 111 * @hw: hardware information including registers 112 * @reg: register offset 113 * 114 * Return value of register content 115 */ 116 inline u32 rd32(struct irdma_hw *hw, u32 reg) 117 { 118 return readl(hw->hw_addr + reg); 119 } 120 121 /** 122 * rd64 - read a 64 bit hw register 123 * @hw: hardware information including registers 124 * @reg: register offset 125 * 126 * Return value of register content 127 */ 128 inline u64 rd64(struct irdma_hw *hw, u32 reg) 129 { 130 return readq(hw->hw_addr + reg); 131 } 132 133 static void irdma_gid_change_event(struct ib_device *ibdev) 134 { 135 struct ib_event ib_event; 136 137 ib_event.event = IB_EVENT_GID_CHANGE; 138 ib_event.device = ibdev; 139 ib_event.element.port_num = 1; 140 ib_dispatch_event(&ib_event); 141 } 142 143 /** 144 * irdma_inetaddr_event - system notifier for ipv4 addr events 145 * @notifier: not used 146 * @event: event for notifier 147 * @ptr: if address 148 */ 149 int irdma_inetaddr_event(struct notifier_block *notifier, unsigned long event, 150 void *ptr) 151 { 152 struct in_ifaddr *ifa = ptr; 153 struct net_device *real_dev, *netdev = ifa->ifa_dev->dev; 154 struct irdma_device *iwdev; 155 struct ib_device *ibdev; 156 u32 local_ipaddr; 157 158 real_dev = rdma_vlan_dev_real_dev(netdev); 159 if (!real_dev) 160 real_dev = netdev; 161 162 ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA); 163 if (!ibdev) 164 return NOTIFY_DONE; 165 166 iwdev = to_iwdev(ibdev); 167 local_ipaddr = ntohl(ifa->ifa_address); 168 ibdev_dbg(&iwdev->ibdev, 169 "DEV: netdev %p event %lu local_ip=%pI4 MAC=%pM\n", real_dev, 170 event, &local_ipaddr, real_dev->dev_addr); 171 switch (event) { 172 case NETDEV_DOWN: 173 irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr, 174 &local_ipaddr, true, IRDMA_ARP_DELETE); 175 irdma_if_notify(iwdev, real_dev, &local_ipaddr, true, false); 176 irdma_gid_change_event(&iwdev->ibdev); 177 break; 178 case NETDEV_UP: 179 case NETDEV_CHANGEADDR: 180 irdma_add_arp(iwdev->rf, &local_ipaddr, true, real_dev->dev_addr); 181 irdma_if_notify(iwdev, real_dev, &local_ipaddr, true, true); 182 irdma_gid_change_event(&iwdev->ibdev); 183 break; 184 default: 185 break; 186 } 187 188 ib_device_put(ibdev); 189 190 return NOTIFY_DONE; 191 } 192 193 /** 194 * irdma_inet6addr_event - system notifier for ipv6 addr events 195 * @notifier: not used 196 * @event: event for notifier 197 * @ptr: if address 198 */ 199 int irdma_inet6addr_event(struct notifier_block *notifier, unsigned long event, 200 void *ptr) 201 { 202 struct inet6_ifaddr *ifa = ptr; 203 struct net_device *real_dev, *netdev = ifa->idev->dev; 204 struct irdma_device *iwdev; 205 struct ib_device *ibdev; 206 u32 local_ipaddr6[4]; 207 208 real_dev = rdma_vlan_dev_real_dev(netdev); 209 if (!real_dev) 210 real_dev = netdev; 211 212 ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA); 213 if (!ibdev) 214 return NOTIFY_DONE; 215 216 iwdev = to_iwdev(ibdev); 217 irdma_copy_ip_ntohl(local_ipaddr6, ifa->addr.in6_u.u6_addr32); 218 ibdev_dbg(&iwdev->ibdev, 219 "DEV: netdev %p event %lu local_ip=%pI6 MAC=%pM\n", real_dev, 220 event, local_ipaddr6, real_dev->dev_addr); 221 switch (event) { 222 case NETDEV_DOWN: 223 irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr, 224 local_ipaddr6, false, IRDMA_ARP_DELETE); 225 irdma_if_notify(iwdev, real_dev, local_ipaddr6, false, false); 226 irdma_gid_change_event(&iwdev->ibdev); 227 break; 228 case NETDEV_UP: 229 case NETDEV_CHANGEADDR: 230 irdma_add_arp(iwdev->rf, local_ipaddr6, false, 231 real_dev->dev_addr); 232 irdma_if_notify(iwdev, real_dev, local_ipaddr6, false, true); 233 irdma_gid_change_event(&iwdev->ibdev); 234 break; 235 default: 236 break; 237 } 238 239 ib_device_put(ibdev); 240 241 return NOTIFY_DONE; 242 } 243 244 /** 245 * irdma_net_event - system notifier for net events 246 * @notifier: not used 247 * @event: event for notifier 248 * @ptr: neighbor 249 */ 250 int irdma_net_event(struct notifier_block *notifier, unsigned long event, 251 void *ptr) 252 { 253 struct neighbour *neigh = ptr; 254 struct net_device *real_dev, *netdev = (struct net_device *)neigh->dev; 255 struct irdma_device *iwdev; 256 struct ib_device *ibdev; 257 __be32 *p; 258 u32 local_ipaddr[4] = {}; 259 bool ipv4 = true; 260 261 switch (event) { 262 case NETEVENT_NEIGH_UPDATE: 263 real_dev = rdma_vlan_dev_real_dev(netdev); 264 if (!real_dev) 265 real_dev = netdev; 266 ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA); 267 if (!ibdev) 268 return NOTIFY_DONE; 269 270 iwdev = to_iwdev(ibdev); 271 p = (__be32 *)neigh->primary_key; 272 if (neigh->tbl->family == AF_INET6) { 273 ipv4 = false; 274 irdma_copy_ip_ntohl(local_ipaddr, p); 275 } else { 276 local_ipaddr[0] = ntohl(*p); 277 } 278 279 ibdev_dbg(&iwdev->ibdev, 280 "DEV: netdev %p state %d local_ip=%pI4 MAC=%pM\n", 281 iwdev->netdev, neigh->nud_state, local_ipaddr, 282 neigh->ha); 283 284 if (neigh->nud_state & NUD_VALID) 285 irdma_add_arp(iwdev->rf, local_ipaddr, ipv4, neigh->ha); 286 287 else 288 irdma_manage_arp_cache(iwdev->rf, neigh->ha, 289 local_ipaddr, ipv4, 290 IRDMA_ARP_DELETE); 291 ib_device_put(ibdev); 292 break; 293 default: 294 break; 295 } 296 297 return NOTIFY_DONE; 298 } 299 300 /** 301 * irdma_netdevice_event - system notifier for netdev events 302 * @notifier: not used 303 * @event: event for notifier 304 * @ptr: netdev 305 */ 306 int irdma_netdevice_event(struct notifier_block *notifier, unsigned long event, 307 void *ptr) 308 { 309 struct irdma_device *iwdev; 310 struct ib_device *ibdev; 311 struct net_device *netdev = netdev_notifier_info_to_dev(ptr); 312 313 ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_IRDMA); 314 if (!ibdev) 315 return NOTIFY_DONE; 316 317 iwdev = to_iwdev(ibdev); 318 iwdev->iw_status = 1; 319 switch (event) { 320 case NETDEV_DOWN: 321 iwdev->iw_status = 0; 322 fallthrough; 323 default: 324 break; 325 } 326 ib_device_put(ibdev); 327 328 return NOTIFY_DONE; 329 } 330 331 /** 332 * irdma_add_ipv6_addr - add ipv6 address to the hw arp table 333 * @iwdev: irdma device 334 */ 335 static void irdma_add_ipv6_addr(struct irdma_device *iwdev) 336 { 337 struct net_device *ip_dev; 338 struct inet6_dev *idev; 339 struct inet6_ifaddr *ifp, *tmp; 340 u32 local_ipaddr6[4]; 341 342 rcu_read_lock(); 343 for_each_netdev_rcu (&init_net, ip_dev) { 344 if (((rdma_vlan_dev_vlan_id(ip_dev) < 0xFFFF && 345 rdma_vlan_dev_real_dev(ip_dev) == iwdev->netdev) || 346 ip_dev == iwdev->netdev) && 347 (READ_ONCE(ip_dev->flags) & IFF_UP)) { 348 idev = __in6_dev_get(ip_dev); 349 if (!idev) { 350 ibdev_err(&iwdev->ibdev, "ipv6 inet device not found\n"); 351 break; 352 } 353 list_for_each_entry_safe (ifp, tmp, &idev->addr_list, 354 if_list) { 355 ibdev_dbg(&iwdev->ibdev, 356 "INIT: IP=%pI6, vlan_id=%d, MAC=%pM\n", 357 &ifp->addr, 358 rdma_vlan_dev_vlan_id(ip_dev), 359 ip_dev->dev_addr); 360 361 irdma_copy_ip_ntohl(local_ipaddr6, 362 ifp->addr.in6_u.u6_addr32); 363 irdma_manage_arp_cache(iwdev->rf, 364 ip_dev->dev_addr, 365 local_ipaddr6, false, 366 IRDMA_ARP_ADD); 367 } 368 } 369 } 370 rcu_read_unlock(); 371 } 372 373 /** 374 * irdma_add_ipv4_addr - add ipv4 address to the hw arp table 375 * @iwdev: irdma device 376 */ 377 static void irdma_add_ipv4_addr(struct irdma_device *iwdev) 378 { 379 struct net_device *dev; 380 struct in_device *idev; 381 u32 ip_addr; 382 383 rcu_read_lock(); 384 for_each_netdev_rcu (&init_net, dev) { 385 if (((rdma_vlan_dev_vlan_id(dev) < 0xFFFF && 386 rdma_vlan_dev_real_dev(dev) == iwdev->netdev) || 387 dev == iwdev->netdev) && (READ_ONCE(dev->flags) & IFF_UP)) { 388 const struct in_ifaddr *ifa; 389 390 idev = __in_dev_get_rcu(dev); 391 if (!idev) 392 continue; 393 394 in_dev_for_each_ifa_rcu(ifa, idev) { 395 ibdev_dbg(&iwdev->ibdev, "CM: IP=%pI4, vlan_id=%d, MAC=%pM\n", 396 &ifa->ifa_address, rdma_vlan_dev_vlan_id(dev), 397 dev->dev_addr); 398 399 ip_addr = ntohl(ifa->ifa_address); 400 irdma_manage_arp_cache(iwdev->rf, dev->dev_addr, 401 &ip_addr, true, 402 IRDMA_ARP_ADD); 403 } 404 } 405 } 406 rcu_read_unlock(); 407 } 408 409 /** 410 * irdma_add_ip - add ip addresses 411 * @iwdev: irdma device 412 * 413 * Add ipv4/ipv6 addresses to the arp cache 414 */ 415 void irdma_add_ip(struct irdma_device *iwdev) 416 { 417 irdma_add_ipv4_addr(iwdev); 418 irdma_add_ipv6_addr(iwdev); 419 } 420 421 /** 422 * irdma_alloc_and_get_cqp_request - get cqp struct 423 * @cqp: device cqp ptr 424 * @wait: cqp to be used in wait mode 425 */ 426 struct irdma_cqp_request *irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp, 427 bool wait) 428 { 429 struct irdma_cqp_request *cqp_request = NULL; 430 unsigned long flags; 431 432 spin_lock_irqsave(&cqp->req_lock, flags); 433 if (!list_empty(&cqp->cqp_avail_reqs)) { 434 cqp_request = list_first_entry(&cqp->cqp_avail_reqs, 435 struct irdma_cqp_request, list); 436 list_del_init(&cqp_request->list); 437 } 438 spin_unlock_irqrestore(&cqp->req_lock, flags); 439 if (!cqp_request) { 440 cqp_request = kzalloc(sizeof(*cqp_request), GFP_ATOMIC); 441 if (cqp_request) { 442 cqp_request->dynamic = true; 443 if (wait) 444 init_waitqueue_head(&cqp_request->waitq); 445 } 446 } 447 if (!cqp_request) { 448 ibdev_dbg(to_ibdev(cqp->sc_cqp.dev), "ERR: CQP Request Fail: No Memory"); 449 return NULL; 450 } 451 452 cqp_request->waiting = wait; 453 refcount_set(&cqp_request->refcnt, 1); 454 memset(&cqp_request->compl_info, 0, sizeof(cqp_request->compl_info)); 455 456 return cqp_request; 457 } 458 459 /** 460 * irdma_get_cqp_request - increase refcount for cqp_request 461 * @cqp_request: pointer to cqp_request instance 462 */ 463 static inline void irdma_get_cqp_request(struct irdma_cqp_request *cqp_request) 464 { 465 refcount_inc(&cqp_request->refcnt); 466 } 467 468 /** 469 * irdma_free_cqp_request - free cqp request 470 * @cqp: cqp ptr 471 * @cqp_request: to be put back in cqp list 472 */ 473 void irdma_free_cqp_request(struct irdma_cqp *cqp, 474 struct irdma_cqp_request *cqp_request) 475 { 476 unsigned long flags; 477 478 if (cqp_request->dynamic) { 479 kfree(cqp_request); 480 } else { 481 WRITE_ONCE(cqp_request->request_done, false); 482 cqp_request->callback_fcn = NULL; 483 cqp_request->waiting = false; 484 cqp_request->pending = false; 485 486 spin_lock_irqsave(&cqp->req_lock, flags); 487 list_add_tail(&cqp_request->list, &cqp->cqp_avail_reqs); 488 spin_unlock_irqrestore(&cqp->req_lock, flags); 489 } 490 wake_up(&cqp->remove_wq); 491 } 492 493 /** 494 * irdma_put_cqp_request - dec ref count and free if 0 495 * @cqp: cqp ptr 496 * @cqp_request: to be put back in cqp list 497 */ 498 void irdma_put_cqp_request(struct irdma_cqp *cqp, 499 struct irdma_cqp_request *cqp_request) 500 { 501 if (refcount_dec_and_test(&cqp_request->refcnt)) 502 irdma_free_cqp_request(cqp, cqp_request); 503 } 504 505 /** 506 * irdma_free_pending_cqp_request -free pending cqp request objs 507 * @cqp: cqp ptr 508 * @cqp_request: to be put back in cqp list 509 */ 510 static void 511 irdma_free_pending_cqp_request(struct irdma_cqp *cqp, 512 struct irdma_cqp_request *cqp_request) 513 { 514 if (cqp_request->waiting) { 515 cqp_request->compl_info.error = true; 516 WRITE_ONCE(cqp_request->request_done, true); 517 wake_up(&cqp_request->waitq); 518 } 519 wait_event_timeout(cqp->remove_wq, 520 refcount_read(&cqp_request->refcnt) == 1, 1000); 521 irdma_put_cqp_request(cqp, cqp_request); 522 } 523 524 /** 525 * irdma_cleanup_deferred_cqp_ops - clean-up cqp with no completions 526 * @dev: sc_dev 527 * @cqp: cqp 528 */ 529 static void irdma_cleanup_deferred_cqp_ops(struct irdma_sc_dev *dev, 530 struct irdma_cqp *cqp) 531 { 532 u64 scratch; 533 534 /* process all CQP requests with deferred/pending completions */ 535 while ((scratch = irdma_sc_cqp_cleanup_handler(dev))) 536 irdma_free_pending_cqp_request(cqp, (struct irdma_cqp_request *) 537 (uintptr_t)scratch); 538 } 539 540 /** 541 * irdma_cleanup_pending_cqp_op - clean-up cqp with no 542 * completions 543 * @rf: RDMA PCI function 544 */ 545 void irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf) 546 { 547 struct irdma_sc_dev *dev = &rf->sc_dev; 548 struct irdma_cqp *cqp = &rf->cqp; 549 struct irdma_cqp_request *cqp_request = NULL; 550 struct cqp_cmds_info *pcmdinfo = NULL; 551 u32 i, pending_work, wqe_idx; 552 553 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_3) 554 irdma_cleanup_deferred_cqp_ops(dev, cqp); 555 pending_work = IRDMA_RING_USED_QUANTA(cqp->sc_cqp.sq_ring); 556 wqe_idx = IRDMA_RING_CURRENT_TAIL(cqp->sc_cqp.sq_ring); 557 for (i = 0; i < pending_work; i++) { 558 cqp_request = (struct irdma_cqp_request *)(unsigned long) 559 cqp->scratch_array[wqe_idx]; 560 if (cqp_request) 561 irdma_free_pending_cqp_request(cqp, cqp_request); 562 wqe_idx = (wqe_idx + 1) % IRDMA_RING_SIZE(cqp->sc_cqp.sq_ring); 563 } 564 565 while (!list_empty(&dev->cqp_cmd_head)) { 566 pcmdinfo = irdma_remove_cqp_head(dev); 567 cqp_request = 568 container_of(pcmdinfo, struct irdma_cqp_request, info); 569 if (cqp_request) 570 irdma_free_pending_cqp_request(cqp, cqp_request); 571 } 572 } 573 574 static int irdma_get_timeout_threshold(struct irdma_sc_dev *dev) 575 { 576 u16 time_s = dev->vc_caps.cqp_timeout_s; 577 578 if (!time_s) 579 return CQP_TIMEOUT_THRESHOLD; 580 581 return time_s * 1000 / dev->hw_attrs.max_cqp_compl_wait_time_ms; 582 } 583 584 static int irdma_get_def_timeout_threshold(struct irdma_sc_dev *dev) 585 { 586 u16 time_s = dev->vc_caps.cqp_def_timeout_s; 587 588 if (!time_s) 589 return CQP_DEF_CMPL_TIMEOUT_THRESHOLD; 590 591 return time_s * 1000 / dev->hw_attrs.max_cqp_compl_wait_time_ms; 592 } 593 594 /** 595 * irdma_wait_event - wait for completion 596 * @rf: RDMA PCI function 597 * @cqp_request: cqp request to wait 598 */ 599 static int irdma_wait_event(struct irdma_pci_f *rf, 600 struct irdma_cqp_request *cqp_request) 601 { 602 struct irdma_cqp_timeout cqp_timeout = {}; 603 int timeout_threshold = irdma_get_timeout_threshold(&rf->sc_dev); 604 bool cqp_error = false; 605 int err_code = 0; 606 607 cqp_timeout.compl_cqp_cmds = atomic64_read(&rf->sc_dev.cqp->completed_ops); 608 do { 609 irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); 610 if (wait_event_timeout(cqp_request->waitq, 611 READ_ONCE(cqp_request->request_done), 612 msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS))) 613 break; 614 615 if (cqp_request->pending) 616 /* There was a deferred or pending completion 617 * received for this CQP request, so we need 618 * to wait longer than usual. 619 */ 620 timeout_threshold = 621 irdma_get_def_timeout_threshold(&rf->sc_dev); 622 623 irdma_check_cqp_progress(&cqp_timeout, &rf->sc_dev); 624 625 if (cqp_timeout.count < timeout_threshold) 626 continue; 627 628 if (!rf->reset) { 629 rf->reset = true; 630 rf->gen_ops.request_reset(rf); 631 } 632 return -ETIMEDOUT; 633 } while (1); 634 635 cqp_error = cqp_request->compl_info.error; 636 if (cqp_error) { 637 err_code = -EIO; 638 if (cqp_request->compl_info.maj_err_code == 0xFFFF) { 639 if (cqp_request->compl_info.min_err_code == 0x8002) 640 err_code = -EBUSY; 641 else if (cqp_request->compl_info.min_err_code == 0x8029) { 642 if (!rf->reset) { 643 rf->reset = true; 644 rf->gen_ops.request_reset(rf); 645 } 646 } 647 } 648 } 649 650 return err_code; 651 } 652 653 static const char *const irdma_cqp_cmd_names[IRDMA_MAX_CQP_OPS] = { 654 [IRDMA_OP_CEQ_DESTROY] = "Destroy CEQ Cmd", 655 [IRDMA_OP_AEQ_DESTROY] = "Destroy AEQ Cmd", 656 [IRDMA_OP_DELETE_ARP_CACHE_ENTRY] = "Delete ARP Cache Cmd", 657 [IRDMA_OP_MANAGE_APBVT_ENTRY] = "Manage APBV Table Entry Cmd", 658 [IRDMA_OP_CEQ_CREATE] = "CEQ Create Cmd", 659 [IRDMA_OP_AEQ_CREATE] = "AEQ Destroy Cmd", 660 [IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY] = "Manage Quad Hash Table Entry Cmd", 661 [IRDMA_OP_QP_MODIFY] = "Modify QP Cmd", 662 [IRDMA_OP_QP_UPLOAD_CONTEXT] = "Upload Context Cmd", 663 [IRDMA_OP_CQ_CREATE] = "Create CQ Cmd", 664 [IRDMA_OP_CQ_DESTROY] = "Destroy CQ Cmd", 665 [IRDMA_OP_QP_CREATE] = "Create QP Cmd", 666 [IRDMA_OP_QP_DESTROY] = "Destroy QP Cmd", 667 [IRDMA_OP_ALLOC_STAG] = "Allocate STag Cmd", 668 [IRDMA_OP_MR_REG_NON_SHARED] = "Register Non-Shared MR Cmd", 669 [IRDMA_OP_DEALLOC_STAG] = "Deallocate STag Cmd", 670 [IRDMA_OP_MW_ALLOC] = "Allocate Memory Window Cmd", 671 [IRDMA_OP_QP_FLUSH_WQES] = "Flush QP Cmd", 672 [IRDMA_OP_ADD_ARP_CACHE_ENTRY] = "Add ARP Cache Cmd", 673 [IRDMA_OP_MANAGE_PUSH_PAGE] = "Manage Push Page Cmd", 674 [IRDMA_OP_UPDATE_PE_SDS] = "Update PE SDs Cmd", 675 [IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE] = "Manage HMC PM Function Table Cmd", 676 [IRDMA_OP_SUSPEND] = "Suspend QP Cmd", 677 [IRDMA_OP_RESUME] = "Resume QP Cmd", 678 [IRDMA_OP_MANAGE_VF_PBLE_BP] = "Manage VF PBLE Backing Pages Cmd", 679 [IRDMA_OP_QUERY_FPM_VAL] = "Query FPM Values Cmd", 680 [IRDMA_OP_COMMIT_FPM_VAL] = "Commit FPM Values Cmd", 681 [IRDMA_OP_AH_CREATE] = "Create Address Handle Cmd", 682 [IRDMA_OP_AH_MODIFY] = "Modify Address Handle Cmd", 683 [IRDMA_OP_AH_DESTROY] = "Destroy Address Handle Cmd", 684 [IRDMA_OP_MC_CREATE] = "Create Multicast Group Cmd", 685 [IRDMA_OP_MC_DESTROY] = "Destroy Multicast Group Cmd", 686 [IRDMA_OP_MC_MODIFY] = "Modify Multicast Group Cmd", 687 [IRDMA_OP_STATS_ALLOCATE] = "Add Statistics Instance Cmd", 688 [IRDMA_OP_STATS_FREE] = "Free Statistics Instance Cmd", 689 [IRDMA_OP_STATS_GATHER] = "Gather Statistics Cmd", 690 [IRDMA_OP_WS_ADD_NODE] = "Add Work Scheduler Node Cmd", 691 [IRDMA_OP_WS_MODIFY_NODE] = "Modify Work Scheduler Node Cmd", 692 [IRDMA_OP_WS_DELETE_NODE] = "Delete Work Scheduler Node Cmd", 693 [IRDMA_OP_SET_UP_MAP] = "Set UP-UP Mapping Cmd", 694 [IRDMA_OP_GEN_AE] = "Generate AE Cmd", 695 [IRDMA_OP_QUERY_RDMA_FEATURES] = "RDMA Get Features Cmd", 696 [IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY] = "Allocate Local MAC Entry Cmd", 697 [IRDMA_OP_ADD_LOCAL_MAC_ENTRY] = "Add Local MAC Entry Cmd", 698 [IRDMA_OP_DELETE_LOCAL_MAC_ENTRY] = "Delete Local MAC Entry Cmd", 699 [IRDMA_OP_CQ_MODIFY] = "CQ Modify Cmd", 700 [IRDMA_OP_SRQ_CREATE] = "Create SRQ Cmd", 701 [IRDMA_OP_SRQ_MODIFY] = "Modify SRQ Cmd", 702 [IRDMA_OP_SRQ_DESTROY] = "Destroy SRQ Cmd", 703 }; 704 705 static const struct irdma_cqp_err_info irdma_noncrit_err_list[] = { 706 {0xffff, 0x8002, "Invalid State"}, 707 {0xffff, 0x8006, "Flush No Wqe Pending"}, 708 {0xffff, 0x8007, "Modify QP Bad Close"}, 709 {0xffff, 0x8009, "LLP Closed"}, 710 {0xffff, 0x800a, "Reset Not Sent"} 711 }; 712 713 /** 714 * irdma_cqp_crit_err - check if CQP error is critical 715 * @dev: pointer to dev structure 716 * @cqp_cmd: code for last CQP operation 717 * @maj_err_code: major error code 718 * @min_err_code: minot error code 719 */ 720 bool irdma_cqp_crit_err(struct irdma_sc_dev *dev, u8 cqp_cmd, 721 u16 maj_err_code, u16 min_err_code) 722 { 723 int i; 724 725 for (i = 0; i < ARRAY_SIZE(irdma_noncrit_err_list); ++i) { 726 if (maj_err_code == irdma_noncrit_err_list[i].maj && 727 min_err_code == irdma_noncrit_err_list[i].min) { 728 ibdev_dbg(to_ibdev(dev), 729 "CQP: [%s Error][%s] maj=0x%x min=0x%x\n", 730 irdma_noncrit_err_list[i].desc, 731 irdma_cqp_cmd_names[cqp_cmd], maj_err_code, 732 min_err_code); 733 return false; 734 } 735 } 736 return true; 737 } 738 739 /** 740 * irdma_handle_cqp_op - process cqp command 741 * @rf: RDMA PCI function 742 * @cqp_request: cqp request to process 743 */ 744 int irdma_handle_cqp_op(struct irdma_pci_f *rf, 745 struct irdma_cqp_request *cqp_request) 746 { 747 struct irdma_sc_dev *dev = &rf->sc_dev; 748 struct cqp_cmds_info *info = &cqp_request->info; 749 int status; 750 bool put_cqp_request = true; 751 752 if (rf->reset) 753 return -EBUSY; 754 755 irdma_get_cqp_request(cqp_request); 756 status = irdma_process_cqp_cmd(dev, info); 757 if (status) 758 goto err; 759 760 if (cqp_request->waiting) { 761 put_cqp_request = false; 762 status = irdma_wait_event(rf, cqp_request); 763 if (status) 764 goto err; 765 } 766 767 return 0; 768 769 err: 770 if (irdma_cqp_crit_err(dev, info->cqp_cmd, 771 cqp_request->compl_info.maj_err_code, 772 cqp_request->compl_info.min_err_code)) 773 ibdev_err(&rf->iwdev->ibdev, 774 "[%s Error][op_code=%d] status=%d waiting=%d completion_err=%d maj=0x%x min=0x%x\n", 775 irdma_cqp_cmd_names[info->cqp_cmd], info->cqp_cmd, status, cqp_request->waiting, 776 cqp_request->compl_info.error, cqp_request->compl_info.maj_err_code, 777 cqp_request->compl_info.min_err_code); 778 779 if (put_cqp_request) 780 irdma_put_cqp_request(&rf->cqp, cqp_request); 781 782 return status; 783 } 784 785 void irdma_qp_add_ref(struct ib_qp *ibqp) 786 { 787 struct irdma_qp *iwqp = (struct irdma_qp *)ibqp; 788 789 refcount_inc(&iwqp->refcnt); 790 } 791 792 void irdma_qp_rem_ref(struct ib_qp *ibqp) 793 { 794 struct irdma_qp *iwqp = to_iwqp(ibqp); 795 struct irdma_device *iwdev = iwqp->iwdev; 796 u32 qp_num; 797 unsigned long flags; 798 799 spin_lock_irqsave(&iwdev->rf->qptable_lock, flags); 800 if (!refcount_dec_and_test(&iwqp->refcnt)) { 801 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); 802 return; 803 } 804 805 qp_num = iwqp->ibqp.qp_num; 806 iwdev->rf->qp_table[qp_num] = NULL; 807 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); 808 complete(&iwqp->free_qp); 809 } 810 811 void irdma_cq_add_ref(struct ib_cq *ibcq) 812 { 813 struct irdma_cq *iwcq = to_iwcq(ibcq); 814 815 refcount_inc(&iwcq->refcnt); 816 } 817 818 void irdma_cq_rem_ref(struct ib_cq *ibcq) 819 { 820 struct ib_device *ibdev = ibcq->device; 821 struct irdma_device *iwdev = to_iwdev(ibdev); 822 struct irdma_cq *iwcq = to_iwcq(ibcq); 823 unsigned long flags; 824 825 spin_lock_irqsave(&iwdev->rf->cqtable_lock, flags); 826 if (!refcount_dec_and_test(&iwcq->refcnt)) { 827 spin_unlock_irqrestore(&iwdev->rf->cqtable_lock, flags); 828 return; 829 } 830 831 iwdev->rf->cq_table[iwcq->cq_num] = NULL; 832 spin_unlock_irqrestore(&iwdev->rf->cqtable_lock, flags); 833 complete(&iwcq->free_cq); 834 } 835 836 struct ib_device *to_ibdev(struct irdma_sc_dev *dev) 837 { 838 return &(container_of(dev, struct irdma_pci_f, sc_dev))->iwdev->ibdev; 839 } 840 841 /** 842 * irdma_get_qp - get qp address 843 * @device: iwarp device 844 * @qpn: qp number 845 */ 846 struct ib_qp *irdma_get_qp(struct ib_device *device, int qpn) 847 { 848 struct irdma_device *iwdev = to_iwdev(device); 849 850 if (qpn < IW_FIRST_QPN || qpn >= iwdev->rf->max_qp) 851 return NULL; 852 853 return &iwdev->rf->qp_table[qpn]->ibqp; 854 } 855 856 /** 857 * irdma_remove_cqp_head - return head entry and remove 858 * @dev: device 859 */ 860 void *irdma_remove_cqp_head(struct irdma_sc_dev *dev) 861 { 862 struct list_head *entry; 863 struct list_head *list = &dev->cqp_cmd_head; 864 865 if (list_empty(list)) 866 return NULL; 867 868 entry = list->next; 869 list_del(entry); 870 871 return entry; 872 } 873 874 /** 875 * irdma_cqp_sds_cmd - create cqp command for sd 876 * @dev: hardware control device structure 877 * @sdinfo: information for sd cqp 878 * 879 */ 880 int irdma_cqp_sds_cmd(struct irdma_sc_dev *dev, 881 struct irdma_update_sds_info *sdinfo) 882 { 883 struct irdma_cqp_request *cqp_request; 884 struct cqp_cmds_info *cqp_info; 885 struct irdma_pci_f *rf = dev_to_rf(dev); 886 int status; 887 888 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 889 if (!cqp_request) 890 return -ENOMEM; 891 892 cqp_info = &cqp_request->info; 893 memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo, 894 sizeof(cqp_info->in.u.update_pe_sds.info)); 895 cqp_info->cqp_cmd = IRDMA_OP_UPDATE_PE_SDS; 896 cqp_info->post_sq = 1; 897 cqp_info->in.u.update_pe_sds.dev = dev; 898 cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request; 899 900 status = irdma_handle_cqp_op(rf, cqp_request); 901 irdma_put_cqp_request(&rf->cqp, cqp_request); 902 903 return status; 904 } 905 906 /** 907 * irdma_cqp_qp_suspend_resume - cqp command for suspend/resume 908 * @qp: hardware control qp 909 * @op: suspend or resume 910 */ 911 int irdma_cqp_qp_suspend_resume(struct irdma_sc_qp *qp, u8 op) 912 { 913 struct irdma_sc_dev *dev = qp->dev; 914 struct irdma_cqp_request *cqp_request; 915 struct irdma_sc_cqp *cqp = dev->cqp; 916 struct cqp_cmds_info *cqp_info; 917 struct irdma_pci_f *rf = dev_to_rf(dev); 918 int status; 919 920 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false); 921 if (!cqp_request) 922 return -ENOMEM; 923 924 cqp_info = &cqp_request->info; 925 cqp_info->cqp_cmd = op; 926 cqp_info->in.u.suspend_resume.cqp = cqp; 927 cqp_info->in.u.suspend_resume.qp = qp; 928 cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request; 929 930 status = irdma_handle_cqp_op(rf, cqp_request); 931 irdma_put_cqp_request(&rf->cqp, cqp_request); 932 933 return status; 934 } 935 936 /** 937 * irdma_term_modify_qp - modify qp for term message 938 * @qp: hardware control qp 939 * @next_state: qp's next state 940 * @term: terminate code 941 * @term_len: length 942 */ 943 void irdma_term_modify_qp(struct irdma_sc_qp *qp, u8 next_state, u8 term, 944 u8 term_len) 945 { 946 struct irdma_qp *iwqp; 947 948 iwqp = qp->qp_uk.back_qp; 949 irdma_next_iw_state(iwqp, next_state, 0, term, term_len); 950 }; 951 952 /** 953 * irdma_terminate_done - after terminate is completed 954 * @qp: hardware control qp 955 * @timeout_occurred: indicates if terminate timer expired 956 */ 957 void irdma_terminate_done(struct irdma_sc_qp *qp, int timeout_occurred) 958 { 959 struct irdma_qp *iwqp; 960 u8 hte = 0; 961 bool first_time; 962 unsigned long flags; 963 964 iwqp = qp->qp_uk.back_qp; 965 spin_lock_irqsave(&iwqp->lock, flags); 966 if (iwqp->hte_added) { 967 iwqp->hte_added = 0; 968 hte = 1; 969 } 970 first_time = !(qp->term_flags & IRDMA_TERM_DONE); 971 qp->term_flags |= IRDMA_TERM_DONE; 972 spin_unlock_irqrestore(&iwqp->lock, flags); 973 if (first_time) { 974 if (!timeout_occurred) 975 irdma_terminate_del_timer(qp); 976 977 irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, hte, 0, 0); 978 irdma_cm_disconn(iwqp); 979 } 980 } 981 982 static void irdma_terminate_timeout(struct timer_list *t) 983 { 984 struct irdma_qp *iwqp = timer_container_of(iwqp, t, terminate_timer); 985 struct irdma_sc_qp *qp = &iwqp->sc_qp; 986 987 irdma_terminate_done(qp, 1); 988 irdma_qp_rem_ref(&iwqp->ibqp); 989 } 990 991 /** 992 * irdma_terminate_start_timer - start terminate timeout 993 * @qp: hardware control qp 994 */ 995 void irdma_terminate_start_timer(struct irdma_sc_qp *qp) 996 { 997 struct irdma_qp *iwqp; 998 999 iwqp = qp->qp_uk.back_qp; 1000 irdma_qp_add_ref(&iwqp->ibqp); 1001 timer_setup(&iwqp->terminate_timer, irdma_terminate_timeout, 0); 1002 iwqp->terminate_timer.expires = jiffies + HZ; 1003 1004 add_timer(&iwqp->terminate_timer); 1005 } 1006 1007 /** 1008 * irdma_terminate_del_timer - delete terminate timeout 1009 * @qp: hardware control qp 1010 */ 1011 void irdma_terminate_del_timer(struct irdma_sc_qp *qp) 1012 { 1013 struct irdma_qp *iwqp; 1014 int ret; 1015 1016 iwqp = qp->qp_uk.back_qp; 1017 ret = timer_delete(&iwqp->terminate_timer); 1018 if (ret) 1019 irdma_qp_rem_ref(&iwqp->ibqp); 1020 } 1021 1022 /** 1023 * irdma_cqp_cq_create_cmd - create a cq for the cqp 1024 * @dev: device pointer 1025 * @cq: pointer to created cq 1026 */ 1027 int irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq) 1028 { 1029 struct irdma_pci_f *rf = dev_to_rf(dev); 1030 struct irdma_cqp *iwcqp = &rf->cqp; 1031 struct irdma_cqp_request *cqp_request; 1032 struct cqp_cmds_info *cqp_info; 1033 int status; 1034 1035 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true); 1036 if (!cqp_request) 1037 return -ENOMEM; 1038 1039 cqp_info = &cqp_request->info; 1040 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE; 1041 cqp_info->post_sq = 1; 1042 cqp_info->in.u.cq_create.cq = cq; 1043 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request; 1044 1045 status = irdma_handle_cqp_op(rf, cqp_request); 1046 irdma_put_cqp_request(iwcqp, cqp_request); 1047 1048 return status; 1049 } 1050 1051 /** 1052 * irdma_cqp_qp_create_cmd - create a qp for the cqp 1053 * @dev: device pointer 1054 * @qp: pointer to created qp 1055 */ 1056 int irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp) 1057 { 1058 struct irdma_pci_f *rf = dev_to_rf(dev); 1059 struct irdma_cqp *iwcqp = &rf->cqp; 1060 struct irdma_cqp_request *cqp_request; 1061 struct cqp_cmds_info *cqp_info; 1062 struct irdma_create_qp_info *qp_info; 1063 int status; 1064 1065 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true); 1066 if (!cqp_request) 1067 return -ENOMEM; 1068 1069 cqp_info = &cqp_request->info; 1070 qp_info = &cqp_request->info.in.u.qp_create.info; 1071 memset(qp_info, 0, sizeof(*qp_info)); 1072 qp_info->cq_num_valid = true; 1073 qp_info->next_iwarp_state = IRDMA_QP_STATE_RTS; 1074 cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE; 1075 cqp_info->post_sq = 1; 1076 cqp_info->in.u.qp_create.qp = qp; 1077 cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request; 1078 1079 status = irdma_handle_cqp_op(rf, cqp_request); 1080 irdma_put_cqp_request(iwcqp, cqp_request); 1081 1082 return status; 1083 } 1084 1085 /** 1086 * irdma_dealloc_push_page - free a push page for qp 1087 * @rf: RDMA PCI function 1088 * @qp: hardware control qp 1089 */ 1090 static void irdma_dealloc_push_page(struct irdma_pci_f *rf, 1091 struct irdma_sc_qp *qp) 1092 { 1093 struct irdma_cqp_request *cqp_request; 1094 struct cqp_cmds_info *cqp_info; 1095 int status; 1096 1097 if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) 1098 return; 1099 1100 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false); 1101 if (!cqp_request) 1102 return; 1103 1104 cqp_info = &cqp_request->info; 1105 cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE; 1106 cqp_info->post_sq = 1; 1107 cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx; 1108 cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle; 1109 cqp_info->in.u.manage_push_page.info.free_page = 1; 1110 cqp_info->in.u.manage_push_page.info.push_page_type = 0; 1111 cqp_info->in.u.manage_push_page.cqp = &rf->cqp.sc_cqp; 1112 cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request; 1113 status = irdma_handle_cqp_op(rf, cqp_request); 1114 if (!status) 1115 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX; 1116 irdma_put_cqp_request(&rf->cqp, cqp_request); 1117 } 1118 1119 static void irdma_free_gsi_qp_rsrc(struct irdma_qp *iwqp, u32 qp_num) 1120 { 1121 struct irdma_device *iwdev = iwqp->iwdev; 1122 struct irdma_pci_f *rf = iwdev->rf; 1123 unsigned long flags; 1124 1125 if (rf->sc_dev.hw_attrs.uk_attrs.hw_rev < IRDMA_GEN_3) 1126 return; 1127 1128 irdma_vchnl_req_del_vport(&rf->sc_dev, iwdev->vport_id, qp_num); 1129 1130 if (qp_num == 1) { 1131 spin_lock_irqsave(&rf->rsrc_lock, flags); 1132 rf->hwqp1_rsvd = false; 1133 spin_unlock_irqrestore(&rf->rsrc_lock, flags); 1134 } else if (qp_num > 2) { 1135 irdma_free_rsrc(rf, rf->allocated_qps, qp_num); 1136 } 1137 } 1138 1139 /** 1140 * irdma_free_qp_rsrc - free up memory resources for qp 1141 * @iwqp: qp ptr (user or kernel) 1142 */ 1143 void irdma_free_qp_rsrc(struct irdma_qp *iwqp) 1144 { 1145 struct irdma_device *iwdev = iwqp->iwdev; 1146 struct irdma_pci_f *rf = iwdev->rf; 1147 u32 qp_num = iwqp->sc_qp.qp_uk.qp_id; 1148 1149 irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp); 1150 irdma_dealloc_push_page(rf, &iwqp->sc_qp); 1151 if (iwqp->sc_qp.vsi) { 1152 irdma_qp_rem_qos(&iwqp->sc_qp); 1153 iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi, 1154 iwqp->sc_qp.user_pri); 1155 } 1156 1157 if (iwqp->ibqp.qp_type == IB_QPT_GSI) { 1158 irdma_free_gsi_qp_rsrc(iwqp, qp_num); 1159 } else { 1160 if (qp_num > 2) 1161 irdma_free_rsrc(rf, rf->allocated_qps, qp_num); 1162 } 1163 dma_free_coherent(rf->sc_dev.hw->device, iwqp->q2_ctx_mem.size, 1164 iwqp->q2_ctx_mem.va, iwqp->q2_ctx_mem.pa); 1165 iwqp->q2_ctx_mem.va = NULL; 1166 dma_free_coherent(rf->sc_dev.hw->device, iwqp->kqp.dma_mem.size, 1167 iwqp->kqp.dma_mem.va, iwqp->kqp.dma_mem.pa); 1168 iwqp->kqp.dma_mem.va = NULL; 1169 kfree(iwqp->kqp.sq_wrid_mem); 1170 kfree(iwqp->kqp.rq_wrid_mem); 1171 } 1172 1173 /** 1174 * irdma_srq_wq_destroy - send srq destroy cqp 1175 * @rf: RDMA PCI function 1176 * @srq: hardware control srq 1177 */ 1178 void irdma_srq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_srq *srq) 1179 { 1180 struct irdma_cqp_request *cqp_request; 1181 struct cqp_cmds_info *cqp_info; 1182 1183 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1184 if (!cqp_request) 1185 return; 1186 1187 cqp_info = &cqp_request->info; 1188 cqp_info->cqp_cmd = IRDMA_OP_SRQ_DESTROY; 1189 cqp_info->post_sq = 1; 1190 cqp_info->in.u.srq_destroy.srq = srq; 1191 cqp_info->in.u.srq_destroy.scratch = (uintptr_t)cqp_request; 1192 1193 irdma_handle_cqp_op(rf, cqp_request); 1194 irdma_put_cqp_request(&rf->cqp, cqp_request); 1195 } 1196 1197 /** 1198 * irdma_cq_wq_destroy - send cq destroy cqp 1199 * @rf: RDMA PCI function 1200 * @cq: hardware control cq 1201 */ 1202 void irdma_cq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_cq *cq) 1203 { 1204 struct irdma_cqp_request *cqp_request; 1205 struct cqp_cmds_info *cqp_info; 1206 1207 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1208 if (!cqp_request) 1209 return; 1210 1211 cqp_info = &cqp_request->info; 1212 cqp_info->cqp_cmd = IRDMA_OP_CQ_DESTROY; 1213 cqp_info->post_sq = 1; 1214 cqp_info->in.u.cq_destroy.cq = cq; 1215 cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request; 1216 1217 irdma_handle_cqp_op(rf, cqp_request); 1218 irdma_put_cqp_request(&rf->cqp, cqp_request); 1219 } 1220 1221 /** 1222 * irdma_hw_modify_qp_callback - handle state for modifyQPs that don't wait 1223 * @cqp_request: modify QP completion 1224 */ 1225 static void irdma_hw_modify_qp_callback(struct irdma_cqp_request *cqp_request) 1226 { 1227 struct cqp_cmds_info *cqp_info; 1228 struct irdma_qp *iwqp; 1229 1230 cqp_info = &cqp_request->info; 1231 iwqp = cqp_info->in.u.qp_modify.qp->qp_uk.back_qp; 1232 atomic_dec(&iwqp->hw_mod_qp_pend); 1233 wake_up(&iwqp->mod_qp_waitq); 1234 } 1235 1236 /** 1237 * irdma_hw_modify_qp - setup cqp for modify qp 1238 * @iwdev: RDMA device 1239 * @iwqp: qp ptr (user or kernel) 1240 * @info: info for modify qp 1241 * @wait: flag to wait or not for modify qp completion 1242 */ 1243 int irdma_hw_modify_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp, 1244 struct irdma_modify_qp_info *info, bool wait) 1245 { 1246 int status; 1247 struct irdma_pci_f *rf = iwdev->rf; 1248 struct irdma_cqp_request *cqp_request; 1249 struct cqp_cmds_info *cqp_info; 1250 struct irdma_modify_qp_info *m_info; 1251 1252 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait); 1253 if (!cqp_request) 1254 return -ENOMEM; 1255 1256 if (!wait) { 1257 cqp_request->callback_fcn = irdma_hw_modify_qp_callback; 1258 atomic_inc(&iwqp->hw_mod_qp_pend); 1259 } 1260 cqp_info = &cqp_request->info; 1261 m_info = &cqp_info->in.u.qp_modify.info; 1262 memcpy(m_info, info, sizeof(*m_info)); 1263 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY; 1264 cqp_info->post_sq = 1; 1265 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp; 1266 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request; 1267 status = irdma_handle_cqp_op(rf, cqp_request); 1268 irdma_put_cqp_request(&rf->cqp, cqp_request); 1269 if (status) { 1270 if (rdma_protocol_roce(&iwdev->ibdev, 1)) 1271 return status; 1272 1273 switch (m_info->next_iwarp_state) { 1274 struct irdma_gen_ae_info ae_info; 1275 1276 case IRDMA_QP_STATE_RTS: 1277 case IRDMA_QP_STATE_IDLE: 1278 case IRDMA_QP_STATE_TERMINATE: 1279 case IRDMA_QP_STATE_CLOSING: 1280 if (info->curr_iwarp_state == IRDMA_QP_STATE_IDLE) 1281 irdma_send_reset(iwqp->cm_node); 1282 else 1283 iwqp->sc_qp.term_flags = IRDMA_TERM_DONE; 1284 if (!wait) { 1285 ae_info.ae_code = IRDMA_AE_BAD_CLOSE; 1286 ae_info.ae_src = 0; 1287 irdma_gen_ae(rf, &iwqp->sc_qp, &ae_info, false); 1288 } else { 1289 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, 1290 wait); 1291 if (!cqp_request) 1292 return -ENOMEM; 1293 1294 cqp_info = &cqp_request->info; 1295 m_info = &cqp_info->in.u.qp_modify.info; 1296 memcpy(m_info, info, sizeof(*m_info)); 1297 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY; 1298 cqp_info->post_sq = 1; 1299 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp; 1300 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request; 1301 m_info->next_iwarp_state = IRDMA_QP_STATE_ERROR; 1302 m_info->reset_tcp_conn = true; 1303 irdma_handle_cqp_op(rf, cqp_request); 1304 irdma_put_cqp_request(&rf->cqp, cqp_request); 1305 } 1306 break; 1307 case IRDMA_QP_STATE_ERROR: 1308 default: 1309 break; 1310 } 1311 } 1312 1313 return status; 1314 } 1315 1316 /** 1317 * irdma_cqp_cq_destroy_cmd - destroy the cqp cq 1318 * @dev: device pointer 1319 * @cq: pointer to cq 1320 */ 1321 void irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq) 1322 { 1323 struct irdma_pci_f *rf = dev_to_rf(dev); 1324 1325 irdma_cq_wq_destroy(rf, cq); 1326 } 1327 1328 /** 1329 * irdma_cqp_qp_destroy_cmd - destroy the cqp 1330 * @dev: device pointer 1331 * @qp: pointer to qp 1332 */ 1333 int irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp) 1334 { 1335 struct irdma_pci_f *rf = dev_to_rf(dev); 1336 struct irdma_cqp *iwcqp = &rf->cqp; 1337 struct irdma_cqp_request *cqp_request; 1338 struct cqp_cmds_info *cqp_info; 1339 int status; 1340 1341 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true); 1342 if (!cqp_request) 1343 return -ENOMEM; 1344 1345 cqp_info = &cqp_request->info; 1346 memset(cqp_info, 0, sizeof(*cqp_info)); 1347 cqp_info->cqp_cmd = IRDMA_OP_QP_DESTROY; 1348 cqp_info->post_sq = 1; 1349 cqp_info->in.u.qp_destroy.qp = qp; 1350 cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request; 1351 cqp_info->in.u.qp_destroy.remove_hash_idx = true; 1352 1353 status = irdma_handle_cqp_op(rf, cqp_request); 1354 irdma_put_cqp_request(&rf->cqp, cqp_request); 1355 1356 return status; 1357 } 1358 1359 /** 1360 * irdma_ieq_mpa_crc_ae - generate AE for crc error 1361 * @dev: hardware control device structure 1362 * @qp: hardware control qp 1363 */ 1364 void irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp) 1365 { 1366 struct irdma_gen_ae_info info = {}; 1367 struct irdma_pci_f *rf = dev_to_rf(dev); 1368 1369 ibdev_dbg(&rf->iwdev->ibdev, "AEQ: Generate MPA CRC AE\n"); 1370 info.ae_code = IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR; 1371 info.ae_src = IRDMA_AE_SOURCE_RQ; 1372 irdma_gen_ae(rf, qp, &info, false); 1373 } 1374 1375 /** 1376 * irdma_ieq_check_mpacrc - check if mpa crc is OK 1377 * @addr: address of buffer for crc 1378 * @len: length of buffer 1379 * @val: value to be compared 1380 */ 1381 int irdma_ieq_check_mpacrc(const void *addr, u32 len, u32 val) 1382 { 1383 if ((__force u32)cpu_to_le32(~crc32c(~0, addr, len)) != val) 1384 return -EINVAL; 1385 1386 return 0; 1387 } 1388 1389 /** 1390 * irdma_ieq_get_qp - get qp based on quad in puda buffer 1391 * @dev: hardware control device structure 1392 * @buf: receive puda buffer on exception q 1393 */ 1394 struct irdma_sc_qp *irdma_ieq_get_qp(struct irdma_sc_dev *dev, 1395 struct irdma_puda_buf *buf) 1396 { 1397 struct irdma_qp *iwqp; 1398 struct irdma_cm_node *cm_node; 1399 struct irdma_device *iwdev = buf->vsi->back_vsi; 1400 u32 loc_addr[4] = {}; 1401 u32 rem_addr[4] = {}; 1402 u16 loc_port, rem_port; 1403 struct ipv6hdr *ip6h; 1404 struct iphdr *iph = (struct iphdr *)buf->iph; 1405 struct tcphdr *tcph = (struct tcphdr *)buf->tcph; 1406 1407 if (iph->version == 4) { 1408 loc_addr[0] = ntohl(iph->daddr); 1409 rem_addr[0] = ntohl(iph->saddr); 1410 } else { 1411 ip6h = (struct ipv6hdr *)buf->iph; 1412 irdma_copy_ip_ntohl(loc_addr, ip6h->daddr.in6_u.u6_addr32); 1413 irdma_copy_ip_ntohl(rem_addr, ip6h->saddr.in6_u.u6_addr32); 1414 } 1415 loc_port = ntohs(tcph->dest); 1416 rem_port = ntohs(tcph->source); 1417 cm_node = irdma_find_node(&iwdev->cm_core, rem_port, rem_addr, loc_port, 1418 loc_addr, buf->vlan_valid ? buf->vlan_id : 0xFFFF); 1419 if (!cm_node) 1420 return NULL; 1421 1422 iwqp = cm_node->iwqp; 1423 irdma_rem_ref_cm_node(cm_node); 1424 1425 return &iwqp->sc_qp; 1426 } 1427 1428 /** 1429 * irdma_send_ieq_ack - ACKs for duplicate or OOO partials FPDUs 1430 * @qp: qp ptr 1431 */ 1432 void irdma_send_ieq_ack(struct irdma_sc_qp *qp) 1433 { 1434 struct irdma_cm_node *cm_node = ((struct irdma_qp *)qp->qp_uk.back_qp)->cm_node; 1435 struct irdma_puda_buf *buf = qp->pfpdu.lastrcv_buf; 1436 struct tcphdr *tcph = (struct tcphdr *)buf->tcph; 1437 1438 cm_node->tcp_cntxt.rcv_nxt = qp->pfpdu.nextseqnum; 1439 cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq); 1440 1441 irdma_send_ack(cm_node); 1442 } 1443 1444 /** 1445 * irdma_puda_ieq_get_ah_info - get AH info from IEQ buffer 1446 * @qp: qp pointer 1447 * @ah_info: AH info pointer 1448 */ 1449 void irdma_puda_ieq_get_ah_info(struct irdma_sc_qp *qp, 1450 struct irdma_ah_info *ah_info) 1451 { 1452 struct irdma_puda_buf *buf = qp->pfpdu.ah_buf; 1453 struct iphdr *iph; 1454 struct ipv6hdr *ip6h; 1455 1456 memset(ah_info, 0, sizeof(*ah_info)); 1457 ah_info->do_lpbk = true; 1458 ah_info->vlan_tag = buf->vlan_id; 1459 ah_info->insert_vlan_tag = buf->vlan_valid; 1460 ah_info->ipv4_valid = buf->ipv4; 1461 ah_info->vsi = qp->vsi; 1462 1463 if (buf->smac_valid) 1464 ether_addr_copy(ah_info->mac_addr, buf->smac); 1465 1466 if (buf->ipv4) { 1467 ah_info->ipv4_valid = true; 1468 iph = (struct iphdr *)buf->iph; 1469 ah_info->hop_ttl = iph->ttl; 1470 ah_info->tc_tos = iph->tos; 1471 ah_info->dest_ip_addr[0] = ntohl(iph->daddr); 1472 ah_info->src_ip_addr[0] = ntohl(iph->saddr); 1473 } else { 1474 ip6h = (struct ipv6hdr *)buf->iph; 1475 ah_info->hop_ttl = ip6h->hop_limit; 1476 ah_info->tc_tos = ip6h->priority; 1477 irdma_copy_ip_ntohl(ah_info->dest_ip_addr, 1478 ip6h->daddr.in6_u.u6_addr32); 1479 irdma_copy_ip_ntohl(ah_info->src_ip_addr, 1480 ip6h->saddr.in6_u.u6_addr32); 1481 } 1482 1483 ah_info->dst_arpindex = irdma_arp_table(dev_to_rf(qp->dev), 1484 ah_info->dest_ip_addr, 1485 ah_info->ipv4_valid, 1486 NULL, IRDMA_ARP_RESOLVE); 1487 } 1488 1489 /** 1490 * irdma_gen1_ieq_update_tcpip_info - update tcpip in the buffer 1491 * @buf: puda to update 1492 * @len: length of buffer 1493 * @seqnum: seq number for tcp 1494 */ 1495 static void irdma_gen1_ieq_update_tcpip_info(struct irdma_puda_buf *buf, 1496 u16 len, u32 seqnum) 1497 { 1498 struct tcphdr *tcph; 1499 struct iphdr *iph; 1500 u16 iphlen; 1501 u16 pktsize; 1502 u8 *addr = buf->mem.va; 1503 1504 iphlen = (buf->ipv4) ? 20 : 40; 1505 iph = (struct iphdr *)(addr + buf->maclen); 1506 tcph = (struct tcphdr *)(addr + buf->maclen + iphlen); 1507 pktsize = len + buf->tcphlen + iphlen; 1508 iph->tot_len = htons(pktsize); 1509 tcph->seq = htonl(seqnum); 1510 } 1511 1512 /** 1513 * irdma_ieq_update_tcpip_info - update tcpip in the buffer 1514 * @buf: puda to update 1515 * @len: length of buffer 1516 * @seqnum: seq number for tcp 1517 */ 1518 void irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len, 1519 u32 seqnum) 1520 { 1521 struct tcphdr *tcph; 1522 u8 *addr; 1523 1524 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 1525 return irdma_gen1_ieq_update_tcpip_info(buf, len, seqnum); 1526 1527 addr = buf->mem.va; 1528 tcph = (struct tcphdr *)addr; 1529 tcph->seq = htonl(seqnum); 1530 } 1531 1532 /** 1533 * irdma_gen1_puda_get_tcpip_info - get tcpip info from puda 1534 * buffer 1535 * @info: to get information 1536 * @buf: puda buffer 1537 */ 1538 static int irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info, 1539 struct irdma_puda_buf *buf) 1540 { 1541 struct iphdr *iph; 1542 struct ipv6hdr *ip6h; 1543 struct tcphdr *tcph; 1544 u16 iphlen; 1545 u16 pkt_len; 1546 u8 *mem = buf->mem.va; 1547 struct ethhdr *ethh = buf->mem.va; 1548 1549 if (ethh->h_proto == htons(0x8100)) { 1550 info->vlan_valid = true; 1551 buf->vlan_id = ntohs(((struct vlan_ethhdr *)ethh)->h_vlan_TCI) & 1552 VLAN_VID_MASK; 1553 } 1554 1555 buf->maclen = (info->vlan_valid) ? 18 : 14; 1556 iphlen = (info->l3proto) ? 40 : 20; 1557 buf->ipv4 = (info->l3proto) ? false : true; 1558 buf->iph = mem + buf->maclen; 1559 iph = (struct iphdr *)buf->iph; 1560 buf->tcph = buf->iph + iphlen; 1561 tcph = (struct tcphdr *)buf->tcph; 1562 1563 if (buf->ipv4) { 1564 pkt_len = ntohs(iph->tot_len); 1565 } else { 1566 ip6h = (struct ipv6hdr *)buf->iph; 1567 pkt_len = ntohs(ip6h->payload_len) + iphlen; 1568 } 1569 1570 buf->totallen = pkt_len + buf->maclen; 1571 1572 if (info->payload_len < buf->totallen) { 1573 ibdev_dbg(to_ibdev(buf->vsi->dev), 1574 "ERR: payload_len = 0x%x totallen expected0x%x\n", 1575 info->payload_len, buf->totallen); 1576 return -EINVAL; 1577 } 1578 1579 buf->tcphlen = tcph->doff << 2; 1580 buf->datalen = pkt_len - iphlen - buf->tcphlen; 1581 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL; 1582 buf->hdrlen = buf->maclen + iphlen + buf->tcphlen; 1583 buf->seqnum = ntohl(tcph->seq); 1584 1585 return 0; 1586 } 1587 1588 /** 1589 * irdma_puda_get_tcpip_info - get tcpip info from puda buffer 1590 * @info: to get information 1591 * @buf: puda buffer 1592 */ 1593 int irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info, 1594 struct irdma_puda_buf *buf) 1595 { 1596 struct tcphdr *tcph; 1597 u32 pkt_len; 1598 u8 *mem; 1599 1600 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 1601 return irdma_gen1_puda_get_tcpip_info(info, buf); 1602 1603 mem = buf->mem.va; 1604 buf->vlan_valid = info->vlan_valid; 1605 if (info->vlan_valid) 1606 buf->vlan_id = info->vlan; 1607 1608 buf->ipv4 = info->ipv4; 1609 if (buf->ipv4) 1610 buf->iph = mem + IRDMA_IPV4_PAD; 1611 else 1612 buf->iph = mem; 1613 1614 buf->tcph = mem + IRDMA_TCP_OFFSET; 1615 tcph = (struct tcphdr *)buf->tcph; 1616 pkt_len = info->payload_len; 1617 buf->totallen = pkt_len; 1618 buf->tcphlen = tcph->doff << 2; 1619 buf->datalen = pkt_len - IRDMA_TCP_OFFSET - buf->tcphlen; 1620 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL; 1621 buf->hdrlen = IRDMA_TCP_OFFSET + buf->tcphlen; 1622 buf->seqnum = ntohl(tcph->seq); 1623 1624 if (info->smac_valid) { 1625 ether_addr_copy(buf->smac, info->smac); 1626 buf->smac_valid = true; 1627 } 1628 1629 return 0; 1630 } 1631 1632 /** 1633 * irdma_hw_stats_timeout - Stats timer-handler which updates all HW stats 1634 * @t: timer_list pointer 1635 */ 1636 static void irdma_hw_stats_timeout(struct timer_list *t) 1637 { 1638 struct irdma_vsi_pestat *pf_devstat = 1639 timer_container_of(pf_devstat, t, stats_timer); 1640 struct irdma_sc_vsi *sc_vsi = pf_devstat->vsi; 1641 1642 if (sc_vsi->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 1643 irdma_cqp_gather_stats_cmd(sc_vsi->dev, sc_vsi->pestat, false); 1644 else 1645 irdma_cqp_gather_stats_gen1(sc_vsi->dev, sc_vsi->pestat); 1646 1647 mod_timer(&pf_devstat->stats_timer, 1648 jiffies + msecs_to_jiffies(STATS_TIMER_DELAY)); 1649 } 1650 1651 /** 1652 * irdma_hw_stats_start_timer - Start periodic stats timer 1653 * @vsi: vsi structure pointer 1654 */ 1655 void irdma_hw_stats_start_timer(struct irdma_sc_vsi *vsi) 1656 { 1657 struct irdma_vsi_pestat *devstat = vsi->pestat; 1658 1659 timer_setup(&devstat->stats_timer, irdma_hw_stats_timeout, 0); 1660 mod_timer(&devstat->stats_timer, 1661 jiffies + msecs_to_jiffies(STATS_TIMER_DELAY)); 1662 } 1663 1664 /** 1665 * irdma_hw_stats_stop_timer - Delete periodic stats timer 1666 * @vsi: pointer to vsi structure 1667 */ 1668 void irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi) 1669 { 1670 struct irdma_vsi_pestat *devstat = vsi->pestat; 1671 1672 timer_delete_sync(&devstat->stats_timer); 1673 } 1674 1675 /** 1676 * irdma_process_stats - Checking for wrap and update stats 1677 * @pestat: stats structure pointer 1678 */ 1679 static inline void irdma_process_stats(struct irdma_vsi_pestat *pestat) 1680 { 1681 sc_vsi_update_stats(pestat->vsi); 1682 } 1683 1684 /** 1685 * irdma_cqp_gather_stats_gen1 - Gather stats 1686 * @dev: pointer to device structure 1687 * @pestat: statistics structure 1688 */ 1689 void irdma_cqp_gather_stats_gen1(struct irdma_sc_dev *dev, 1690 struct irdma_vsi_pestat *pestat) 1691 { 1692 struct irdma_gather_stats *gather_stats = 1693 pestat->gather_info.gather_stats_va; 1694 const struct irdma_hw_stat_map *map = dev->hw_stats_map; 1695 u16 max_stats_idx = dev->hw_attrs.max_stat_idx; 1696 u32 stats_inst_offset_32; 1697 u32 stats_inst_offset_64; 1698 u64 new_val; 1699 u16 i; 1700 1701 stats_inst_offset_32 = (pestat->gather_info.use_stats_inst) ? 1702 pestat->gather_info.stats_inst_index : 1703 pestat->hw->hmc.hmc_fn_id; 1704 stats_inst_offset_32 *= 4; 1705 stats_inst_offset_64 = stats_inst_offset_32 * 2; 1706 1707 for (i = 0; i < max_stats_idx; i++) { 1708 if (map[i].bitmask <= IRDMA_MAX_STATS_32) 1709 new_val = rd32(dev->hw, 1710 dev->hw_stats_regs[i] + stats_inst_offset_32); 1711 else 1712 new_val = rd64(dev->hw, 1713 dev->hw_stats_regs[i] + stats_inst_offset_64); 1714 gather_stats->val[map[i].byteoff / sizeof(u64)] = new_val; 1715 } 1716 1717 irdma_process_stats(pestat); 1718 } 1719 1720 /** 1721 * irdma_process_cqp_stats - Checking for wrap and update stats 1722 * @cqp_request: cqp_request structure pointer 1723 */ 1724 static void irdma_process_cqp_stats(struct irdma_cqp_request *cqp_request) 1725 { 1726 struct irdma_vsi_pestat *pestat = cqp_request->param; 1727 1728 irdma_process_stats(pestat); 1729 } 1730 1731 /** 1732 * irdma_cqp_gather_stats_cmd - Gather stats 1733 * @dev: pointer to device structure 1734 * @pestat: pointer to stats info 1735 * @wait: flag to wait or not wait for stats 1736 */ 1737 int irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev, 1738 struct irdma_vsi_pestat *pestat, bool wait) 1739 1740 { 1741 struct irdma_pci_f *rf = dev_to_rf(dev); 1742 struct irdma_cqp *iwcqp = &rf->cqp; 1743 struct irdma_cqp_request *cqp_request; 1744 struct cqp_cmds_info *cqp_info; 1745 int status; 1746 1747 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait); 1748 if (!cqp_request) 1749 return -ENOMEM; 1750 1751 cqp_info = &cqp_request->info; 1752 memset(cqp_info, 0, sizeof(*cqp_info)); 1753 cqp_info->cqp_cmd = IRDMA_OP_STATS_GATHER; 1754 cqp_info->post_sq = 1; 1755 cqp_info->in.u.stats_gather.info = pestat->gather_info; 1756 cqp_info->in.u.stats_gather.scratch = (uintptr_t)cqp_request; 1757 cqp_info->in.u.stats_gather.cqp = &rf->cqp.sc_cqp; 1758 cqp_request->param = pestat; 1759 if (!wait) 1760 cqp_request->callback_fcn = irdma_process_cqp_stats; 1761 status = irdma_handle_cqp_op(rf, cqp_request); 1762 if (wait) 1763 irdma_process_stats(pestat); 1764 irdma_put_cqp_request(&rf->cqp, cqp_request); 1765 1766 return status; 1767 } 1768 1769 /** 1770 * irdma_cqp_stats_inst_cmd - Allocate/free stats instance 1771 * @vsi: pointer to vsi structure 1772 * @cmd: command to allocate or free 1773 * @stats_info: pointer to allocate stats info 1774 */ 1775 int irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd, 1776 struct irdma_stats_inst_info *stats_info) 1777 { 1778 struct irdma_pci_f *rf = dev_to_rf(vsi->dev); 1779 struct irdma_cqp *iwcqp = &rf->cqp; 1780 struct irdma_cqp_request *cqp_request; 1781 struct cqp_cmds_info *cqp_info; 1782 int status; 1783 bool wait = false; 1784 1785 if (cmd == IRDMA_OP_STATS_ALLOCATE) 1786 wait = true; 1787 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait); 1788 if (!cqp_request) 1789 return -ENOMEM; 1790 1791 cqp_info = &cqp_request->info; 1792 memset(cqp_info, 0, sizeof(*cqp_info)); 1793 cqp_info->cqp_cmd = cmd; 1794 cqp_info->post_sq = 1; 1795 cqp_info->in.u.stats_manage.info = *stats_info; 1796 cqp_info->in.u.stats_manage.scratch = (uintptr_t)cqp_request; 1797 cqp_info->in.u.stats_manage.cqp = &rf->cqp.sc_cqp; 1798 status = irdma_handle_cqp_op(rf, cqp_request); 1799 if (wait) 1800 stats_info->stats_idx = cqp_request->compl_info.op_ret_val; 1801 irdma_put_cqp_request(iwcqp, cqp_request); 1802 1803 return status; 1804 } 1805 1806 /** 1807 * irdma_cqp_ceq_cmd - Create/Destroy CEQ's after CEQ 0 1808 * @dev: pointer to device info 1809 * @sc_ceq: pointer to ceq structure 1810 * @op: Create or Destroy 1811 */ 1812 int irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_ceq *sc_ceq, 1813 u8 op) 1814 { 1815 struct irdma_cqp_request *cqp_request; 1816 struct cqp_cmds_info *cqp_info; 1817 struct irdma_pci_f *rf = dev_to_rf(dev); 1818 int status; 1819 1820 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1821 if (!cqp_request) 1822 return -ENOMEM; 1823 1824 cqp_info = &cqp_request->info; 1825 cqp_info->post_sq = 1; 1826 cqp_info->cqp_cmd = op; 1827 cqp_info->in.u.ceq_create.ceq = sc_ceq; 1828 cqp_info->in.u.ceq_create.scratch = (uintptr_t)cqp_request; 1829 1830 status = irdma_handle_cqp_op(rf, cqp_request); 1831 irdma_put_cqp_request(&rf->cqp, cqp_request); 1832 1833 return status; 1834 } 1835 1836 /** 1837 * irdma_cqp_aeq_cmd - Create/Destroy AEQ 1838 * @dev: pointer to device info 1839 * @sc_aeq: pointer to aeq structure 1840 * @op: Create or Destroy 1841 */ 1842 int irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_aeq *sc_aeq, 1843 u8 op) 1844 { 1845 struct irdma_cqp_request *cqp_request; 1846 struct cqp_cmds_info *cqp_info; 1847 struct irdma_pci_f *rf = dev_to_rf(dev); 1848 int status; 1849 1850 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1851 if (!cqp_request) 1852 return -ENOMEM; 1853 1854 cqp_info = &cqp_request->info; 1855 cqp_info->post_sq = 1; 1856 cqp_info->cqp_cmd = op; 1857 cqp_info->in.u.aeq_create.aeq = sc_aeq; 1858 cqp_info->in.u.aeq_create.scratch = (uintptr_t)cqp_request; 1859 1860 status = irdma_handle_cqp_op(rf, cqp_request); 1861 irdma_put_cqp_request(&rf->cqp, cqp_request); 1862 1863 return status; 1864 } 1865 1866 /** 1867 * irdma_cqp_ws_node_cmd - Add/modify/delete ws node 1868 * @dev: pointer to device structure 1869 * @cmd: Add, modify or delete 1870 * @node_info: pointer to ws node info 1871 */ 1872 int irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd, 1873 struct irdma_ws_node_info *node_info) 1874 { 1875 struct irdma_pci_f *rf = dev_to_rf(dev); 1876 struct irdma_cqp *iwcqp = &rf->cqp; 1877 struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp; 1878 struct irdma_cqp_request *cqp_request; 1879 struct cqp_cmds_info *cqp_info; 1880 int status; 1881 bool poll; 1882 1883 if (!rf->sc_dev.ceq_valid) 1884 poll = true; 1885 else 1886 poll = false; 1887 1888 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, !poll); 1889 if (!cqp_request) 1890 return -ENOMEM; 1891 1892 cqp_info = &cqp_request->info; 1893 memset(cqp_info, 0, sizeof(*cqp_info)); 1894 cqp_info->cqp_cmd = cmd; 1895 cqp_info->post_sq = 1; 1896 cqp_info->in.u.ws_node.info = *node_info; 1897 cqp_info->in.u.ws_node.cqp = cqp; 1898 cqp_info->in.u.ws_node.scratch = (uintptr_t)cqp_request; 1899 status = irdma_handle_cqp_op(rf, cqp_request); 1900 if (status) 1901 goto exit; 1902 1903 if (poll) { 1904 struct irdma_ccq_cqe_info compl_info; 1905 1906 status = irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_WORK_SCHED_NODE, 1907 &compl_info); 1908 node_info->qs_handle = compl_info.op_ret_val; 1909 ibdev_dbg(&rf->iwdev->ibdev, "DCB: opcode=%d, compl_info.retval=%d\n", 1910 compl_info.op_code, compl_info.op_ret_val); 1911 } else { 1912 node_info->qs_handle = cqp_request->compl_info.op_ret_val; 1913 } 1914 1915 exit: 1916 irdma_put_cqp_request(&rf->cqp, cqp_request); 1917 1918 return status; 1919 } 1920 1921 /** 1922 * irdma_ah_cqp_op - perform an AH cqp operation 1923 * @rf: RDMA PCI function 1924 * @sc_ah: address handle 1925 * @cmd: AH operation 1926 * @wait: wait if true 1927 * @callback_fcn: Callback function on CQP op completion 1928 * @cb_param: parameter for callback function 1929 * 1930 * returns errno 1931 */ 1932 int irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd, 1933 bool wait, 1934 void (*callback_fcn)(struct irdma_cqp_request *), 1935 void *cb_param) 1936 { 1937 struct irdma_cqp_request *cqp_request; 1938 struct cqp_cmds_info *cqp_info; 1939 int status; 1940 1941 if (cmd != IRDMA_OP_AH_CREATE && cmd != IRDMA_OP_AH_DESTROY) 1942 return -EINVAL; 1943 1944 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait); 1945 if (!cqp_request) 1946 return -ENOMEM; 1947 1948 cqp_info = &cqp_request->info; 1949 cqp_info->cqp_cmd = cmd; 1950 cqp_info->post_sq = 1; 1951 if (cmd == IRDMA_OP_AH_CREATE) { 1952 cqp_info->in.u.ah_create.info = sc_ah->ah_info; 1953 cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request; 1954 cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp; 1955 } else if (cmd == IRDMA_OP_AH_DESTROY) { 1956 cqp_info->in.u.ah_destroy.info = sc_ah->ah_info; 1957 cqp_info->in.u.ah_destroy.scratch = (uintptr_t)cqp_request; 1958 cqp_info->in.u.ah_destroy.cqp = &rf->cqp.sc_cqp; 1959 } 1960 1961 if (!wait) { 1962 cqp_request->callback_fcn = callback_fcn; 1963 cqp_request->param = cb_param; 1964 } 1965 status = irdma_handle_cqp_op(rf, cqp_request); 1966 irdma_put_cqp_request(&rf->cqp, cqp_request); 1967 1968 if (status) 1969 return -ENOMEM; 1970 1971 if (wait) 1972 sc_ah->ah_info.ah_valid = (cmd == IRDMA_OP_AH_CREATE); 1973 1974 return 0; 1975 } 1976 1977 /** 1978 * irdma_ieq_ah_cb - callback after creation of AH for IEQ 1979 * @cqp_request: pointer to cqp_request of create AH 1980 */ 1981 static void irdma_ieq_ah_cb(struct irdma_cqp_request *cqp_request) 1982 { 1983 struct irdma_sc_qp *qp = cqp_request->param; 1984 struct irdma_sc_ah *sc_ah = qp->pfpdu.ah; 1985 unsigned long flags; 1986 1987 spin_lock_irqsave(&qp->pfpdu.lock, flags); 1988 if (!cqp_request->compl_info.op_ret_val) { 1989 sc_ah->ah_info.ah_valid = true; 1990 irdma_ieq_process_fpdus(qp, qp->vsi->ieq); 1991 } else { 1992 sc_ah->ah_info.ah_valid = false; 1993 irdma_ieq_cleanup_qp(qp->vsi->ieq, qp); 1994 } 1995 spin_unlock_irqrestore(&qp->pfpdu.lock, flags); 1996 } 1997 1998 /** 1999 * irdma_ilq_ah_cb - callback after creation of AH for ILQ 2000 * @cqp_request: pointer to cqp_request of create AH 2001 */ 2002 static void irdma_ilq_ah_cb(struct irdma_cqp_request *cqp_request) 2003 { 2004 struct irdma_cm_node *cm_node = cqp_request->param; 2005 struct irdma_sc_ah *sc_ah = cm_node->ah; 2006 2007 sc_ah->ah_info.ah_valid = !cqp_request->compl_info.op_ret_val; 2008 irdma_add_conn_est_qh(cm_node); 2009 } 2010 2011 /** 2012 * irdma_puda_create_ah - create AH for ILQ/IEQ qp's 2013 * @dev: device pointer 2014 * @ah_info: Address handle info 2015 * @wait: When true will wait for operation to complete 2016 * @type: ILQ/IEQ 2017 * @cb_param: Callback param when not waiting 2018 * @ah_ret: Returned pointer to address handle if created 2019 * 2020 */ 2021 int irdma_puda_create_ah(struct irdma_sc_dev *dev, 2022 struct irdma_ah_info *ah_info, bool wait, 2023 enum puda_rsrc_type type, void *cb_param, 2024 struct irdma_sc_ah **ah_ret) 2025 { 2026 struct irdma_sc_ah *ah; 2027 struct irdma_pci_f *rf = dev_to_rf(dev); 2028 int err; 2029 2030 ah = kzalloc(sizeof(*ah), GFP_ATOMIC); 2031 *ah_ret = ah; 2032 if (!ah) 2033 return -ENOMEM; 2034 2035 err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah, 2036 &ah_info->ah_idx, &rf->next_ah); 2037 if (err) 2038 goto err_free; 2039 2040 ah->dev = dev; 2041 ah->ah_info = *ah_info; 2042 2043 if (type == IRDMA_PUDA_RSRC_TYPE_ILQ) 2044 err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait, 2045 irdma_ilq_ah_cb, cb_param); 2046 else 2047 err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait, 2048 irdma_ieq_ah_cb, cb_param); 2049 2050 if (err) 2051 goto error; 2052 return 0; 2053 2054 error: 2055 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx); 2056 err_free: 2057 kfree(ah); 2058 *ah_ret = NULL; 2059 return -ENOMEM; 2060 } 2061 2062 /** 2063 * irdma_puda_free_ah - free a puda address handle 2064 * @dev: device pointer 2065 * @ah: The address handle to free 2066 */ 2067 void irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah) 2068 { 2069 struct irdma_pci_f *rf = dev_to_rf(dev); 2070 2071 if (!ah) 2072 return; 2073 2074 if (ah->ah_info.ah_valid) { 2075 irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_DESTROY, false, NULL, NULL); 2076 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx); 2077 } 2078 2079 kfree(ah); 2080 } 2081 2082 /** 2083 * irdma_gsi_ud_qp_ah_cb - callback after creation of AH for GSI/ID QP 2084 * @cqp_request: pointer to cqp_request of create AH 2085 */ 2086 void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request) 2087 { 2088 struct irdma_sc_ah *sc_ah = cqp_request->param; 2089 2090 if (!cqp_request->compl_info.op_ret_val) 2091 sc_ah->ah_info.ah_valid = true; 2092 else 2093 sc_ah->ah_info.ah_valid = false; 2094 } 2095 2096 /** 2097 * irdma_prm_add_pble_mem - add moemory to pble resources 2098 * @pprm: pble resource manager 2099 * @pchunk: chunk of memory to add 2100 */ 2101 int irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm, 2102 struct irdma_chunk *pchunk) 2103 { 2104 u64 sizeofbitmap; 2105 2106 if (pchunk->size & 0xfff) 2107 return -EINVAL; 2108 2109 sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift; 2110 2111 pchunk->bitmapbuf = bitmap_zalloc(sizeofbitmap, GFP_KERNEL); 2112 if (!pchunk->bitmapbuf) 2113 return -ENOMEM; 2114 2115 pchunk->sizeofbitmap = sizeofbitmap; 2116 /* each pble is 8 bytes hence shift by 3 */ 2117 pprm->total_pble_alloc += pchunk->size >> 3; 2118 pprm->free_pble_cnt += pchunk->size >> 3; 2119 2120 return 0; 2121 } 2122 2123 /** 2124 * irdma_prm_get_pbles - get pble's from prm 2125 * @pprm: pble resource manager 2126 * @chunkinfo: nformation about chunk where pble's were acquired 2127 * @mem_size: size of pble memory needed 2128 * @vaddr: returns virtual address of pble memory 2129 * @fpm_addr: returns fpm address of pble memory 2130 */ 2131 int irdma_prm_get_pbles(struct irdma_pble_prm *pprm, 2132 struct irdma_pble_chunkinfo *chunkinfo, u64 mem_size, 2133 u64 **vaddr, u64 *fpm_addr) 2134 { 2135 u64 bits_needed; 2136 u64 bit_idx = PBLE_INVALID_IDX; 2137 struct irdma_chunk *pchunk = NULL; 2138 struct list_head *chunk_entry = pprm->clist.next; 2139 u32 offset; 2140 unsigned long flags; 2141 *vaddr = NULL; 2142 *fpm_addr = 0; 2143 2144 bits_needed = DIV_ROUND_UP_ULL(mem_size, BIT_ULL(pprm->pble_shift)); 2145 2146 spin_lock_irqsave(&pprm->prm_lock, flags); 2147 while (chunk_entry != &pprm->clist) { 2148 pchunk = (struct irdma_chunk *)chunk_entry; 2149 bit_idx = bitmap_find_next_zero_area(pchunk->bitmapbuf, 2150 pchunk->sizeofbitmap, 0, 2151 bits_needed, 0); 2152 if (bit_idx < pchunk->sizeofbitmap) 2153 break; 2154 2155 /* list.next used macro */ 2156 chunk_entry = pchunk->list.next; 2157 } 2158 2159 if (!pchunk || bit_idx >= pchunk->sizeofbitmap) { 2160 spin_unlock_irqrestore(&pprm->prm_lock, flags); 2161 return -ENOMEM; 2162 } 2163 2164 bitmap_set(pchunk->bitmapbuf, bit_idx, bits_needed); 2165 offset = bit_idx << pprm->pble_shift; 2166 *vaddr = pchunk->vaddr + offset; 2167 *fpm_addr = pchunk->fpm_addr + offset; 2168 2169 chunkinfo->pchunk = pchunk; 2170 chunkinfo->bit_idx = bit_idx; 2171 chunkinfo->bits_used = bits_needed; 2172 /* 3 is sizeof pble divide */ 2173 pprm->free_pble_cnt -= chunkinfo->bits_used << (pprm->pble_shift - 3); 2174 spin_unlock_irqrestore(&pprm->prm_lock, flags); 2175 2176 return 0; 2177 } 2178 2179 /** 2180 * irdma_prm_return_pbles - return pbles back to prm 2181 * @pprm: pble resource manager 2182 * @chunkinfo: chunk where pble's were acquired and to be freed 2183 */ 2184 void irdma_prm_return_pbles(struct irdma_pble_prm *pprm, 2185 struct irdma_pble_chunkinfo *chunkinfo) 2186 { 2187 unsigned long flags; 2188 2189 spin_lock_irqsave(&pprm->prm_lock, flags); 2190 pprm->free_pble_cnt += chunkinfo->bits_used << (pprm->pble_shift - 3); 2191 bitmap_clear(chunkinfo->pchunk->bitmapbuf, chunkinfo->bit_idx, 2192 chunkinfo->bits_used); 2193 spin_unlock_irqrestore(&pprm->prm_lock, flags); 2194 } 2195 2196 int irdma_map_vm_page_list(struct irdma_hw *hw, void *va, dma_addr_t *pg_dma, 2197 u32 pg_cnt) 2198 { 2199 struct page *vm_page; 2200 int i; 2201 u8 *addr; 2202 2203 addr = (u8 *)(uintptr_t)va; 2204 for (i = 0; i < pg_cnt; i++) { 2205 vm_page = vmalloc_to_page(addr); 2206 if (!vm_page) 2207 goto err; 2208 2209 pg_dma[i] = dma_map_page(hw->device, vm_page, 0, PAGE_SIZE, 2210 DMA_BIDIRECTIONAL); 2211 if (dma_mapping_error(hw->device, pg_dma[i])) 2212 goto err; 2213 2214 addr += PAGE_SIZE; 2215 } 2216 2217 return 0; 2218 2219 err: 2220 irdma_unmap_vm_page_list(hw, pg_dma, i); 2221 return -ENOMEM; 2222 } 2223 2224 void irdma_unmap_vm_page_list(struct irdma_hw *hw, dma_addr_t *pg_dma, u32 pg_cnt) 2225 { 2226 int i; 2227 2228 for (i = 0; i < pg_cnt; i++) 2229 dma_unmap_page(hw->device, pg_dma[i], PAGE_SIZE, DMA_BIDIRECTIONAL); 2230 } 2231 2232 /** 2233 * irdma_pble_free_paged_mem - free virtual paged memory 2234 * @chunk: chunk to free with paged memory 2235 */ 2236 void irdma_pble_free_paged_mem(struct irdma_chunk *chunk) 2237 { 2238 if (!chunk->pg_cnt) 2239 goto done; 2240 2241 irdma_unmap_vm_page_list(chunk->dev->hw, chunk->dmainfo.dmaaddrs, 2242 chunk->pg_cnt); 2243 2244 done: 2245 kfree(chunk->dmainfo.dmaaddrs); 2246 chunk->dmainfo.dmaaddrs = NULL; 2247 vfree(chunk->vaddr); 2248 chunk->vaddr = NULL; 2249 chunk->type = 0; 2250 } 2251 2252 /** 2253 * irdma_pble_get_paged_mem -allocate paged memory for pbles 2254 * @chunk: chunk to add for paged memory 2255 * @pg_cnt: number of pages needed 2256 */ 2257 int irdma_pble_get_paged_mem(struct irdma_chunk *chunk, u32 pg_cnt) 2258 { 2259 u32 size; 2260 void *va; 2261 2262 chunk->dmainfo.dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL); 2263 if (!chunk->dmainfo.dmaaddrs) 2264 return -ENOMEM; 2265 2266 size = PAGE_SIZE * pg_cnt; 2267 va = vmalloc(size); 2268 if (!va) 2269 goto err; 2270 2271 if (irdma_map_vm_page_list(chunk->dev->hw, va, chunk->dmainfo.dmaaddrs, 2272 pg_cnt)) { 2273 vfree(va); 2274 goto err; 2275 } 2276 chunk->vaddr = va; 2277 chunk->size = size; 2278 chunk->pg_cnt = pg_cnt; 2279 chunk->type = PBLE_SD_PAGED; 2280 2281 return 0; 2282 err: 2283 kfree(chunk->dmainfo.dmaaddrs); 2284 chunk->dmainfo.dmaaddrs = NULL; 2285 2286 return -ENOMEM; 2287 } 2288 2289 /** 2290 * irdma_alloc_ws_node_id - Allocate a tx scheduler node ID 2291 * @dev: device pointer 2292 */ 2293 u16 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev) 2294 { 2295 struct irdma_pci_f *rf = dev_to_rf(dev); 2296 u32 next = 1; 2297 u32 node_id; 2298 2299 if (irdma_alloc_rsrc(rf, rf->allocated_ws_nodes, rf->max_ws_node_id, 2300 &node_id, &next)) 2301 return IRDMA_WS_NODE_INVALID; 2302 2303 return (u16)node_id; 2304 } 2305 2306 /** 2307 * irdma_free_ws_node_id - Free a tx scheduler node ID 2308 * @dev: device pointer 2309 * @node_id: Work scheduler node ID 2310 */ 2311 void irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id) 2312 { 2313 struct irdma_pci_f *rf = dev_to_rf(dev); 2314 2315 irdma_free_rsrc(rf, rf->allocated_ws_nodes, (u32)node_id); 2316 } 2317 2318 /** 2319 * irdma_modify_qp_to_err - Modify a QP to error 2320 * @sc_qp: qp structure 2321 */ 2322 void irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp) 2323 { 2324 struct irdma_qp *qp = sc_qp->qp_uk.back_qp; 2325 struct ib_qp_attr attr; 2326 2327 if (qp->iwdev->rf->reset) 2328 return; 2329 attr.qp_state = IB_QPS_ERR; 2330 2331 if (rdma_protocol_roce(qp->ibqp.device, 1)) 2332 irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL); 2333 else 2334 irdma_modify_qp(&qp->ibqp, &attr, IB_QP_STATE, NULL); 2335 } 2336 2337 void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event) 2338 { 2339 struct ib_event ibevent; 2340 2341 if (!iwqp->ibqp.event_handler) 2342 return; 2343 2344 switch (event) { 2345 case IRDMA_QP_EVENT_CATASTROPHIC: 2346 ibevent.event = IB_EVENT_QP_FATAL; 2347 break; 2348 case IRDMA_QP_EVENT_ACCESS_ERR: 2349 ibevent.event = IB_EVENT_QP_ACCESS_ERR; 2350 break; 2351 case IRDMA_QP_EVENT_REQ_ERR: 2352 ibevent.event = IB_EVENT_QP_REQ_ERR; 2353 break; 2354 } 2355 ibevent.device = iwqp->ibqp.device; 2356 ibevent.element.qp = &iwqp->ibqp; 2357 iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context); 2358 } 2359 2360 bool irdma_cq_empty(struct irdma_cq *iwcq) 2361 { 2362 struct irdma_cq_uk *ukcq; 2363 u64 qword3; 2364 __le64 *cqe; 2365 u8 polarity; 2366 2367 ukcq = &iwcq->sc_cq.cq_uk; 2368 if (ukcq->avoid_mem_cflct) 2369 cqe = IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(ukcq); 2370 else 2371 cqe = IRDMA_GET_CURRENT_CQ_ELEM(ukcq); 2372 get_64bit_val(cqe, 24, &qword3); 2373 polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3); 2374 2375 return polarity != ukcq->polarity; 2376 } 2377 2378 void irdma_remove_cmpls_list(struct irdma_cq *iwcq) 2379 { 2380 struct irdma_cmpl_gen *cmpl_node; 2381 struct list_head *tmp_node, *list_node; 2382 2383 list_for_each_safe (list_node, tmp_node, &iwcq->cmpl_generated) { 2384 cmpl_node = list_entry(list_node, struct irdma_cmpl_gen, list); 2385 list_del(&cmpl_node->list); 2386 kfree(cmpl_node); 2387 } 2388 } 2389 2390 int irdma_generated_cmpls(struct irdma_cq *iwcq, struct irdma_cq_poll_info *cq_poll_info) 2391 { 2392 struct irdma_cmpl_gen *cmpl; 2393 2394 if (list_empty(&iwcq->cmpl_generated)) 2395 return -ENOENT; 2396 cmpl = list_first_entry_or_null(&iwcq->cmpl_generated, struct irdma_cmpl_gen, list); 2397 list_del(&cmpl->list); 2398 memcpy(cq_poll_info, &cmpl->cpi, sizeof(*cq_poll_info)); 2399 kfree(cmpl); 2400 2401 ibdev_dbg(iwcq->ibcq.device, 2402 "VERBS: %s: Poll artificially generated completion for QP 0x%X, op %u, wr_id=0x%llx\n", 2403 __func__, cq_poll_info->qp_id, cq_poll_info->op_type, 2404 cq_poll_info->wr_id); 2405 2406 return 0; 2407 } 2408 2409 /** 2410 * irdma_set_cpi_common_values - fill in values for polling info struct 2411 * @cpi: resulting structure of cq_poll_info type 2412 * @qp: QPair 2413 * @qp_num: id of the QP 2414 */ 2415 static void irdma_set_cpi_common_values(struct irdma_cq_poll_info *cpi, 2416 struct irdma_qp_uk *qp, u32 qp_num) 2417 { 2418 cpi->comp_status = IRDMA_COMPL_STATUS_FLUSHED; 2419 cpi->error = true; 2420 cpi->major_err = IRDMA_FLUSH_MAJOR_ERR; 2421 cpi->minor_err = FLUSH_GENERAL_ERR; 2422 cpi->qp_handle = (irdma_qp_handle)(uintptr_t)qp; 2423 cpi->qp_id = qp_num; 2424 } 2425 2426 static inline void irdma_comp_handler(struct irdma_cq *cq) 2427 { 2428 if (!cq->ibcq.comp_handler) 2429 return; 2430 if (atomic_cmpxchg(&cq->armed, 1, 0)) 2431 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); 2432 } 2433 2434 void irdma_generate_flush_completions(struct irdma_qp *iwqp) 2435 { 2436 struct irdma_qp_uk *qp = &iwqp->sc_qp.qp_uk; 2437 struct irdma_ring *sq_ring = &qp->sq_ring; 2438 struct irdma_ring *rq_ring = &qp->rq_ring; 2439 struct irdma_cmpl_gen *cmpl; 2440 __le64 *sw_wqe; 2441 u64 wqe_qword; 2442 u32 wqe_idx; 2443 bool compl_generated = false; 2444 unsigned long flags1; 2445 2446 spin_lock_irqsave(&iwqp->iwscq->lock, flags1); 2447 if (irdma_cq_empty(iwqp->iwscq)) { 2448 unsigned long flags2; 2449 2450 spin_lock_irqsave(&iwqp->lock, flags2); 2451 while (IRDMA_RING_MORE_WORK(*sq_ring)) { 2452 cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC); 2453 if (!cmpl) { 2454 spin_unlock_irqrestore(&iwqp->lock, flags2); 2455 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); 2456 return; 2457 } 2458 2459 wqe_idx = sq_ring->tail; 2460 irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id); 2461 2462 cmpl->cpi.wr_id = qp->sq_wrtrk_array[wqe_idx].wrid; 2463 sw_wqe = qp->sq_base[wqe_idx].elem; 2464 get_64bit_val(sw_wqe, 24, &wqe_qword); 2465 cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, IRDMAQPSQ_OPCODE); 2466 cmpl->cpi.q_type = IRDMA_CQE_QTYPE_SQ; 2467 /* remove the SQ WR by moving SQ tail*/ 2468 IRDMA_RING_SET_TAIL(*sq_ring, 2469 sq_ring->tail + qp->sq_wrtrk_array[sq_ring->tail].quanta); 2470 if (cmpl->cpi.op_type == IRDMAQP_OP_NOP) { 2471 kfree(cmpl); 2472 continue; 2473 } 2474 ibdev_dbg(iwqp->iwscq->ibcq.device, 2475 "DEV: %s: adding wr_id = 0x%llx SQ Completion to list qp_id=%d\n", 2476 __func__, cmpl->cpi.wr_id, qp->qp_id); 2477 list_add_tail(&cmpl->list, &iwqp->iwscq->cmpl_generated); 2478 compl_generated = true; 2479 } 2480 spin_unlock_irqrestore(&iwqp->lock, flags2); 2481 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); 2482 if (compl_generated) 2483 irdma_comp_handler(iwqp->iwscq); 2484 } else { 2485 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); 2486 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, 2487 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS)); 2488 } 2489 2490 spin_lock_irqsave(&iwqp->iwrcq->lock, flags1); 2491 if (irdma_cq_empty(iwqp->iwrcq)) { 2492 unsigned long flags2; 2493 2494 spin_lock_irqsave(&iwqp->lock, flags2); 2495 while (IRDMA_RING_MORE_WORK(*rq_ring)) { 2496 cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC); 2497 if (!cmpl) { 2498 spin_unlock_irqrestore(&iwqp->lock, flags2); 2499 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); 2500 return; 2501 } 2502 2503 wqe_idx = rq_ring->tail; 2504 irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id); 2505 2506 cmpl->cpi.wr_id = qp->rq_wrid_array[wqe_idx]; 2507 cmpl->cpi.op_type = IRDMA_OP_TYPE_REC; 2508 cmpl->cpi.q_type = IRDMA_CQE_QTYPE_RQ; 2509 /* remove the RQ WR by moving RQ tail */ 2510 IRDMA_RING_SET_TAIL(*rq_ring, rq_ring->tail + 1); 2511 ibdev_dbg(iwqp->iwrcq->ibcq.device, 2512 "DEV: %s: adding wr_id = 0x%llx RQ Completion to list qp_id=%d, wqe_idx=%d\n", 2513 __func__, cmpl->cpi.wr_id, qp->qp_id, 2514 wqe_idx); 2515 list_add_tail(&cmpl->list, &iwqp->iwrcq->cmpl_generated); 2516 2517 compl_generated = true; 2518 } 2519 spin_unlock_irqrestore(&iwqp->lock, flags2); 2520 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); 2521 if (compl_generated) 2522 irdma_comp_handler(iwqp->iwrcq); 2523 } else { 2524 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); 2525 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, 2526 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS)); 2527 } 2528 } 2529