1 /*- 2 * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 3 * 4 * Copyright (c) 2015 - 2023 Intel Corporation 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenFabrics.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #include "irdma_main.h" 36 37 LIST_HEAD(irdma_handlers); 38 DEFINE_SPINLOCK(irdma_handler_lock); 39 40 static const struct ae_desc ae_desc_list[] = { 41 {IRDMA_AE_AMP_UNALLOCATED_STAG, "Unallocated memory key (L-Key/R-Key)"}, 42 {IRDMA_AE_AMP_INVALID_STAG, "Invalid memory key (L-Key/R-Key)"}, 43 {IRDMA_AE_AMP_BAD_QP, 44 "Memory protection error: Accessing Memory Window (MW) which belongs to a different QP"}, 45 {IRDMA_AE_AMP_BAD_PD, 46 "Memory protection error: Accessing Memory Window (MW)/Memory Region (MR) which belongs to a different PD"}, 47 {IRDMA_AE_AMP_BAD_STAG_KEY, "Bad memory key (L-Key/R-Key)"}, 48 {IRDMA_AE_AMP_BAD_STAG_INDEX, "Bad memory key (L-Key/R-Key): Too large memory key index"}, 49 {IRDMA_AE_AMP_BOUNDS_VIOLATION, "Memory Window (MW)/Memory Region (MR) bounds violation"}, 50 {IRDMA_AE_AMP_RIGHTS_VIOLATION, "Memory Window (MW)/Memory Region (MR) rights violation"}, 51 {IRDMA_AE_AMP_TO_WRAP, 52 "Memory protection error: The address within Memory Window (MW)/Memory Region (MR) wraps"}, 53 {IRDMA_AE_AMP_FASTREG_VALID_STAG, 54 "Fastreg error: Registration to a valid MR"}, 55 {IRDMA_AE_AMP_FASTREG_MW_STAG, 56 "Fastreg error: Registration to a valid Memory Window (MW)"}, 57 {IRDMA_AE_AMP_FASTREG_INVALID_RIGHTS, "Fastreg error: Invalid rights"}, 58 {IRDMA_AE_AMP_FASTREG_INVALID_LENGTH, "Fastreg error: Invalid length"}, 59 {IRDMA_AE_AMP_INVALIDATE_SHARED, "Attempt to invalidate a shared MR"}, 60 {IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS, 61 "Attempt to remotely invalidate Memory Window (MW)/Memory Region (MR) without rights"}, 62 {IRDMA_AE_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS, 63 "Attempt to invalidate MR with a bound Memory Window (MW)"}, 64 {IRDMA_AE_AMP_MWBIND_VALID_STAG, 65 "Attempt to bind an Memory Window (MW) with a valid MW memory key (L-Key/R-Key)"}, 66 {IRDMA_AE_AMP_MWBIND_OF_MR_STAG, 67 "Attempt to bind an Memory Window (MW) with an MR memory key (L-Key/R-Key)"}, 68 {IRDMA_AE_AMP_MWBIND_TO_ZERO_BASED_STAG, 69 "Attempt to bind an Memory Window (MW) to a zero based MR"}, 70 {IRDMA_AE_AMP_MWBIND_TO_MW_STAG, 71 "Attempt to bind an Memory Window (MW) using MW memory key (L-Key/R-Key) instead of MR memory key (L-Key/R-Key)"}, 72 {IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS, "Memory Window (MW) bind error: Invalid rights"}, 73 {IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS, "Memory Window (MW) bind error: Invalid bounds"}, 74 {IRDMA_AE_AMP_MWBIND_TO_INVALID_PARENT, 75 "Memory Window (MW) bind error: Invalid parent MR"}, 76 {IRDMA_AE_AMP_MWBIND_BIND_DISABLED, 77 "Memory Window (MW) bind error: Disabled bind support"}, 78 {IRDMA_AE_PRIV_OPERATION_DENIED, 79 "Denying a privileged operation on a non-privileged QP"}, 80 {IRDMA_AE_AMP_INVALIDATE_TYPE1_MW, "Memory Window (MW) error: Invalidate type 1 MW"}, 81 {IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW, 82 "Memory Window (MW) bind error: Zero-based addressing for type 1 MW"}, 83 {IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG, 84 "Fastreg error: Invalid host page size config"}, 85 {IRDMA_AE_AMP_MWBIND_WRONG_TYPE, "MB bind error: Wrong Memory Window (MW) type"}, 86 {IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH, 87 "Fastreg error: Invalid request to change physical MR to virtual or vice versa"}, 88 {IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG, 89 "Userspace Direct Access (UDA) QP xmit error: Packet length exceeds the QP MTU"}, 90 {IRDMA_AE_UDA_XMIT_BAD_PD, 91 "Userspace Direct Access (UDA) QP xmit error: Attempt to access a different PD"}, 92 {IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT, 93 "Userspace Direct Access (UDA) QP xmit error: Too short packet length"}, 94 {IRDMA_AE_UDA_L4LEN_INVALID, 95 "Userspace Direct Access (UDA) error: Invalid packet length field"}, 96 {IRDMA_AE_BAD_CLOSE, 97 "iWARP error: Data is received when QP state is closing"}, 98 {IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE, 99 "iWARP error: FIN is received when xmit data is pending"}, 100 {IRDMA_AE_CQ_OPERATION_ERROR, "CQ overflow"}, 101 {IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO, 102 "QP error: Attempted RDMA Read when the outbound RDMA Read queue depth is zero"}, 103 {IRDMA_AE_STAG_ZERO_INVALID, 104 "Zero invalid memory key (L-Key/R-Key) on inbound RDMA R/W"}, 105 {IRDMA_AE_IB_RREQ_AND_Q1_FULL, 106 "QP error: Received RDMA Read request when the inbound RDMA Read queue is full"}, 107 {IRDMA_AE_IB_INVALID_REQUEST, 108 "QP error: Invalid operation detected by the remote peer"}, 109 {IRDMA_AE_WQE_UNEXPECTED_OPCODE, 110 "QP error: Invalid opcode in SQ WQE"}, 111 {IRDMA_AE_WQE_INVALID_PARAMETER, 112 "QP error: Invalid parameter in a WQE"}, 113 {IRDMA_AE_WQE_INVALID_FRAG_DATA, 114 "QP error: Invalid fragment in a WQE"}, 115 {IRDMA_AE_IB_REMOTE_ACCESS_ERROR, 116 "RoCEv2 error: Remote access error"}, 117 {IRDMA_AE_IB_REMOTE_OP_ERROR, 118 "RoCEv2 error: Remote operation error"}, 119 {IRDMA_AE_WQE_LSMM_TOO_LONG, "iWARP error: Connection error"}, 120 {IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN, 121 "iWARP error: Invalid message sequence number"}, 122 {IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER, 123 "iWARP error: Inbound message is too long for the available buffer"}, 124 {IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION, "iWARP error: Invalid DDP protocol version"}, 125 {IRDMA_AE_DDP_UBE_INVALID_MO, "Received message with too large offset"}, 126 {IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE, 127 "iWARP error: Inbound Send message when no receive buffer is available"}, 128 {IRDMA_AE_DDP_UBE_INVALID_QN, "iWARP error: Invalid QP number in inbound packet"}, 129 {IRDMA_AE_DDP_NO_L_BIT, 130 "iWARP error: Last bit not set in an inbound packet which completes RDMA Read"}, 131 {IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION, "iWARP error: Invalid RDMAP protocol version"}, 132 {IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE, "QP error: Invalid opcode"}, 133 {IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST, "Inbound Read request when QP isn't enabled for RDMA Read"}, 134 {IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP, 135 "Inbound RDMA Read response or RDMA Write when QP isn't enabled for RDMA R/W"}, 136 {IRDMA_AE_ROCE_RSP_LENGTH_ERROR, "RoCEv2 error: Received packet with incorrect length field"}, 137 {IRDMA_AE_ROCE_EMPTY_MCG, "RoCEv2 error: Multicast group has no valid members"}, 138 {IRDMA_AE_ROCE_BAD_MC_IP_ADDR, "RoCEv2 error: Multicast IP address doesn't match"}, 139 {IRDMA_AE_ROCE_BAD_MC_QPID, "RoCEv2 error: Multicast packet QP number isn't 0xffffff"}, 140 {IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH, "RoCEv2 error: Multicast packet protocol mismatch"}, 141 {IRDMA_AE_INVALID_ARP_ENTRY, "Invalid ARP entry"}, 142 {IRDMA_AE_INVALID_TCP_OPTION_RCVD, "iWARP error: Invalid TCP option"}, 143 {IRDMA_AE_STALE_ARP_ENTRY, "Stale ARP entry"}, 144 {IRDMA_AE_INVALID_AH_ENTRY, "Invalid AH entry"}, 145 {IRDMA_AE_LLP_CLOSE_COMPLETE, 146 "iWARP event: Graceful close complete"}, 147 {IRDMA_AE_LLP_CONNECTION_RESET, 148 "iWARP event: Received a TCP packet with a RST bit set"}, 149 {IRDMA_AE_LLP_FIN_RECEIVED, 150 "iWARP event: Received a TCP packet with a FIN bit set"}, 151 {IRDMA_AE_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH, 152 "iWARP error: Unable to close a gap in the TCP sequence"}, 153 {IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR, "Received an ICRC error"}, 154 {IRDMA_AE_LLP_SEGMENT_TOO_SMALL, 155 "iWARP error: Received a packet with insufficient space for protocol headers"}, 156 {IRDMA_AE_LLP_SYN_RECEIVED, 157 "iWARP event: Received a TCP packet with a SYN bit set"}, 158 {IRDMA_AE_LLP_TERMINATE_RECEIVED, 159 "iWARP error: Received a terminate message"}, 160 {IRDMA_AE_LLP_TOO_MANY_RETRIES, "Connection error: The max number of retries has been reached"}, 161 {IRDMA_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES, 162 "Connection error: The max number of keepalive retries has been reached"}, 163 {IRDMA_AE_LLP_DOUBT_REACHABILITY, 164 "Connection error: Doubt reachability (usually occurs after the max number of retries has been reached)"}, 165 {IRDMA_AE_LLP_CONNECTION_ESTABLISHED, 166 "iWARP event: Connection established"}, 167 {IRDMA_AE_LLP_TOO_MANY_RNRS, "RoCEv2: Too many RNR NACKs"}, 168 {IRDMA_AE_RESOURCE_EXHAUSTION, 169 "QP error: Resource exhaustion"}, 170 {IRDMA_AE_RESET_SENT, 171 "Reset sent (as requested via Modify QP)"}, 172 {IRDMA_AE_TERMINATE_SENT, 173 "Terminate sent (as requested via Modify QP)"}, 174 {IRDMA_AE_RESET_NOT_SENT, 175 "Reset not sent (but requested via Modify QP)"}, 176 {IRDMA_AE_LCE_QP_CATASTROPHIC, 177 "QP error: HW transaction resulted in catastrophic error"}, 178 {IRDMA_AE_LCE_FUNCTION_CATASTROPHIC, 179 "PCIe function error: HW transaction resulted in catastrophic error"}, 180 {IRDMA_AE_LCE_CQ_CATASTROPHIC, 181 "CQ error: HW transaction resulted in catastrophic error"}, 182 {IRDMA_AE_QP_SUSPEND_COMPLETE, "QP event: Suspend complete"}, 183 }; 184 185 /** 186 * irdma_get_ae_desc - returns AE description 187 * @ae_id: the AE number 188 */ 189 const char * 190 irdma_get_ae_desc(u16 ae_id) 191 { 192 const char *desc = ""; 193 int i; 194 195 for (i = 0; i < ARRAY_SIZE(ae_desc_list); i++) { 196 if (ae_desc_list[i].id == ae_id) { 197 desc = ae_desc_list[i].desc; 198 break; 199 } 200 } 201 return desc; 202 } 203 204 /** 205 * irdma_arp_table -manage arp table 206 * @rf: RDMA PCI function 207 * @ip_addr: ip address for device 208 * @mac_addr: mac address ptr 209 * @action: modify, delete or add 210 */ 211 int 212 irdma_arp_table(struct irdma_pci_f *rf, u32 *ip_addr, const u8 *mac_addr, 213 u32 action) 214 { 215 unsigned long flags; 216 int arp_index; 217 u32 ip[4] = {}; 218 219 memcpy(ip, ip_addr, sizeof(ip)); 220 221 spin_lock_irqsave(&rf->arp_lock, flags); 222 for (arp_index = 0; (u32)arp_index < rf->arp_table_size; arp_index++) { 223 if (!memcmp(rf->arp_table[arp_index].ip_addr, ip, sizeof(ip))) 224 break; 225 } 226 227 switch (action) { 228 case IRDMA_ARP_ADD: 229 if (arp_index != rf->arp_table_size) { 230 arp_index = -1; 231 break; 232 } 233 234 arp_index = 0; 235 if (irdma_alloc_rsrc(rf, rf->allocated_arps, rf->arp_table_size, 236 (u32 *)&arp_index, &rf->next_arp_index)) { 237 arp_index = -1; 238 break; 239 } 240 241 memcpy(rf->arp_table[arp_index].ip_addr, ip, 242 sizeof(rf->arp_table[arp_index].ip_addr)); 243 ether_addr_copy(rf->arp_table[arp_index].mac_addr, mac_addr); 244 break; 245 case IRDMA_ARP_RESOLVE: 246 if (arp_index == rf->arp_table_size) 247 arp_index = -1; 248 break; 249 case IRDMA_ARP_DELETE: 250 if (arp_index == rf->arp_table_size) { 251 arp_index = -1; 252 break; 253 } 254 255 memset(rf->arp_table[arp_index].ip_addr, 0, 256 sizeof(rf->arp_table[arp_index].ip_addr)); 257 eth_zero_addr(rf->arp_table[arp_index].mac_addr); 258 irdma_free_rsrc(rf, rf->allocated_arps, arp_index); 259 break; 260 default: 261 arp_index = -1; 262 break; 263 } 264 265 spin_unlock_irqrestore(&rf->arp_lock, flags); 266 return arp_index; 267 } 268 269 /** 270 * irdma_add_arp - add a new arp entry if needed 271 * @rf: RDMA function 272 * @ip: IP address 273 * @mac: MAC address 274 */ 275 int 276 irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, const u8 *mac) 277 { 278 int arpidx; 279 280 arpidx = irdma_arp_table(rf, &ip[0], NULL, IRDMA_ARP_RESOLVE); 281 if (arpidx >= 0) { 282 if (ether_addr_equal(rf->arp_table[arpidx].mac_addr, mac)) 283 return arpidx; 284 285 irdma_manage_arp_cache(rf, rf->arp_table[arpidx].mac_addr, ip, 286 IRDMA_ARP_DELETE); 287 } 288 289 irdma_manage_arp_cache(rf, mac, ip, IRDMA_ARP_ADD); 290 291 return irdma_arp_table(rf, ip, NULL, IRDMA_ARP_RESOLVE); 292 } 293 294 /** 295 * irdma_netdevice_event - system notifier for netdev events 296 * @notifier: not used 297 * @event: event for notifier 298 * @ptr: netdev 299 */ 300 int 301 irdma_netdevice_event(struct notifier_block *notifier, unsigned long event, 302 void *ptr) 303 { 304 struct irdma_device *iwdev; 305 struct ifnet *netdev = netdev_notifier_info_to_ifp(ptr); 306 307 iwdev = container_of(notifier, struct irdma_device, nb_netdevice_event); 308 if (iwdev->netdev != netdev) 309 return NOTIFY_DONE; 310 311 iwdev->iw_status = 1; 312 switch (event) { 313 case NETDEV_DOWN: 314 iwdev->iw_status = 0; 315 /* fallthrough */ 316 case NETDEV_UP: 317 irdma_port_ibevent(iwdev); 318 break; 319 default: 320 break; 321 } 322 323 return NOTIFY_DONE; 324 } 325 326 void 327 irdma_unregister_notifiers(struct irdma_device *iwdev) 328 { 329 unregister_netdevice_notifier(&iwdev->nb_netdevice_event); 330 } 331 332 int 333 irdma_register_notifiers(struct irdma_device *iwdev) 334 { 335 int ret; 336 337 iwdev->nb_netdevice_event.notifier_call = irdma_netdevice_event; 338 ret = register_netdevice_notifier(&iwdev->nb_netdevice_event); 339 if (ret) { 340 irdma_dev_err(&iwdev->ibdev, "register_netdevice_notifier failed\n"); 341 return ret; 342 } 343 return ret; 344 } 345 /** 346 * irdma_alloc_and_get_cqp_request - get cqp struct 347 * @cqp: device cqp ptr 348 * @wait: cqp to be used in wait mode 349 */ 350 struct irdma_cqp_request * 351 irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp, 352 bool wait) 353 { 354 struct irdma_cqp_request *cqp_request = NULL; 355 unsigned long flags; 356 357 spin_lock_irqsave(&cqp->req_lock, flags); 358 if (!list_empty(&cqp->cqp_avail_reqs)) { 359 cqp_request = list_entry(cqp->cqp_avail_reqs.next, 360 struct irdma_cqp_request, list); 361 list_del_init(&cqp_request->list); 362 } 363 spin_unlock_irqrestore(&cqp->req_lock, flags); 364 if (!cqp_request) { 365 cqp_request = kzalloc(sizeof(*cqp_request), GFP_ATOMIC); 366 if (cqp_request) { 367 cqp_request->dynamic = true; 368 if (wait) 369 init_waitqueue_head(&cqp_request->waitq); 370 } 371 } 372 if (!cqp_request) { 373 irdma_debug(cqp->sc_cqp.dev, IRDMA_DEBUG_ERR, "CQP Request Fail: No Memory"); 374 return NULL; 375 } 376 377 cqp_request->waiting = wait; 378 atomic_set(&cqp_request->refcnt, 1); 379 memset(&cqp_request->compl_info, 0, sizeof(cqp_request->compl_info)); 380 381 return cqp_request; 382 } 383 384 /** 385 * irdma_get_cqp_request - increase refcount for cqp_request 386 * @cqp_request: pointer to cqp_request instance 387 */ 388 static inline void 389 irdma_get_cqp_request(struct irdma_cqp_request *cqp_request) 390 { 391 atomic_inc(&cqp_request->refcnt); 392 } 393 394 /** 395 * irdma_free_cqp_request - free cqp request 396 * @cqp: cqp ptr 397 * @cqp_request: to be put back in cqp list 398 */ 399 void 400 irdma_free_cqp_request(struct irdma_cqp *cqp, 401 struct irdma_cqp_request *cqp_request) 402 { 403 unsigned long flags; 404 405 if (cqp_request->dynamic) { 406 kfree(cqp_request); 407 } else { 408 WRITE_ONCE(cqp_request->request_done, false); 409 cqp_request->callback_fcn = NULL; 410 cqp_request->waiting = false; 411 412 spin_lock_irqsave(&cqp->req_lock, flags); 413 list_add_tail(&cqp_request->list, &cqp->cqp_avail_reqs); 414 spin_unlock_irqrestore(&cqp->req_lock, flags); 415 } 416 wake_up(&cqp->remove_wq); 417 } 418 419 /** 420 * irdma_put_cqp_request - dec ref count and free if 0 421 * @cqp: cqp ptr 422 * @cqp_request: to be put back in cqp list 423 */ 424 void 425 irdma_put_cqp_request(struct irdma_cqp *cqp, 426 struct irdma_cqp_request *cqp_request) 427 { 428 if (atomic_dec_and_test(&cqp_request->refcnt)) 429 irdma_free_cqp_request(cqp, cqp_request); 430 } 431 432 /** 433 * irdma_free_pending_cqp_request -free pending cqp request objs 434 * @cqp: cqp ptr 435 * @cqp_request: to be put back in cqp list 436 */ 437 static void 438 irdma_free_pending_cqp_request(struct irdma_cqp *cqp, 439 struct irdma_cqp_request *cqp_request) 440 { 441 cqp_request->compl_info.error = true; 442 WRITE_ONCE(cqp_request->request_done, true); 443 444 if (cqp_request->waiting) 445 wake_up(&cqp_request->waitq); 446 wait_event_timeout(cqp->remove_wq, 447 atomic_read(&cqp_request->refcnt) == 1, 1000); 448 irdma_put_cqp_request(cqp, cqp_request); 449 } 450 451 /** 452 * irdma_cleanup_pending_cqp_op - clean-up cqp with no 453 * completions 454 * @rf: RDMA PCI function 455 */ 456 void 457 irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf) 458 { 459 struct irdma_sc_dev *dev = &rf->sc_dev; 460 struct irdma_cqp *cqp = &rf->cqp; 461 struct irdma_cqp_request *cqp_request = NULL; 462 struct cqp_cmds_info *pcmdinfo = NULL; 463 u32 i, pending_work, wqe_idx; 464 465 pending_work = IRDMA_RING_USED_QUANTA(cqp->sc_cqp.sq_ring); 466 wqe_idx = IRDMA_RING_CURRENT_TAIL(cqp->sc_cqp.sq_ring); 467 for (i = 0; i < pending_work; i++) { 468 cqp_request = (struct irdma_cqp_request *)(uintptr_t) 469 cqp->scratch_array[wqe_idx]; 470 if (cqp_request) 471 irdma_free_pending_cqp_request(cqp, cqp_request); 472 wqe_idx = (wqe_idx + 1) % IRDMA_RING_SIZE(cqp->sc_cqp.sq_ring); 473 } 474 475 while (!list_empty(&dev->cqp_cmd_head)) { 476 pcmdinfo = irdma_remove_cqp_head(dev); 477 cqp_request = 478 container_of(pcmdinfo, struct irdma_cqp_request, info); 479 if (cqp_request) 480 irdma_free_pending_cqp_request(cqp, cqp_request); 481 } 482 } 483 484 /** 485 * irdma_wait_event - wait for completion 486 * @rf: RDMA PCI function 487 * @cqp_request: cqp request to wait 488 */ 489 static int 490 irdma_wait_event(struct irdma_pci_f *rf, 491 struct irdma_cqp_request *cqp_request) 492 { 493 struct irdma_cqp_timeout cqp_timeout = {0}; 494 bool cqp_error = false; 495 int err_code = 0; 496 497 cqp_timeout.compl_cqp_cmds = atomic64_read(&rf->sc_dev.cqp->completed_ops); 498 do { 499 int wait_time_ms = rf->sc_dev.hw_attrs.max_cqp_compl_wait_time_ms; 500 501 irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); 502 if (wait_event_timeout(cqp_request->waitq, 503 READ_ONCE(cqp_request->request_done), 504 msecs_to_jiffies(wait_time_ms))) 505 break; 506 507 irdma_check_cqp_progress(&cqp_timeout, &rf->sc_dev); 508 509 if (cqp_timeout.count < CQP_TIMEOUT_THRESHOLD) 510 continue; 511 512 if (!rf->reset) { 513 rf->reset = true; 514 rf->gen_ops.request_reset(rf); 515 } 516 return -ETIMEDOUT; 517 } while (1); 518 519 cqp_error = cqp_request->compl_info.error; 520 if (cqp_error) { 521 err_code = -EIO; 522 if (cqp_request->compl_info.maj_err_code == 0xFFFF) { 523 if (cqp_request->compl_info.min_err_code == 0x8002) { 524 err_code = -EBUSY; 525 } else if (cqp_request->compl_info.min_err_code == 0x8029) { 526 if (!rf->reset) { 527 rf->reset = true; 528 rf->gen_ops.request_reset(rf); 529 } 530 } 531 } 532 } 533 534 return err_code; 535 } 536 537 static const char *const irdma_cqp_cmd_names[IRDMA_MAX_CQP_OPS] = { 538 [IRDMA_OP_CEQ_DESTROY] = "Destroy CEQ Cmd", 539 [IRDMA_OP_AEQ_DESTROY] = "Destroy AEQ Cmd", 540 [IRDMA_OP_DELETE_ARP_CACHE_ENTRY] = "Delete ARP Cache Cmd", 541 [IRDMA_OP_MANAGE_APBVT_ENTRY] = "Manage APBV Table Entry Cmd", 542 [IRDMA_OP_CEQ_CREATE] = "CEQ Create Cmd", 543 [IRDMA_OP_AEQ_CREATE] = "AEQ Destroy Cmd", 544 [IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY] = "Manage Quad Hash Table Entry Cmd", 545 [IRDMA_OP_QP_MODIFY] = "Modify QP Cmd", 546 [IRDMA_OP_QP_UPLOAD_CONTEXT] = "Upload Context Cmd", 547 [IRDMA_OP_CQ_CREATE] = "Create CQ Cmd", 548 [IRDMA_OP_CQ_DESTROY] = "Destroy CQ Cmd", 549 [IRDMA_OP_QP_CREATE] = "Create QP Cmd", 550 [IRDMA_OP_QP_DESTROY] = "Destroy QP Cmd", 551 [IRDMA_OP_ALLOC_STAG] = "Allocate STag Cmd", 552 [IRDMA_OP_MR_REG_NON_SHARED] = "Register Non-Shared MR Cmd", 553 [IRDMA_OP_DEALLOC_STAG] = "Deallocate STag Cmd", 554 [IRDMA_OP_MW_ALLOC] = "Allocate Memory Window Cmd", 555 [IRDMA_OP_QP_FLUSH_WQES] = "Flush QP Cmd", 556 [IRDMA_OP_ADD_ARP_CACHE_ENTRY] = "Add ARP Cache Cmd", 557 [IRDMA_OP_MANAGE_PUSH_PAGE] = "Manage Push Page Cmd", 558 [IRDMA_OP_UPDATE_PE_SDS] = "Update PE SDs Cmd", 559 [IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE] = "Manage HMC PM Function Table Cmd", 560 [IRDMA_OP_SUSPEND] = "Suspend QP Cmd", 561 [IRDMA_OP_RESUME] = "Resume QP Cmd", 562 [IRDMA_OP_QUERY_FPM_VAL] = "Query FPM Values Cmd", 563 [IRDMA_OP_COMMIT_FPM_VAL] = "Commit FPM Values Cmd", 564 [IRDMA_OP_AH_CREATE] = "Create Address Handle Cmd", 565 [IRDMA_OP_AH_MODIFY] = "Modify Address Handle Cmd", 566 [IRDMA_OP_AH_DESTROY] = "Destroy Address Handle Cmd", 567 [IRDMA_OP_MC_CREATE] = "Create Multicast Group Cmd", 568 [IRDMA_OP_MC_DESTROY] = "Destroy Multicast Group Cmd", 569 [IRDMA_OP_MC_MODIFY] = "Modify Multicast Group Cmd", 570 [IRDMA_OP_STATS_ALLOCATE] = "Add Statistics Instance Cmd", 571 [IRDMA_OP_STATS_FREE] = "Free Statistics Instance Cmd", 572 [IRDMA_OP_STATS_GATHER] = "Gather Statistics Cmd", 573 [IRDMA_OP_WS_ADD_NODE] = "Add Work Scheduler Node Cmd", 574 [IRDMA_OP_WS_MODIFY_NODE] = "Modify Work Scheduler Node Cmd", 575 [IRDMA_OP_WS_DELETE_NODE] = "Delete Work Scheduler Node Cmd", 576 [IRDMA_OP_WS_FAILOVER_START] = "Failover Start Cmd", 577 [IRDMA_OP_WS_FAILOVER_COMPLETE] = "Failover Complete Cmd", 578 [IRDMA_OP_SET_UP_MAP] = "Set UP-UP Mapping Cmd", 579 [IRDMA_OP_GEN_AE] = "Generate AE Cmd", 580 [IRDMA_OP_QUERY_RDMA_FEATURES] = "RDMA Get Features Cmd", 581 [IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY] = "Allocate Local MAC Entry Cmd", 582 [IRDMA_OP_ADD_LOCAL_MAC_ENTRY] = "Add Local MAC Entry Cmd", 583 [IRDMA_OP_DELETE_LOCAL_MAC_ENTRY] = "Delete Local MAC Entry Cmd", 584 [IRDMA_OP_CQ_MODIFY] = "CQ Modify Cmd", 585 }; 586 587 static const struct irdma_cqp_err_info irdma_noncrit_err_list[] = { 588 {0xffff, 0x8002, "Invalid State"}, 589 {0xffff, 0x8006, "Flush No Wqe Pending"}, 590 {0xffff, 0x8007, "Modify QP Bad Close"}, 591 {0xffff, 0x8009, "LLP Closed"}, 592 {0xffff, 0x800a, "Reset Not Sent"}, 593 {0xffff, 0x0200, "Failover Pending"}, 594 }; 595 596 /** 597 * irdma_cqp_crit_err - check if CQP error is critical 598 * @dev: pointer to dev structure 599 * @cqp_cmd: code for last CQP operation 600 * @maj_err_code: major error code 601 * @min_err_code: minot error code 602 */ 603 bool 604 irdma_cqp_crit_err(struct irdma_sc_dev *dev, u8 cqp_cmd, 605 u16 maj_err_code, u16 min_err_code) 606 { 607 int i; 608 609 for (i = 0; i < ARRAY_SIZE(irdma_noncrit_err_list); ++i) { 610 if (maj_err_code == irdma_noncrit_err_list[i].maj && 611 min_err_code == irdma_noncrit_err_list[i].min) { 612 irdma_debug(dev, IRDMA_DEBUG_CQP, 613 "[%s Error][%s] maj=0x%x min=0x%x\n", 614 irdma_noncrit_err_list[i].desc, 615 irdma_cqp_cmd_names[cqp_cmd], maj_err_code, 616 min_err_code); 617 return false; 618 } 619 } 620 return true; 621 } 622 623 /** 624 * irdma_handle_cqp_op - process cqp command 625 * @rf: RDMA PCI function 626 * @cqp_request: cqp request to process 627 */ 628 int 629 irdma_handle_cqp_op(struct irdma_pci_f *rf, 630 struct irdma_cqp_request *cqp_request) 631 { 632 struct irdma_sc_dev *dev = &rf->sc_dev; 633 struct cqp_cmds_info *info = &cqp_request->info; 634 int status; 635 bool put_cqp_request = true; 636 637 if (rf->reset) 638 return 0; 639 640 irdma_get_cqp_request(cqp_request); 641 status = irdma_process_cqp_cmd(dev, info); 642 if (status) 643 goto err; 644 645 if (cqp_request->waiting) { 646 put_cqp_request = false; 647 status = irdma_wait_event(rf, cqp_request); 648 if (status) 649 goto err; 650 } 651 652 return 0; 653 654 err: 655 if (irdma_cqp_crit_err(dev, info->cqp_cmd, 656 cqp_request->compl_info.maj_err_code, 657 cqp_request->compl_info.min_err_code)) 658 irdma_dev_err(&rf->iwdev->ibdev, 659 "[%s Error][op_code=%d] status=%d waiting=%d completion_err=%d maj=0x%x min=0x%x\n", 660 irdma_cqp_cmd_names[info->cqp_cmd], info->cqp_cmd, status, 661 cqp_request->waiting, cqp_request->compl_info.error, 662 cqp_request->compl_info.maj_err_code, 663 cqp_request->compl_info.min_err_code); 664 665 if (put_cqp_request) 666 irdma_put_cqp_request(&rf->cqp, cqp_request); 667 668 return status; 669 } 670 671 void 672 irdma_qp_add_ref(struct ib_qp *ibqp) 673 { 674 struct irdma_qp *iwqp = to_iwqp(ibqp); 675 676 atomic_inc(&iwqp->refcnt); 677 } 678 679 void 680 irdma_qp_rem_ref(struct ib_qp *ibqp) 681 { 682 struct irdma_qp *iwqp = to_iwqp(ibqp); 683 struct irdma_device *iwdev = iwqp->iwdev; 684 unsigned long flags; 685 686 spin_lock_irqsave(&iwdev->rf->qptable_lock, flags); 687 if (!atomic_dec_and_test(&iwqp->refcnt)) { 688 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); 689 return; 690 } 691 692 iwdev->rf->qp_table[iwqp->ibqp.qp_num] = NULL; 693 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); 694 complete(&iwqp->free_qp); 695 } 696 697 void 698 irdma_cq_add_ref(struct ib_cq *ibcq) 699 { 700 struct irdma_cq *iwcq = to_iwcq(ibcq); 701 702 atomic_inc(&iwcq->refcnt); 703 } 704 705 void 706 irdma_cq_rem_ref(struct ib_cq *ibcq) 707 { 708 struct irdma_cq *iwcq = to_iwcq(ibcq); 709 struct irdma_pci_f *rf = container_of(iwcq->sc_cq.dev, struct irdma_pci_f, sc_dev); 710 unsigned long flags; 711 712 spin_lock_irqsave(&rf->cqtable_lock, flags); 713 if (!atomic_dec_and_test(&iwcq->refcnt)) { 714 spin_unlock_irqrestore(&rf->cqtable_lock, flags); 715 return; 716 } 717 718 rf->cq_table[iwcq->cq_num] = NULL; 719 spin_unlock_irqrestore(&rf->cqtable_lock, flags); 720 complete(&iwcq->free_cq); 721 } 722 723 struct ib_device * 724 to_ibdev(struct irdma_sc_dev *dev) 725 { 726 return &(container_of(dev, struct irdma_pci_f, sc_dev))->iwdev->ibdev; 727 } 728 729 /** 730 * irdma_get_qp - get qp address 731 * @device: iwarp device 732 * @qpn: qp number 733 */ 734 struct ib_qp * 735 irdma_get_qp(struct ib_device *device, int qpn) 736 { 737 struct irdma_device *iwdev = to_iwdev(device); 738 739 if (qpn < IW_FIRST_QPN || qpn >= iwdev->rf->max_qp) 740 return NULL; 741 742 return &iwdev->rf->qp_table[qpn]->ibqp; 743 } 744 745 /** 746 * irdma_remove_cqp_head - return head entry and remove 747 * @dev: device 748 */ 749 void * 750 irdma_remove_cqp_head(struct irdma_sc_dev *dev) 751 { 752 struct list_head *entry; 753 struct list_head *list = &dev->cqp_cmd_head; 754 755 if (list_empty(list)) 756 return NULL; 757 758 entry = list->next; 759 list_del(entry); 760 761 return entry; 762 } 763 764 /** 765 * irdma_cqp_sds_cmd - create cqp command for sd 766 * @dev: hardware control device structure 767 * @sdinfo: information for sd cqp 768 * 769 */ 770 int 771 irdma_cqp_sds_cmd(struct irdma_sc_dev *dev, 772 struct irdma_update_sds_info *sdinfo) 773 { 774 struct irdma_cqp_request *cqp_request; 775 struct cqp_cmds_info *cqp_info; 776 struct irdma_pci_f *rf = dev_to_rf(dev); 777 int status; 778 779 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 780 if (!cqp_request) 781 return -ENOMEM; 782 783 cqp_info = &cqp_request->info; 784 memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo, 785 sizeof(cqp_info->in.u.update_pe_sds.info)); 786 cqp_info->cqp_cmd = IRDMA_OP_UPDATE_PE_SDS; 787 cqp_info->post_sq = 1; 788 cqp_info->in.u.update_pe_sds.dev = dev; 789 cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request; 790 791 status = irdma_handle_cqp_op(rf, cqp_request); 792 irdma_put_cqp_request(&rf->cqp, cqp_request); 793 794 return status; 795 } 796 797 /** 798 * irdma_cqp_qp_suspend_resume - cqp command for suspend/resume 799 * @qp: hardware control qp 800 * @op: suspend or resume 801 */ 802 int 803 irdma_cqp_qp_suspend_resume(struct irdma_sc_qp *qp, u8 op) 804 { 805 struct irdma_sc_dev *dev = qp->dev; 806 struct irdma_cqp_request *cqp_request; 807 struct irdma_sc_cqp *cqp = dev->cqp; 808 struct cqp_cmds_info *cqp_info; 809 struct irdma_pci_f *rf = dev_to_rf(dev); 810 int status; 811 812 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false); 813 if (!cqp_request) 814 return -ENOMEM; 815 816 cqp_info = &cqp_request->info; 817 cqp_info->cqp_cmd = op; 818 cqp_info->in.u.suspend_resume.cqp = cqp; 819 cqp_info->in.u.suspend_resume.qp = qp; 820 cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request; 821 822 status = irdma_handle_cqp_op(rf, cqp_request); 823 irdma_put_cqp_request(&rf->cqp, cqp_request); 824 825 return status; 826 } 827 828 /** 829 * irdma_term_modify_qp - modify qp for term message 830 * @qp: hardware control qp 831 * @next_state: qp's next state 832 * @term: terminate code 833 * @term_len: length 834 */ 835 void 836 irdma_term_modify_qp(struct irdma_sc_qp *qp, u8 next_state, u8 term, 837 u8 term_len) 838 { 839 struct irdma_qp *iwqp; 840 841 iwqp = qp->qp_uk.back_qp; 842 irdma_next_iw_state(iwqp, next_state, 0, term, term_len); 843 }; 844 845 /** 846 * irdma_terminate_done - after terminate is completed 847 * @qp: hardware control qp 848 * @timeout_occurred: indicates if terminate timer expired 849 */ 850 void 851 irdma_terminate_done(struct irdma_sc_qp *qp, int timeout_occurred) 852 { 853 struct irdma_qp *iwqp; 854 u8 hte = 0; 855 bool first_time; 856 unsigned long flags; 857 858 iwqp = qp->qp_uk.back_qp; 859 spin_lock_irqsave(&iwqp->lock, flags); 860 if (iwqp->hte_added) { 861 iwqp->hte_added = 0; 862 hte = 1; 863 } 864 first_time = !(qp->term_flags & IRDMA_TERM_DONE); 865 qp->term_flags |= IRDMA_TERM_DONE; 866 spin_unlock_irqrestore(&iwqp->lock, flags); 867 if (first_time) { 868 if (!timeout_occurred) 869 irdma_terminate_del_timer(qp); 870 871 irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, hte, 0, 0); 872 irdma_cm_disconn(iwqp); 873 } 874 } 875 876 static void 877 irdma_terminate_timeout(struct timer_list *t) 878 { 879 struct irdma_qp *iwqp = from_timer(iwqp, t, terminate_timer); 880 struct irdma_sc_qp *qp = &iwqp->sc_qp; 881 882 irdma_terminate_done(qp, 1); 883 irdma_qp_rem_ref(&iwqp->ibqp); 884 } 885 886 /** 887 * irdma_terminate_start_timer - start terminate timeout 888 * @qp: hardware control qp 889 */ 890 void 891 irdma_terminate_start_timer(struct irdma_sc_qp *qp) 892 { 893 struct irdma_qp *iwqp; 894 895 iwqp = qp->qp_uk.back_qp; 896 irdma_qp_add_ref(&iwqp->ibqp); 897 timer_setup(&iwqp->terminate_timer, irdma_terminate_timeout, 0); 898 iwqp->terminate_timer.expires = jiffies + HZ; 899 900 add_timer(&iwqp->terminate_timer); 901 } 902 903 /** 904 * irdma_terminate_del_timer - delete terminate timeout 905 * @qp: hardware control qp 906 */ 907 void 908 irdma_terminate_del_timer(struct irdma_sc_qp *qp) 909 { 910 struct irdma_qp *iwqp; 911 int ret; 912 913 iwqp = qp->qp_uk.back_qp; 914 ret = irdma_del_timer_compat(&iwqp->terminate_timer); 915 if (ret) 916 irdma_qp_rem_ref(&iwqp->ibqp); 917 } 918 919 /** 920 * irdma_cqp_query_fpm_val_cmd - send cqp command for fpm 921 * @dev: function device struct 922 * @val_mem: buffer for fpm 923 * @hmc_fn_id: function id for fpm 924 */ 925 int 926 irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev *dev, 927 struct irdma_dma_mem *val_mem, u16 hmc_fn_id) 928 { 929 struct irdma_cqp_request *cqp_request; 930 struct cqp_cmds_info *cqp_info; 931 struct irdma_pci_f *rf = dev_to_rf(dev); 932 int status; 933 934 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 935 if (!cqp_request) 936 return -ENOMEM; 937 938 cqp_info = &cqp_request->info; 939 cqp_request->param = NULL; 940 cqp_info->in.u.query_fpm_val.cqp = dev->cqp; 941 cqp_info->in.u.query_fpm_val.fpm_val_pa = val_mem->pa; 942 cqp_info->in.u.query_fpm_val.fpm_val_va = val_mem->va; 943 cqp_info->in.u.query_fpm_val.hmc_fn_id = hmc_fn_id; 944 cqp_info->cqp_cmd = IRDMA_OP_QUERY_FPM_VAL; 945 cqp_info->post_sq = 1; 946 cqp_info->in.u.query_fpm_val.scratch = (uintptr_t)cqp_request; 947 948 status = irdma_handle_cqp_op(rf, cqp_request); 949 irdma_put_cqp_request(&rf->cqp, cqp_request); 950 951 return status; 952 } 953 954 /** 955 * irdma_cqp_commit_fpm_val_cmd - commit fpm values in hw 956 * @dev: hardware control device structure 957 * @val_mem: buffer with fpm values 958 * @hmc_fn_id: function id for fpm 959 */ 960 int 961 irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev *dev, 962 struct irdma_dma_mem *val_mem, u16 hmc_fn_id) 963 { 964 struct irdma_cqp_request *cqp_request; 965 struct cqp_cmds_info *cqp_info; 966 struct irdma_pci_f *rf = dev_to_rf(dev); 967 int status; 968 969 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 970 if (!cqp_request) 971 return -ENOMEM; 972 973 cqp_info = &cqp_request->info; 974 cqp_request->param = NULL; 975 cqp_info->in.u.commit_fpm_val.cqp = dev->cqp; 976 cqp_info->in.u.commit_fpm_val.fpm_val_pa = val_mem->pa; 977 cqp_info->in.u.commit_fpm_val.fpm_val_va = val_mem->va; 978 cqp_info->in.u.commit_fpm_val.hmc_fn_id = hmc_fn_id; 979 cqp_info->cqp_cmd = IRDMA_OP_COMMIT_FPM_VAL; 980 cqp_info->post_sq = 1; 981 cqp_info->in.u.commit_fpm_val.scratch = (uintptr_t)cqp_request; 982 983 status = irdma_handle_cqp_op(rf, cqp_request); 984 irdma_put_cqp_request(&rf->cqp, cqp_request); 985 986 return status; 987 } 988 989 /** 990 * irdma_cqp_cq_create_cmd - create a cq for the cqp 991 * @dev: device pointer 992 * @cq: pointer to created cq 993 */ 994 int 995 irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq) 996 { 997 struct irdma_pci_f *rf = dev_to_rf(dev); 998 struct irdma_cqp *iwcqp = &rf->cqp; 999 struct irdma_cqp_request *cqp_request; 1000 struct cqp_cmds_info *cqp_info; 1001 int status; 1002 1003 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true); 1004 if (!cqp_request) 1005 return -ENOMEM; 1006 1007 cqp_info = &cqp_request->info; 1008 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE; 1009 cqp_info->post_sq = 1; 1010 cqp_info->in.u.cq_create.cq = cq; 1011 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request; 1012 1013 status = irdma_handle_cqp_op(rf, cqp_request); 1014 irdma_put_cqp_request(iwcqp, cqp_request); 1015 1016 return status; 1017 } 1018 1019 /** 1020 * irdma_cqp_qp_create_cmd - create a qp for the cqp 1021 * @dev: device pointer 1022 * @qp: pointer to created qp 1023 */ 1024 int 1025 irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp) 1026 { 1027 struct irdma_pci_f *rf = dev_to_rf(dev); 1028 struct irdma_cqp *iwcqp = &rf->cqp; 1029 struct irdma_cqp_request *cqp_request; 1030 struct cqp_cmds_info *cqp_info; 1031 struct irdma_create_qp_info *qp_info; 1032 int status; 1033 1034 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true); 1035 if (!cqp_request) 1036 return -ENOMEM; 1037 1038 cqp_info = &cqp_request->info; 1039 qp_info = &cqp_request->info.in.u.qp_create.info; 1040 memset(qp_info, 0, sizeof(*qp_info)); 1041 qp_info->cq_num_valid = true; 1042 qp_info->next_iwarp_state = IRDMA_QP_STATE_RTS; 1043 cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE; 1044 cqp_info->post_sq = 1; 1045 cqp_info->in.u.qp_create.qp = qp; 1046 cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request; 1047 1048 status = irdma_handle_cqp_op(rf, cqp_request); 1049 irdma_put_cqp_request(iwcqp, cqp_request); 1050 1051 return status; 1052 } 1053 1054 /** 1055 * irdma_dealloc_push_page - free a push page for qp 1056 * @rf: RDMA PCI function 1057 * @iwqp: QP pointer 1058 */ 1059 void 1060 irdma_dealloc_push_page(struct irdma_pci_f *rf, 1061 struct irdma_qp *iwqp) 1062 { 1063 struct irdma_cqp_request *cqp_request; 1064 struct cqp_cmds_info *cqp_info; 1065 int status; 1066 struct irdma_sc_qp *qp = &iwqp->sc_qp; 1067 1068 if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) 1069 return; 1070 1071 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false); 1072 if (!cqp_request) 1073 return; 1074 1075 cqp_info = &cqp_request->info; 1076 cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE; 1077 cqp_info->post_sq = 1; 1078 cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx; 1079 cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle; 1080 cqp_info->in.u.manage_push_page.info.free_page = 1; 1081 cqp_info->in.u.manage_push_page.info.push_page_type = 0; 1082 cqp_info->in.u.manage_push_page.cqp = &rf->cqp.sc_cqp; 1083 cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request; 1084 status = irdma_handle_cqp_op(rf, cqp_request); 1085 if (!status) 1086 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX; 1087 irdma_put_cqp_request(&rf->cqp, cqp_request); 1088 } 1089 1090 /** 1091 * irdma_cq_wq_destroy - send cq destroy cqp 1092 * @rf: RDMA PCI function 1093 * @cq: hardware control cq 1094 */ 1095 void 1096 irdma_cq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_cq *cq) 1097 { 1098 struct irdma_cqp_request *cqp_request; 1099 struct cqp_cmds_info *cqp_info; 1100 1101 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1102 if (!cqp_request) 1103 return; 1104 1105 cqp_info = &cqp_request->info; 1106 cqp_info->cqp_cmd = IRDMA_OP_CQ_DESTROY; 1107 cqp_info->post_sq = 1; 1108 cqp_info->in.u.cq_destroy.cq = cq; 1109 cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request; 1110 1111 irdma_handle_cqp_op(rf, cqp_request); 1112 irdma_put_cqp_request(&rf->cqp, cqp_request); 1113 } 1114 1115 /** 1116 * irdma_hw_modify_qp_callback - handle state for modifyQPs that don't wait 1117 * @cqp_request: modify QP completion 1118 */ 1119 static void 1120 irdma_hw_modify_qp_callback(struct irdma_cqp_request *cqp_request) 1121 { 1122 struct cqp_cmds_info *cqp_info; 1123 struct irdma_qp *iwqp; 1124 1125 cqp_info = &cqp_request->info; 1126 iwqp = cqp_info->in.u.qp_modify.qp->qp_uk.back_qp; 1127 atomic_dec(&iwqp->hw_mod_qp_pend); 1128 wake_up(&iwqp->mod_qp_waitq); 1129 } 1130 1131 /** 1132 * irdma_hw_modify_qp - setup cqp for modify qp 1133 * @iwdev: RDMA device 1134 * @iwqp: qp ptr (user or kernel) 1135 * @info: info for modify qp 1136 * @wait: flag to wait or not for modify qp completion 1137 */ 1138 int 1139 irdma_hw_modify_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp, 1140 struct irdma_modify_qp_info *info, bool wait) 1141 { 1142 int status; 1143 struct irdma_pci_f *rf = iwdev->rf; 1144 struct irdma_cqp_request *cqp_request; 1145 struct cqp_cmds_info *cqp_info; 1146 struct irdma_modify_qp_info *m_info; 1147 1148 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait); 1149 if (!cqp_request) 1150 return -ENOMEM; 1151 1152 if (!wait) { 1153 cqp_request->callback_fcn = irdma_hw_modify_qp_callback; 1154 atomic_inc(&iwqp->hw_mod_qp_pend); 1155 } 1156 cqp_info = &cqp_request->info; 1157 m_info = &cqp_info->in.u.qp_modify.info; 1158 memcpy(m_info, info, sizeof(*m_info)); 1159 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY; 1160 cqp_info->post_sq = 1; 1161 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp; 1162 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request; 1163 status = irdma_handle_cqp_op(rf, cqp_request); 1164 irdma_put_cqp_request(&rf->cqp, cqp_request); 1165 if (status) { 1166 if (rdma_protocol_roce(&iwdev->ibdev, 1)) 1167 return status; 1168 1169 switch (m_info->next_iwarp_state) { 1170 struct irdma_gen_ae_info ae_info; 1171 1172 case IRDMA_QP_STATE_RTS: 1173 case IRDMA_QP_STATE_IDLE: 1174 case IRDMA_QP_STATE_TERMINATE: 1175 case IRDMA_QP_STATE_CLOSING: 1176 if (info->curr_iwarp_state == IRDMA_QP_STATE_IDLE) 1177 irdma_send_reset(iwqp->cm_node); 1178 else 1179 iwqp->sc_qp.term_flags = IRDMA_TERM_DONE; 1180 if (!wait) { 1181 ae_info.ae_code = IRDMA_AE_BAD_CLOSE; 1182 ae_info.ae_src = 0; 1183 irdma_gen_ae(rf, &iwqp->sc_qp, &ae_info, false); 1184 } else { 1185 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, 1186 wait); 1187 if (!cqp_request) 1188 return -ENOMEM; 1189 1190 cqp_info = &cqp_request->info; 1191 m_info = &cqp_info->in.u.qp_modify.info; 1192 memcpy(m_info, info, sizeof(*m_info)); 1193 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY; 1194 cqp_info->post_sq = 1; 1195 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp; 1196 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request; 1197 m_info->next_iwarp_state = IRDMA_QP_STATE_ERROR; 1198 m_info->reset_tcp_conn = true; 1199 irdma_handle_cqp_op(rf, cqp_request); 1200 irdma_put_cqp_request(&rf->cqp, cqp_request); 1201 } 1202 break; 1203 case IRDMA_QP_STATE_ERROR: 1204 default: 1205 break; 1206 } 1207 } 1208 1209 return status; 1210 } 1211 1212 /** 1213 * irdma_cqp_cq_destroy_cmd - destroy the cqp cq 1214 * @dev: device pointer 1215 * @cq: pointer to cq 1216 */ 1217 void 1218 irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq) 1219 { 1220 struct irdma_pci_f *rf = dev_to_rf(dev); 1221 1222 irdma_cq_wq_destroy(rf, cq); 1223 } 1224 1225 /** 1226 * irdma_cqp_qp_destroy_cmd - destroy the cqp 1227 * @dev: device pointer 1228 * @qp: pointer to qp 1229 */ 1230 int 1231 irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp) 1232 { 1233 struct irdma_pci_f *rf = dev_to_rf(dev); 1234 struct irdma_cqp *iwcqp = &rf->cqp; 1235 struct irdma_cqp_request *cqp_request; 1236 struct cqp_cmds_info *cqp_info; 1237 int status; 1238 1239 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true); 1240 if (!cqp_request) 1241 return -ENOMEM; 1242 1243 cqp_info = &cqp_request->info; 1244 memset(cqp_info, 0, sizeof(*cqp_info)); 1245 cqp_info->cqp_cmd = IRDMA_OP_QP_DESTROY; 1246 cqp_info->post_sq = 1; 1247 cqp_info->in.u.qp_destroy.qp = qp; 1248 cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request; 1249 cqp_info->in.u.qp_destroy.remove_hash_idx = true; 1250 1251 status = irdma_handle_cqp_op(rf, cqp_request); 1252 irdma_put_cqp_request(&rf->cqp, cqp_request); 1253 1254 return status; 1255 } 1256 1257 /** 1258 * irdma_ieq_mpa_crc_ae - generate AE for crc error 1259 * @dev: hardware control device structure 1260 * @qp: hardware control qp 1261 */ 1262 void 1263 irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp) 1264 { 1265 struct irdma_gen_ae_info info = {0}; 1266 struct irdma_pci_f *rf = dev_to_rf(dev); 1267 1268 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_AEQ, "Generate MPA CRC AE\n"); 1269 info.ae_code = IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR; 1270 info.ae_src = IRDMA_AE_SOURCE_RQ; 1271 irdma_gen_ae(rf, qp, &info, false); 1272 } 1273 1274 /** 1275 * irdma_ieq_get_qp - get qp based on quad in puda buffer 1276 * @dev: hardware control device structure 1277 * @buf: receive puda buffer on exception q 1278 */ 1279 struct irdma_sc_qp * 1280 irdma_ieq_get_qp(struct irdma_sc_dev *dev, 1281 struct irdma_puda_buf *buf) 1282 { 1283 struct irdma_qp *iwqp; 1284 struct irdma_cm_node *cm_node; 1285 struct irdma_device *iwdev = buf->vsi->back_vsi; 1286 u32 loc_addr[4] = {0}; 1287 u32 rem_addr[4] = {0}; 1288 u16 loc_port, rem_port; 1289 struct ip6_hdr *ip6h; 1290 struct ip *iph = (struct ip *)buf->iph; 1291 struct tcphdr *tcph = (struct tcphdr *)buf->tcph; 1292 1293 if (iph->ip_v == 4) { 1294 loc_addr[0] = ntohl(iph->ip_dst.s_addr); 1295 rem_addr[0] = ntohl(iph->ip_src.s_addr); 1296 } else { 1297 ip6h = (struct ip6_hdr *)buf->iph; 1298 irdma_copy_ip_ntohl(loc_addr, ip6h->ip6_dst.__u6_addr.__u6_addr32); 1299 irdma_copy_ip_ntohl(rem_addr, ip6h->ip6_src.__u6_addr.__u6_addr32); 1300 } 1301 loc_port = ntohs(tcph->th_dport); 1302 rem_port = ntohs(tcph->th_sport); 1303 cm_node = irdma_find_node(&iwdev->cm_core, rem_port, rem_addr, loc_port, 1304 loc_addr, buf->vlan_valid ? buf->vlan_id : 0xFFFF); 1305 if (!cm_node) 1306 return NULL; 1307 1308 iwqp = cm_node->iwqp; 1309 irdma_rem_ref_cm_node(cm_node); 1310 1311 return &iwqp->sc_qp; 1312 } 1313 1314 /** 1315 * irdma_send_ieq_ack - ACKs for duplicate or OOO partials FPDUs 1316 * @qp: qp ptr 1317 */ 1318 void 1319 irdma_send_ieq_ack(struct irdma_sc_qp *qp) 1320 { 1321 struct irdma_cm_node *cm_node = ((struct irdma_qp *)qp->qp_uk.back_qp)->cm_node; 1322 struct irdma_puda_buf *buf = qp->pfpdu.lastrcv_buf; 1323 struct tcphdr *tcph = (struct tcphdr *)buf->tcph; 1324 1325 cm_node->tcp_cntxt.rcv_nxt = qp->pfpdu.nextseqnum; 1326 cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->th_ack); 1327 1328 irdma_send_ack(cm_node); 1329 } 1330 1331 /** 1332 * irdma_puda_ieq_get_ah_info - get AH info from IEQ buffer 1333 * @qp: qp pointer 1334 * @ah_info: AH info pointer 1335 */ 1336 void 1337 irdma_puda_ieq_get_ah_info(struct irdma_sc_qp *qp, 1338 struct irdma_ah_info *ah_info) 1339 { 1340 struct irdma_puda_buf *buf = qp->pfpdu.ah_buf; 1341 struct ip *iph; 1342 struct ip6_hdr *ip6h; 1343 1344 memset(ah_info, 0, sizeof(*ah_info)); 1345 ah_info->do_lpbk = true; 1346 ah_info->vlan_tag = buf->vlan_id; 1347 ah_info->insert_vlan_tag = buf->vlan_valid; 1348 ah_info->ipv4_valid = buf->ipv4; 1349 ah_info->vsi = qp->vsi; 1350 1351 if (buf->smac_valid) 1352 ether_addr_copy(ah_info->mac_addr, buf->smac); 1353 1354 if (buf->ipv4) { 1355 ah_info->ipv4_valid = true; 1356 iph = (struct ip *)buf->iph; 1357 ah_info->hop_ttl = iph->ip_ttl; 1358 ah_info->tc_tos = iph->ip_tos; 1359 ah_info->dest_ip_addr[0] = ntohl(iph->ip_dst.s_addr); 1360 ah_info->src_ip_addr[0] = ntohl(iph->ip_src.s_addr); 1361 } else { 1362 ip6h = (struct ip6_hdr *)buf->iph; 1363 ah_info->hop_ttl = ip6h->ip6_hops; 1364 ah_info->tc_tos = ip6h->ip6_vfc; 1365 irdma_copy_ip_ntohl(ah_info->dest_ip_addr, 1366 ip6h->ip6_dst.__u6_addr.__u6_addr32); 1367 irdma_copy_ip_ntohl(ah_info->src_ip_addr, 1368 ip6h->ip6_src.__u6_addr.__u6_addr32); 1369 } 1370 1371 ah_info->dst_arpindex = irdma_arp_table(dev_to_rf(qp->dev), 1372 ah_info->dest_ip_addr, 1373 NULL, IRDMA_ARP_RESOLVE); 1374 } 1375 1376 /** 1377 * irdma_gen1_ieq_update_tcpip_info - update tcpip in the buffer 1378 * @buf: puda to update 1379 * @len: length of buffer 1380 * @seqnum: seq number for tcp 1381 */ 1382 static void 1383 irdma_gen1_ieq_update_tcpip_info(struct irdma_puda_buf *buf, 1384 u16 len, u32 seqnum) 1385 { 1386 struct tcphdr *tcph; 1387 struct ip *iph; 1388 u16 iphlen; 1389 u16 pktsize; 1390 u8 *addr = buf->mem.va; 1391 1392 iphlen = (buf->ipv4) ? 20 : 40; 1393 iph = (struct ip *)(addr + buf->maclen); 1394 tcph = (struct tcphdr *)(addr + buf->maclen + iphlen); 1395 pktsize = len + buf->tcphlen + iphlen; 1396 iph->ip_len = htons(pktsize); 1397 tcph->th_seq = htonl(seqnum); 1398 } 1399 1400 /** 1401 * irdma_ieq_update_tcpip_info - update tcpip in the buffer 1402 * @buf: puda to update 1403 * @len: length of buffer 1404 * @seqnum: seq number for tcp 1405 */ 1406 void 1407 irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len, 1408 u32 seqnum) 1409 { 1410 struct tcphdr *tcph; 1411 u8 *addr; 1412 1413 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 1414 return irdma_gen1_ieq_update_tcpip_info(buf, len, seqnum); 1415 1416 addr = buf->mem.va; 1417 tcph = (struct tcphdr *)addr; 1418 tcph->th_seq = htonl(seqnum); 1419 } 1420 1421 /** 1422 * irdma_gen1_puda_get_tcpip_info - get tcpip info from puda 1423 * buffer 1424 * @info: to get information 1425 * @buf: puda buffer 1426 */ 1427 static int 1428 irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info, 1429 struct irdma_puda_buf *buf) 1430 { 1431 struct ip *iph; 1432 struct ip6_hdr *ip6h; 1433 struct tcphdr *tcph; 1434 u16 iphlen; 1435 u16 pkt_len; 1436 u8 *mem = buf->mem.va; 1437 struct ether_header *ethh = buf->mem.va; 1438 1439 if (ethh->ether_type == htons(0x8100)) { 1440 info->vlan_valid = true; 1441 buf->vlan_id = ntohs(((struct ether_vlan_header *)ethh)->evl_tag) & 1442 EVL_VLID_MASK; 1443 } 1444 1445 buf->maclen = (info->vlan_valid) ? 18 : 14; 1446 iphlen = (info->l3proto) ? 40 : 20; 1447 buf->ipv4 = (info->l3proto) ? false : true; 1448 buf->iph = mem + buf->maclen; 1449 iph = (struct ip *)buf->iph; 1450 buf->tcph = buf->iph + iphlen; 1451 tcph = (struct tcphdr *)buf->tcph; 1452 1453 if (buf->ipv4) { 1454 pkt_len = ntohs(iph->ip_len); 1455 } else { 1456 ip6h = (struct ip6_hdr *)buf->iph; 1457 pkt_len = ntohs(ip6h->ip6_plen) + iphlen; 1458 } 1459 1460 buf->totallen = pkt_len + buf->maclen; 1461 1462 if (info->payload_len < buf->totallen) { 1463 irdma_debug(buf->vsi->dev, IRDMA_DEBUG_ERR, 1464 "payload_len = 0x%x totallen expected0x%x\n", 1465 info->payload_len, buf->totallen); 1466 return -EINVAL; 1467 } 1468 1469 buf->tcphlen = tcph->th_off << 2; 1470 buf->datalen = pkt_len - iphlen - buf->tcphlen; 1471 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL; 1472 buf->hdrlen = buf->maclen + iphlen + buf->tcphlen; 1473 buf->seqnum = ntohl(tcph->th_seq); 1474 1475 return 0; 1476 } 1477 1478 /** 1479 * irdma_puda_get_tcpip_info - get tcpip info from puda buffer 1480 * @info: to get information 1481 * @buf: puda buffer 1482 */ 1483 int 1484 irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info, 1485 struct irdma_puda_buf *buf) 1486 { 1487 struct tcphdr *tcph; 1488 u32 pkt_len; 1489 u8 *mem; 1490 1491 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 1492 return irdma_gen1_puda_get_tcpip_info(info, buf); 1493 1494 mem = buf->mem.va; 1495 buf->vlan_valid = info->vlan_valid; 1496 if (info->vlan_valid) 1497 buf->vlan_id = info->vlan; 1498 1499 buf->ipv4 = info->ipv4; 1500 if (buf->ipv4) 1501 buf->iph = mem + IRDMA_IPV4_PAD; 1502 else 1503 buf->iph = mem; 1504 1505 buf->tcph = mem + IRDMA_TCP_OFFSET; 1506 tcph = (struct tcphdr *)buf->tcph; 1507 pkt_len = info->payload_len; 1508 buf->totallen = pkt_len; 1509 buf->tcphlen = tcph->th_off << 2; 1510 buf->datalen = pkt_len - IRDMA_TCP_OFFSET - buf->tcphlen; 1511 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL; 1512 buf->hdrlen = IRDMA_TCP_OFFSET + buf->tcphlen; 1513 buf->seqnum = ntohl(tcph->th_seq); 1514 1515 if (info->smac_valid) { 1516 ether_addr_copy(buf->smac, info->smac); 1517 buf->smac_valid = true; 1518 } 1519 1520 return 0; 1521 } 1522 1523 /** 1524 * irdma_hw_stats_timeout - Stats timer-handler which updates all HW stats 1525 * @t: timer_list pointer 1526 */ 1527 static void 1528 irdma_hw_stats_timeout(struct timer_list *t) 1529 { 1530 struct irdma_vsi_pestat *pf_devstat = 1531 from_timer(pf_devstat, t, stats_timer); 1532 struct irdma_sc_vsi *sc_vsi = pf_devstat->vsi; 1533 1534 if (sc_vsi->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 1535 irdma_cqp_gather_stats_cmd(sc_vsi->dev, sc_vsi->pestat, false); 1536 1537 mod_timer(&pf_devstat->stats_timer, 1538 jiffies + msecs_to_jiffies(STATS_TIMER_DELAY)); 1539 } 1540 1541 /** 1542 * irdma_hw_stats_start_timer - Start periodic stats timer 1543 * @vsi: vsi structure pointer 1544 */ 1545 void 1546 irdma_hw_stats_start_timer(struct irdma_sc_vsi *vsi) 1547 { 1548 struct irdma_vsi_pestat *devstat = vsi->pestat; 1549 1550 timer_setup(&devstat->stats_timer, irdma_hw_stats_timeout, 0); 1551 mod_timer(&devstat->stats_timer, 1552 jiffies + msecs_to_jiffies(STATS_TIMER_DELAY)); 1553 } 1554 1555 /** 1556 * irdma_hw_stats_stop_timer - Delete periodic stats timer 1557 * @vsi: pointer to vsi structure 1558 */ 1559 void 1560 irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi) 1561 { 1562 struct irdma_vsi_pestat *devstat = vsi->pestat; 1563 1564 del_timer_sync(&devstat->stats_timer); 1565 } 1566 1567 /** 1568 * irdma_process_cqp_stats - Checking for wrap and update stats 1569 * @cqp_request: cqp_request structure pointer 1570 */ 1571 static void 1572 irdma_process_cqp_stats(struct irdma_cqp_request *cqp_request) 1573 { 1574 struct irdma_vsi_pestat *pestat = cqp_request->param; 1575 1576 sc_vsi_update_stats(pestat->vsi); 1577 } 1578 1579 /** 1580 * irdma_cqp_gather_stats_cmd - Gather stats 1581 * @dev: pointer to device structure 1582 * @pestat: pointer to stats info 1583 * @wait: flag to wait or not wait for stats 1584 */ 1585 int 1586 irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev, 1587 struct irdma_vsi_pestat *pestat, bool wait) 1588 { 1589 1590 struct irdma_pci_f *rf = dev_to_rf(dev); 1591 struct irdma_cqp *iwcqp = &rf->cqp; 1592 struct irdma_cqp_request *cqp_request; 1593 struct cqp_cmds_info *cqp_info; 1594 int status; 1595 1596 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait); 1597 if (!cqp_request) 1598 return -ENOMEM; 1599 1600 cqp_info = &cqp_request->info; 1601 memset(cqp_info, 0, sizeof(*cqp_info)); 1602 cqp_info->cqp_cmd = IRDMA_OP_STATS_GATHER; 1603 cqp_info->post_sq = 1; 1604 cqp_info->in.u.stats_gather.info = pestat->gather_info; 1605 cqp_info->in.u.stats_gather.scratch = (uintptr_t)cqp_request; 1606 cqp_info->in.u.stats_gather.cqp = &rf->cqp.sc_cqp; 1607 cqp_request->param = pestat; 1608 if (!wait) 1609 cqp_request->callback_fcn = irdma_process_cqp_stats; 1610 status = irdma_handle_cqp_op(rf, cqp_request); 1611 if (wait) 1612 sc_vsi_update_stats(pestat->vsi); 1613 irdma_put_cqp_request(&rf->cqp, cqp_request); 1614 1615 return status; 1616 } 1617 1618 /** 1619 * irdma_cqp_stats_inst_cmd - Allocate/free stats instance 1620 * @vsi: pointer to vsi structure 1621 * @cmd: command to allocate or free 1622 * @stats_info: pointer to allocate stats info 1623 */ 1624 int 1625 irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd, 1626 struct irdma_stats_inst_info *stats_info) 1627 { 1628 struct irdma_pci_f *rf = dev_to_rf(vsi->dev); 1629 struct irdma_cqp *iwcqp = &rf->cqp; 1630 struct irdma_cqp_request *cqp_request; 1631 struct cqp_cmds_info *cqp_info; 1632 int status; 1633 bool wait = false; 1634 1635 if (cmd == IRDMA_OP_STATS_ALLOCATE) 1636 wait = true; 1637 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait); 1638 if (!cqp_request) 1639 return -ENOMEM; 1640 1641 cqp_info = &cqp_request->info; 1642 memset(cqp_info, 0, sizeof(*cqp_info)); 1643 cqp_info->cqp_cmd = cmd; 1644 cqp_info->post_sq = 1; 1645 cqp_info->in.u.stats_manage.info = *stats_info; 1646 cqp_info->in.u.stats_manage.scratch = (uintptr_t)cqp_request; 1647 cqp_info->in.u.stats_manage.cqp = &rf->cqp.sc_cqp; 1648 status = irdma_handle_cqp_op(rf, cqp_request); 1649 if (wait) 1650 stats_info->stats_idx = cqp_request->compl_info.op_ret_val; 1651 irdma_put_cqp_request(iwcqp, cqp_request); 1652 1653 return status; 1654 } 1655 1656 /** 1657 * irdma_cqp_ceq_cmd - Create/Destroy CEQ's after CEQ 0 1658 * @dev: pointer to device info 1659 * @sc_ceq: pointer to ceq structure 1660 * @op: Create or Destroy 1661 */ 1662 int 1663 irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_ceq *sc_ceq, 1664 u8 op) 1665 { 1666 struct irdma_cqp_request *cqp_request; 1667 struct cqp_cmds_info *cqp_info; 1668 struct irdma_pci_f *rf = dev_to_rf(dev); 1669 int status; 1670 1671 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1672 if (!cqp_request) 1673 return -ENOMEM; 1674 1675 cqp_info = &cqp_request->info; 1676 cqp_info->post_sq = 1; 1677 cqp_info->cqp_cmd = op; 1678 cqp_info->in.u.ceq_create.ceq = sc_ceq; 1679 cqp_info->in.u.ceq_create.scratch = (uintptr_t)cqp_request; 1680 1681 status = irdma_handle_cqp_op(rf, cqp_request); 1682 irdma_put_cqp_request(&rf->cqp, cqp_request); 1683 1684 return status; 1685 } 1686 1687 /** 1688 * irdma_cqp_aeq_cmd - Create/Destroy AEQ 1689 * @dev: pointer to device info 1690 * @sc_aeq: pointer to aeq structure 1691 * @op: Create or Destroy 1692 */ 1693 int 1694 irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_aeq *sc_aeq, 1695 u8 op) 1696 { 1697 struct irdma_cqp_request *cqp_request; 1698 struct cqp_cmds_info *cqp_info; 1699 struct irdma_pci_f *rf = dev_to_rf(dev); 1700 int status; 1701 1702 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1703 if (!cqp_request) 1704 return -ENOMEM; 1705 1706 cqp_info = &cqp_request->info; 1707 cqp_info->post_sq = 1; 1708 cqp_info->cqp_cmd = op; 1709 cqp_info->in.u.aeq_create.aeq = sc_aeq; 1710 cqp_info->in.u.aeq_create.scratch = (uintptr_t)cqp_request; 1711 1712 status = irdma_handle_cqp_op(rf, cqp_request); 1713 irdma_put_cqp_request(&rf->cqp, cqp_request); 1714 1715 return status; 1716 } 1717 1718 /** 1719 * irdma_cqp_ws_node_cmd - Add/modify/delete ws node 1720 * @dev: pointer to device structure 1721 * @cmd: Add, modify or delete 1722 * @node_info: pointer to ws node info 1723 */ 1724 int 1725 irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd, 1726 struct irdma_ws_node_info *node_info) 1727 { 1728 struct irdma_pci_f *rf = dev_to_rf(dev); 1729 struct irdma_cqp *iwcqp = &rf->cqp; 1730 struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp; 1731 struct irdma_cqp_request *cqp_request; 1732 struct cqp_cmds_info *cqp_info; 1733 int status; 1734 bool poll; 1735 1736 if (!rf->sc_dev.ceq_valid) 1737 poll = true; 1738 else 1739 poll = false; 1740 1741 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, !poll); 1742 if (!cqp_request) 1743 return -ENOMEM; 1744 1745 cqp_info = &cqp_request->info; 1746 memset(cqp_info, 0, sizeof(*cqp_info)); 1747 cqp_info->cqp_cmd = cmd; 1748 cqp_info->post_sq = 1; 1749 cqp_info->in.u.ws_node.info = *node_info; 1750 cqp_info->in.u.ws_node.cqp = cqp; 1751 cqp_info->in.u.ws_node.scratch = (uintptr_t)cqp_request; 1752 status = irdma_handle_cqp_op(rf, cqp_request); 1753 if (status) 1754 goto exit; 1755 1756 if (poll) { 1757 struct irdma_ccq_cqe_info compl_info; 1758 1759 status = irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_WORK_SCHED_NODE, 1760 &compl_info); 1761 node_info->qs_handle = compl_info.op_ret_val; 1762 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, 1763 "opcode=%d, compl_info.retval=%d\n", 1764 compl_info.op_code, compl_info.op_ret_val); 1765 } else { 1766 node_info->qs_handle = cqp_request->compl_info.op_ret_val; 1767 } 1768 1769 exit: 1770 irdma_put_cqp_request(&rf->cqp, cqp_request); 1771 1772 return status; 1773 } 1774 1775 /** 1776 * irdma_ah_cqp_op - perform an AH cqp operation 1777 * @rf: RDMA PCI function 1778 * @sc_ah: address handle 1779 * @cmd: AH operation 1780 * @wait: wait if true 1781 * @callback_fcn: Callback function on CQP op completion 1782 * @cb_param: parameter for callback function 1783 * 1784 * returns errno 1785 */ 1786 int 1787 irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd, 1788 bool wait, 1789 void (*callback_fcn) (struct irdma_cqp_request *), 1790 void *cb_param) 1791 { 1792 struct irdma_cqp_request *cqp_request; 1793 struct cqp_cmds_info *cqp_info; 1794 int status; 1795 1796 if (cmd != IRDMA_OP_AH_CREATE && cmd != IRDMA_OP_AH_DESTROY) 1797 return -EINVAL; 1798 1799 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait); 1800 if (!cqp_request) 1801 return -ENOMEM; 1802 1803 cqp_info = &cqp_request->info; 1804 cqp_info->cqp_cmd = cmd; 1805 cqp_info->post_sq = 1; 1806 if (cmd == IRDMA_OP_AH_CREATE) { 1807 if (!wait) 1808 irdma_get_cqp_request(cqp_request); 1809 sc_ah->ah_info.cqp_request = cqp_request; 1810 1811 cqp_info->in.u.ah_create.info = sc_ah->ah_info; 1812 cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request; 1813 cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp; 1814 } else if (cmd == IRDMA_OP_AH_DESTROY) { 1815 cqp_info->in.u.ah_destroy.info = sc_ah->ah_info; 1816 cqp_info->in.u.ah_destroy.scratch = (uintptr_t)cqp_request; 1817 cqp_info->in.u.ah_destroy.cqp = &rf->cqp.sc_cqp; 1818 } 1819 1820 if (!wait) { 1821 cqp_request->callback_fcn = callback_fcn; 1822 cqp_request->param = cb_param; 1823 } 1824 status = irdma_handle_cqp_op(rf, cqp_request); 1825 irdma_put_cqp_request(&rf->cqp, cqp_request); 1826 1827 if (status) 1828 return -ENOMEM; 1829 1830 if (wait) 1831 sc_ah->ah_info.ah_valid = (cmd != IRDMA_OP_AH_DESTROY); 1832 1833 return 0; 1834 } 1835 1836 /** 1837 * irdma_ieq_ah_cb - callback after creation of AH for IEQ 1838 * @cqp_request: pointer to cqp_request of create AH 1839 */ 1840 static void 1841 irdma_ieq_ah_cb(struct irdma_cqp_request *cqp_request) 1842 { 1843 struct irdma_sc_qp *qp = cqp_request->param; 1844 struct irdma_sc_ah *sc_ah = qp->pfpdu.ah; 1845 unsigned long flags; 1846 1847 spin_lock_irqsave(&qp->pfpdu.lock, flags); 1848 if (!cqp_request->compl_info.op_ret_val) { 1849 sc_ah->ah_info.ah_valid = true; 1850 irdma_ieq_process_fpdus(qp, qp->vsi->ieq); 1851 } else { 1852 sc_ah->ah_info.ah_valid = false; 1853 irdma_ieq_cleanup_qp(qp->vsi->ieq, qp); 1854 } 1855 spin_unlock_irqrestore(&qp->pfpdu.lock, flags); 1856 } 1857 1858 /** 1859 * irdma_ilq_ah_cb - callback after creation of AH for ILQ 1860 * @cqp_request: pointer to cqp_request of create AH 1861 */ 1862 static void 1863 irdma_ilq_ah_cb(struct irdma_cqp_request *cqp_request) 1864 { 1865 struct irdma_cm_node *cm_node = cqp_request->param; 1866 struct irdma_sc_ah *sc_ah = cm_node->ah; 1867 1868 sc_ah->ah_info.ah_valid = !cqp_request->compl_info.op_ret_val; 1869 irdma_add_conn_est_qh(cm_node); 1870 } 1871 1872 /** 1873 * irdma_puda_create_ah - create AH for ILQ/IEQ qp's 1874 * @dev: device pointer 1875 * @ah_info: Address handle info 1876 * @wait: When true will wait for operation to complete 1877 * @type: ILQ/IEQ 1878 * @cb_param: Callback param when not waiting 1879 * @ah_ret: Returned pointer to address handle if created 1880 * 1881 */ 1882 int 1883 irdma_puda_create_ah(struct irdma_sc_dev *dev, 1884 struct irdma_ah_info *ah_info, bool wait, 1885 enum puda_rsrc_type type, void *cb_param, 1886 struct irdma_sc_ah **ah_ret) 1887 { 1888 struct irdma_sc_ah *ah; 1889 struct irdma_pci_f *rf = dev_to_rf(dev); 1890 int err; 1891 1892 ah = kzalloc(sizeof(*ah), GFP_ATOMIC); 1893 *ah_ret = ah; 1894 if (!ah) 1895 return -ENOMEM; 1896 1897 err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah, 1898 &ah_info->ah_idx, &rf->next_ah); 1899 if (err) 1900 goto err_free; 1901 1902 ah->dev = dev; 1903 ah->ah_info = *ah_info; 1904 1905 if (type == IRDMA_PUDA_RSRC_TYPE_ILQ) 1906 err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait, 1907 irdma_ilq_ah_cb, cb_param); 1908 else 1909 err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait, 1910 irdma_ieq_ah_cb, cb_param); 1911 1912 if (err) 1913 goto error; 1914 return 0; 1915 1916 error: 1917 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx); 1918 err_free: 1919 kfree(ah); 1920 *ah_ret = NULL; 1921 return -ENOMEM; 1922 } 1923 1924 /** 1925 * irdma_puda_free_ah - free a puda address handle 1926 * @dev: device pointer 1927 * @ah: The address handle to free 1928 */ 1929 void 1930 irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah) 1931 { 1932 struct irdma_pci_f *rf = dev_to_rf(dev); 1933 1934 if (!ah) 1935 return; 1936 1937 if (ah->ah_info.ah_valid) { 1938 irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_DESTROY, false, NULL, NULL); 1939 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx); 1940 } 1941 1942 kfree(ah); 1943 } 1944 1945 /** 1946 * irdma_prm_add_pble_mem - add moemory to pble resources 1947 * @pprm: pble resource manager 1948 * @pchunk: chunk of memory to add 1949 */ 1950 int 1951 irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm, 1952 struct irdma_chunk *pchunk) 1953 { 1954 u64 sizeofbitmap; 1955 1956 if (pchunk->size & 0xfff) 1957 return -EINVAL; 1958 1959 sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift; 1960 1961 pchunk->bitmapbuf = bitmap_zalloc(sizeofbitmap, GFP_KERNEL); 1962 if (!pchunk->bitmapbuf) 1963 return -ENOMEM; 1964 1965 pchunk->sizeofbitmap = sizeofbitmap; 1966 /* each pble is 8 bytes hence shift by 3 */ 1967 pprm->total_pble_alloc += pchunk->size >> 3; 1968 pprm->free_pble_cnt += pchunk->size >> 3; 1969 1970 return 0; 1971 } 1972 1973 /** 1974 * irdma_prm_get_pbles - get pble's from prm 1975 * @pprm: pble resource manager 1976 * @chunkinfo: nformation about chunk where pble's were acquired 1977 * @mem_size: size of pble memory needed 1978 * @vaddr: returns virtual address of pble memory 1979 * @fpm_addr: returns fpm address of pble memory 1980 */ 1981 int 1982 irdma_prm_get_pbles(struct irdma_pble_prm *pprm, 1983 struct irdma_pble_chunkinfo *chunkinfo, u64 mem_size, 1984 u64 **vaddr, u64 *fpm_addr) 1985 { 1986 u64 bits_needed; 1987 u64 bit_idx = PBLE_INVALID_IDX; 1988 struct irdma_chunk *pchunk = NULL; 1989 struct list_head *chunk_entry = (&pprm->clist)->next; 1990 u32 offset; 1991 unsigned long flags; 1992 1993 *vaddr = NULL; 1994 *fpm_addr = 0; 1995 1996 bits_needed = DIV_ROUND_UP_ULL(mem_size, BIT_ULL(pprm->pble_shift)); 1997 1998 spin_lock_irqsave(&pprm->prm_lock, flags); 1999 while (chunk_entry != &pprm->clist) { 2000 pchunk = (struct irdma_chunk *)chunk_entry; 2001 bit_idx = bitmap_find_next_zero_area(pchunk->bitmapbuf, 2002 pchunk->sizeofbitmap, 0, 2003 bits_needed, 0); 2004 if (bit_idx < pchunk->sizeofbitmap) 2005 break; 2006 2007 /* list.next used macro */ 2008 chunk_entry = (&pchunk->list)->next; 2009 } 2010 2011 if (!pchunk || bit_idx >= pchunk->sizeofbitmap) { 2012 spin_unlock_irqrestore(&pprm->prm_lock, flags); 2013 return -ENOMEM; 2014 } 2015 2016 bitmap_set(pchunk->bitmapbuf, bit_idx, bits_needed); 2017 offset = bit_idx << pprm->pble_shift; 2018 *vaddr = (u64 *)((u8 *)pchunk->vaddr + offset); 2019 *fpm_addr = pchunk->fpm_addr + offset; 2020 2021 chunkinfo->pchunk = pchunk; 2022 chunkinfo->bit_idx = bit_idx; 2023 chunkinfo->bits_used = bits_needed; 2024 /* 3 is sizeof pble divide */ 2025 pprm->free_pble_cnt -= chunkinfo->bits_used << (pprm->pble_shift - 3); 2026 spin_unlock_irqrestore(&pprm->prm_lock, flags); 2027 2028 return 0; 2029 } 2030 2031 /** 2032 * irdma_prm_return_pbles - return pbles back to prm 2033 * @pprm: pble resource manager 2034 * @chunkinfo: chunk where pble's were acquired and to be freed 2035 */ 2036 void 2037 irdma_prm_return_pbles(struct irdma_pble_prm *pprm, 2038 struct irdma_pble_chunkinfo *chunkinfo) 2039 { 2040 unsigned long flags; 2041 2042 spin_lock_irqsave(&pprm->prm_lock, flags); 2043 pprm->free_pble_cnt += chunkinfo->bits_used << (pprm->pble_shift - 3); 2044 bitmap_clear(chunkinfo->pchunk->bitmapbuf, chunkinfo->bit_idx, 2045 chunkinfo->bits_used); 2046 spin_unlock_irqrestore(&pprm->prm_lock, flags); 2047 } 2048 2049 int 2050 irdma_map_vm_page_list(struct irdma_hw *hw, void *va, dma_addr_t * pg_dma, 2051 u32 pg_cnt) 2052 { 2053 struct page *vm_page; 2054 int i; 2055 u8 *addr; 2056 2057 addr = (u8 *)(uintptr_t)va; 2058 for (i = 0; i < pg_cnt; i++) { 2059 vm_page = vmalloc_to_page(addr); 2060 if (!vm_page) 2061 goto err; 2062 2063 pg_dma[i] = dma_map_page(hw_to_dev(hw), vm_page, 0, PAGE_SIZE, DMA_BIDIRECTIONAL); 2064 if (dma_mapping_error(hw_to_dev(hw), pg_dma[i])) 2065 goto err; 2066 2067 addr += PAGE_SIZE; 2068 } 2069 2070 return 0; 2071 2072 err: 2073 irdma_unmap_vm_page_list(hw, pg_dma, i); 2074 return -ENOMEM; 2075 } 2076 2077 void 2078 irdma_unmap_vm_page_list(struct irdma_hw *hw, dma_addr_t * pg_dma, u32 pg_cnt) 2079 { 2080 int i; 2081 2082 for (i = 0; i < pg_cnt; i++) 2083 dma_unmap_page(hw_to_dev(hw), pg_dma[i], PAGE_SIZE, DMA_BIDIRECTIONAL); 2084 } 2085 2086 /** 2087 * irdma_pble_free_paged_mem - free virtual paged memory 2088 * @chunk: chunk to free with paged memory 2089 */ 2090 void 2091 irdma_pble_free_paged_mem(struct irdma_chunk *chunk) 2092 { 2093 if (!chunk->pg_cnt) 2094 goto done; 2095 2096 irdma_unmap_vm_page_list(chunk->dev->hw, chunk->dmainfo.dmaaddrs, 2097 chunk->pg_cnt); 2098 2099 done: 2100 kfree(chunk->dmainfo.dmaaddrs); 2101 chunk->dmainfo.dmaaddrs = NULL; 2102 vfree(chunk->vaddr); 2103 chunk->vaddr = NULL; 2104 chunk->type = 0; 2105 } 2106 2107 /** 2108 * irdma_pble_get_paged_mem -allocate paged memory for pbles 2109 * @chunk: chunk to add for paged memory 2110 * @pg_cnt: number of pages needed 2111 */ 2112 int 2113 irdma_pble_get_paged_mem(struct irdma_chunk *chunk, u32 pg_cnt) 2114 { 2115 u32 size; 2116 void *va; 2117 2118 chunk->dmainfo.dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL); 2119 if (!chunk->dmainfo.dmaaddrs) 2120 return -ENOMEM; 2121 2122 size = PAGE_SIZE * pg_cnt; 2123 va = vmalloc(size); 2124 if (!va) 2125 goto err; 2126 2127 if (irdma_map_vm_page_list(chunk->dev->hw, va, chunk->dmainfo.dmaaddrs, 2128 pg_cnt)) { 2129 vfree(va); 2130 goto err; 2131 } 2132 chunk->vaddr = va; 2133 chunk->size = size; 2134 chunk->pg_cnt = pg_cnt; 2135 chunk->type = PBLE_SD_PAGED; 2136 2137 return 0; 2138 err: 2139 kfree(chunk->dmainfo.dmaaddrs); 2140 chunk->dmainfo.dmaaddrs = NULL; 2141 2142 return -ENOMEM; 2143 } 2144 2145 /** 2146 * irdma_alloc_ws_node_id - Allocate a tx scheduler node ID 2147 * @dev: device pointer 2148 */ 2149 u16 2150 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev) 2151 { 2152 struct irdma_pci_f *rf = dev_to_rf(dev); 2153 u32 next = 1; 2154 u32 node_id; 2155 2156 if (irdma_alloc_rsrc(rf, rf->allocated_ws_nodes, rf->max_ws_node_id, 2157 &node_id, &next)) 2158 return IRDMA_WS_NODE_INVALID; 2159 2160 return (u16)node_id; 2161 } 2162 2163 /** 2164 * irdma_free_ws_node_id - Free a tx scheduler node ID 2165 * @dev: device pointer 2166 * @node_id: Work scheduler node ID 2167 */ 2168 void 2169 irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id) 2170 { 2171 struct irdma_pci_f *rf = dev_to_rf(dev); 2172 2173 irdma_free_rsrc(rf, rf->allocated_ws_nodes, (u32)node_id); 2174 } 2175 2176 /** 2177 * irdma_modify_qp_to_err - Modify a QP to error 2178 * @sc_qp: qp structure 2179 */ 2180 void 2181 irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp) 2182 { 2183 struct irdma_qp *qp = sc_qp->qp_uk.back_qp; 2184 struct ib_qp_attr attr; 2185 2186 if (qp->iwdev->rf->reset) 2187 return; 2188 attr.qp_state = IB_QPS_ERR; 2189 2190 if (rdma_protocol_roce(qp->ibqp.device, 1)) 2191 irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL); 2192 else 2193 irdma_modify_qp(&qp->ibqp, &attr, IB_QP_STATE, NULL); 2194 } 2195 2196 void 2197 irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event) 2198 { 2199 struct ib_event ibevent; 2200 2201 if (!iwqp->ibqp.event_handler) 2202 return; 2203 2204 switch (event) { 2205 case IRDMA_QP_EVENT_CATASTROPHIC: 2206 ibevent.event = IB_EVENT_QP_FATAL; 2207 break; 2208 case IRDMA_QP_EVENT_ACCESS_ERR: 2209 ibevent.event = IB_EVENT_QP_ACCESS_ERR; 2210 break; 2211 case IRDMA_QP_EVENT_REQ_ERR: 2212 ibevent.event = IB_EVENT_QP_REQ_ERR; 2213 break; 2214 } 2215 ibevent.device = iwqp->ibqp.device; 2216 ibevent.element.qp = &iwqp->ibqp; 2217 iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context); 2218 } 2219 2220 static void 2221 clear_qp_ctx_addr(__le64 * ctx) 2222 { 2223 u64 tmp; 2224 2225 get_64bit_val(ctx, 272, &tmp); 2226 tmp &= GENMASK_ULL(63, 58); 2227 set_64bit_val(ctx, 272, tmp); 2228 2229 get_64bit_val(ctx, 296, &tmp); 2230 tmp &= GENMASK_ULL(7, 0); 2231 set_64bit_val(ctx, 296, tmp); 2232 2233 get_64bit_val(ctx, 312, &tmp); 2234 tmp &= GENMASK_ULL(7, 0); 2235 set_64bit_val(ctx, 312, tmp); 2236 2237 set_64bit_val(ctx, 368, 0); 2238 } 2239 2240 /** 2241 * irdma_upload_qp_context - upload raw QP context 2242 * @iwqp: QP pointer 2243 * @freeze: freeze QP 2244 * @raw: raw context flag 2245 */ 2246 int 2247 irdma_upload_qp_context(struct irdma_qp *iwqp, bool freeze, bool raw) 2248 { 2249 struct irdma_dma_mem dma_mem; 2250 struct irdma_sc_dev *dev; 2251 struct irdma_sc_qp *qp; 2252 struct irdma_cqp *iwcqp; 2253 struct irdma_cqp_request *cqp_request; 2254 struct cqp_cmds_info *cqp_info; 2255 struct irdma_upload_context_info *info; 2256 struct irdma_pci_f *rf; 2257 int ret; 2258 u32 *ctx; 2259 2260 rf = iwqp->iwdev->rf; 2261 if (!rf) 2262 return -EINVAL; 2263 2264 qp = &iwqp->sc_qp; 2265 dev = &rf->sc_dev; 2266 iwcqp = &rf->cqp; 2267 2268 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true); 2269 if (!cqp_request) 2270 return -EINVAL; 2271 2272 cqp_info = &cqp_request->info; 2273 info = &cqp_info->in.u.qp_upload_context.info; 2274 memset(info, 0, sizeof(struct irdma_upload_context_info)); 2275 cqp_info->cqp_cmd = IRDMA_OP_QP_UPLOAD_CONTEXT; 2276 cqp_info->post_sq = 1; 2277 cqp_info->in.u.qp_upload_context.dev = dev; 2278 cqp_info->in.u.qp_upload_context.scratch = (uintptr_t)cqp_request; 2279 2280 dma_mem.size = PAGE_SIZE; 2281 dma_mem.va = irdma_allocate_dma_mem(dev->hw, &dma_mem, dma_mem.size, PAGE_SIZE); 2282 if (!dma_mem.va) { 2283 irdma_put_cqp_request(&rf->cqp, cqp_request); 2284 return -ENOMEM; 2285 } 2286 2287 ctx = dma_mem.va; 2288 info->buf_pa = dma_mem.pa; 2289 info->raw_format = raw; 2290 info->freeze_qp = freeze; 2291 info->qp_type = qp->qp_uk.qp_type; /* 1 is iWARP and 2 UDA */ 2292 info->qp_id = qp->qp_uk.qp_id; 2293 ret = irdma_handle_cqp_op(rf, cqp_request); 2294 if (ret) 2295 goto error; 2296 irdma_debug(dev, IRDMA_DEBUG_QP, "PRINT CONTXT QP [%d]\n", info->qp_id); 2297 { 2298 u32 i, j; 2299 2300 clear_qp_ctx_addr(dma_mem.va); 2301 for (i = 0, j = 0; i < 32; i++, j += 4) 2302 irdma_debug(dev, IRDMA_DEBUG_QP, 2303 "%d:\t [%08X %08x %08X %08X]\n", (j * 4), 2304 ctx[j], ctx[j + 1], ctx[j + 2], ctx[j + 3]); 2305 } 2306 error: 2307 irdma_put_cqp_request(iwcqp, cqp_request); 2308 irdma_free_dma_mem(dev->hw, &dma_mem); 2309 2310 return ret; 2311 } 2312 2313 bool 2314 irdma_cq_empty(struct irdma_cq *iwcq) 2315 { 2316 struct irdma_cq_uk *ukcq; 2317 u64 qword3; 2318 __le64 *cqe; 2319 u8 polarity; 2320 2321 ukcq = &iwcq->sc_cq.cq_uk; 2322 cqe = IRDMA_GET_CURRENT_CQ_ELEM(ukcq); 2323 get_64bit_val(cqe, 24, &qword3); 2324 polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3); 2325 2326 return polarity != ukcq->polarity; 2327 } 2328 2329 void 2330 irdma_remove_cmpls_list(struct irdma_cq *iwcq) 2331 { 2332 struct irdma_cmpl_gen *cmpl_node; 2333 struct list_head *tmp_node, *list_node; 2334 2335 list_for_each_safe(list_node, tmp_node, &iwcq->cmpl_generated) { 2336 cmpl_node = list_entry(list_node, struct irdma_cmpl_gen, list); 2337 list_del(&cmpl_node->list); 2338 kfree(cmpl_node); 2339 } 2340 } 2341 2342 int 2343 irdma_generated_cmpls(struct irdma_cq *iwcq, struct irdma_cq_poll_info *cq_poll_info) 2344 { 2345 struct irdma_cmpl_gen *cmpl; 2346 2347 if (list_empty(&iwcq->cmpl_generated)) 2348 return -ENOENT; 2349 cmpl = list_first_entry_or_null(&iwcq->cmpl_generated, struct irdma_cmpl_gen, list); 2350 list_del(&cmpl->list); 2351 memcpy(cq_poll_info, &cmpl->cpi, sizeof(*cq_poll_info)); 2352 kfree(cmpl); 2353 2354 irdma_debug(iwcq->sc_cq.dev, IRDMA_DEBUG_VERBS, 2355 "%s: Poll artificially generated completion for QP 0x%X, op %u, wr_id=0x%lx\n", 2356 __func__, cq_poll_info->qp_id, cq_poll_info->op_type, 2357 cq_poll_info->wr_id); 2358 2359 return 0; 2360 } 2361 2362 /** 2363 * irdma_set_cpi_common_values - fill in values for polling info struct 2364 * @cpi: resulting structure of cq_poll_info type 2365 * @qp: QPair 2366 * @qp_num: id of the QP 2367 */ 2368 static void 2369 irdma_set_cpi_common_values(struct irdma_cq_poll_info *cpi, 2370 struct irdma_qp_uk *qp, u32 qp_num) 2371 { 2372 cpi->comp_status = IRDMA_COMPL_STATUS_FLUSHED; 2373 cpi->error = 1; 2374 cpi->major_err = IRDMA_FLUSH_MAJOR_ERR; 2375 cpi->minor_err = FLUSH_GENERAL_ERR; 2376 cpi->qp_handle = (irdma_qp_handle) (uintptr_t)qp; 2377 cpi->qp_id = qp_num; 2378 } 2379 2380 static inline void 2381 irdma_comp_handler(struct irdma_cq *cq) 2382 { 2383 if (!cq->ibcq.comp_handler) 2384 return; 2385 2386 if (atomic_cmpxchg(&cq->armed, 1, 0)) 2387 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); 2388 } 2389 2390 /** 2391 * irdma_generate_flush_completions - generate completion from WRs 2392 * @iwqp: pointer to QP 2393 */ 2394 void 2395 irdma_generate_flush_completions(struct irdma_qp *iwqp) 2396 { 2397 struct irdma_qp_uk *qp = &iwqp->sc_qp.qp_uk; 2398 struct irdma_ring *sq_ring = &qp->sq_ring; 2399 struct irdma_ring *rq_ring = &qp->rq_ring; 2400 struct irdma_cmpl_gen *cmpl; 2401 __le64 *sw_wqe; 2402 u64 wqe_qword; 2403 u32 wqe_idx; 2404 bool compl_generated = false; 2405 unsigned long flags1; 2406 2407 spin_lock_irqsave(&iwqp->iwscq->lock, flags1); 2408 if (irdma_cq_empty(iwqp->iwscq)) { 2409 unsigned long flags2; 2410 2411 spin_lock_irqsave(&iwqp->lock, flags2); 2412 while (IRDMA_RING_MORE_WORK(*sq_ring)) { 2413 cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC); 2414 if (!cmpl) { 2415 spin_unlock_irqrestore(&iwqp->lock, flags2); 2416 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); 2417 return; 2418 } 2419 2420 wqe_idx = sq_ring->tail; 2421 irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id); 2422 2423 cmpl->cpi.wr_id = qp->sq_wrtrk_array[wqe_idx].wrid; 2424 cmpl->cpi.signaled = qp->sq_wrtrk_array[wqe_idx].signaled; 2425 sw_wqe = qp->sq_base[wqe_idx].elem; 2426 get_64bit_val(sw_wqe, IRDMA_BYTE_24, &wqe_qword); 2427 cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, wqe_qword); 2428 cmpl->cpi.q_type = IRDMA_CQE_QTYPE_SQ; 2429 /* remove the SQ WR by moving SQ tail */ 2430 IRDMA_RING_SET_TAIL(*sq_ring, 2431 sq_ring->tail + qp->sq_wrtrk_array[sq_ring->tail].quanta); 2432 2433 if (cmpl->cpi.op_type == IRDMAQP_OP_NOP) { 2434 kfree(cmpl); 2435 continue; 2436 } 2437 irdma_debug(iwqp->sc_qp.dev, IRDMA_DEBUG_DEV, 2438 "%s: adding wr_id = 0x%lx SQ Completion to list qp_id=%d\n", 2439 __func__, cmpl->cpi.wr_id, qp->qp_id); 2440 list_add_tail(&cmpl->list, &iwqp->iwscq->cmpl_generated); 2441 compl_generated = true; 2442 } 2443 spin_unlock_irqrestore(&iwqp->lock, flags2); 2444 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); 2445 if (compl_generated) { 2446 irdma_comp_handler(iwqp->iwscq); 2447 compl_generated = false; 2448 } 2449 } else { 2450 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); 2451 irdma_sched_qp_flush_work(iwqp); 2452 } 2453 2454 spin_lock_irqsave(&iwqp->iwrcq->lock, flags1); 2455 if (irdma_cq_empty(iwqp->iwrcq)) { 2456 unsigned long flags2; 2457 2458 spin_lock_irqsave(&iwqp->lock, flags2); 2459 while (IRDMA_RING_MORE_WORK(*rq_ring)) { 2460 cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC); 2461 if (!cmpl) { 2462 spin_unlock_irqrestore(&iwqp->lock, flags2); 2463 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); 2464 return; 2465 } 2466 2467 wqe_idx = rq_ring->tail; 2468 irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id); 2469 2470 cmpl->cpi.wr_id = qp->rq_wrid_array[wqe_idx]; 2471 cmpl->cpi.signaled = 1; 2472 cmpl->cpi.op_type = IRDMA_OP_TYPE_REC; 2473 cmpl->cpi.q_type = IRDMA_CQE_QTYPE_RQ; 2474 /* remove the RQ WR by moving RQ tail */ 2475 IRDMA_RING_SET_TAIL(*rq_ring, rq_ring->tail + 1); 2476 irdma_debug(iwqp->sc_qp.dev, IRDMA_DEBUG_DEV, 2477 "%s: adding wr_id = 0x%lx RQ Completion to list qp_id=%d, wqe_idx=%d\n", 2478 __func__, cmpl->cpi.wr_id, qp->qp_id, 2479 wqe_idx); 2480 2481 list_add_tail(&cmpl->list, &iwqp->iwrcq->cmpl_generated); 2482 2483 compl_generated = true; 2484 } 2485 spin_unlock_irqrestore(&iwqp->lock, flags2); 2486 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); 2487 if (compl_generated) 2488 irdma_comp_handler(iwqp->iwrcq); 2489 } else { 2490 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); 2491 irdma_sched_qp_flush_work(iwqp); 2492 } 2493 } 2494 2495 /** 2496 * irdma_udqp_qs_change - change qs for UD QP in a worker thread 2497 * @iwqp: QP pointer 2498 * @user_prio: new user priority value 2499 * @qs_change: when false, only user priority changes, QS handle do not need to change 2500 */ 2501 static void 2502 irdma_udqp_qs_change(struct irdma_qp *iwqp, u8 user_prio, bool qs_change) 2503 { 2504 irdma_qp_rem_qos(&iwqp->sc_qp); 2505 if (qs_change) 2506 iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi, iwqp->ctx_info.user_pri); 2507 2508 iwqp->ctx_info.user_pri = user_prio; 2509 iwqp->sc_qp.user_pri = user_prio; 2510 2511 if (qs_change) 2512 if (iwqp->sc_qp.dev->ws_add(iwqp->sc_qp.vsi, user_prio)) 2513 irdma_dev_warn(&iwqp->iwdev->ibdev, 2514 "WS add failed during %s, qp_id: %x user_pri: %x", 2515 __func__, iwqp->ibqp.qp_num, user_prio); 2516 irdma_qp_add_qos(&iwqp->sc_qp); 2517 } 2518 2519 void 2520 irdma_udqp_qs_worker(struct work_struct *work) 2521 { 2522 struct irdma_udqs_work *udqs_work = container_of(work, struct irdma_udqs_work, work); 2523 2524 irdma_udqp_qs_change(udqs_work->iwqp, udqs_work->user_prio, udqs_work->qs_change); 2525 if (udqs_work->qs_change) 2526 irdma_cqp_qp_suspend_resume(&udqs_work->iwqp->sc_qp, IRDMA_OP_RESUME); 2527 irdma_qp_rem_ref(&udqs_work->iwqp->ibqp); 2528 kfree(udqs_work); 2529 } 2530