1 /*- 2 * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 3 * 4 * Copyright (c) 2015 - 2026 Intel Corporation 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenFabrics.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #include "irdma_main.h" 36 37 LIST_HEAD(irdma_handlers); 38 DEFINE_SPINLOCK(irdma_handler_lock); 39 40 static const char *const irdma_cqp_cmd_names[IRDMA_MAX_CQP_OPS]; 41 static const struct ae_desc ae_desc_list[] = { 42 {IRDMA_AE_AMP_UNALLOCATED_STAG, "Unallocated memory key (L-Key/R-Key)"}, 43 {IRDMA_AE_AMP_INVALID_STAG, "Invalid memory key (L-Key/R-Key)"}, 44 {IRDMA_AE_AMP_BAD_QP, 45 "Memory protection error: Accessing Memory Window (MW) which belongs to a different QP"}, 46 {IRDMA_AE_AMP_BAD_PD, 47 "Memory protection error: Accessing Memory Window (MW)/Memory Region (MR) which belongs to a different PD"}, 48 {IRDMA_AE_AMP_BAD_STAG_KEY, "Bad memory key (L-Key/R-Key)"}, 49 {IRDMA_AE_AMP_BAD_STAG_INDEX, "Bad memory key (L-Key/R-Key): Too large memory key index"}, 50 {IRDMA_AE_AMP_BOUNDS_VIOLATION, "Memory Window (MW)/Memory Region (MR) bounds violation"}, 51 {IRDMA_AE_AMP_RIGHTS_VIOLATION, "Memory Window (MW)/Memory Region (MR) rights violation"}, 52 {IRDMA_AE_AMP_TO_WRAP, 53 "Memory protection error: The address within Memory Window (MW)/Memory Region (MR) wraps"}, 54 {IRDMA_AE_AMP_FASTREG_VALID_STAG, 55 "Fastreg error: Registration to a valid MR"}, 56 {IRDMA_AE_AMP_FASTREG_MW_STAG, 57 "Fastreg error: Registration to a valid Memory Window (MW)"}, 58 {IRDMA_AE_AMP_FASTREG_INVALID_RIGHTS, "Fastreg error: Invalid rights"}, 59 {IRDMA_AE_AMP_FASTREG_INVALID_LENGTH, "Fastreg error: Invalid length"}, 60 {IRDMA_AE_AMP_INVALIDATE_SHARED, "Attempt to invalidate a shared MR"}, 61 {IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS, 62 "Attempt to remotely invalidate Memory Window (MW)/Memory Region (MR) without rights"}, 63 {IRDMA_AE_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS, 64 "Attempt to invalidate MR with a bound Memory Window (MW)"}, 65 {IRDMA_AE_AMP_MWBIND_VALID_STAG, 66 "Attempt to bind an Memory Window (MW) with a valid MW memory key (L-Key/R-Key)"}, 67 {IRDMA_AE_AMP_MWBIND_OF_MR_STAG, 68 "Attempt to bind an Memory Window (MW) with an MR memory key (L-Key/R-Key)"}, 69 {IRDMA_AE_AMP_MWBIND_TO_ZERO_BASED_STAG, 70 "Attempt to bind an Memory Window (MW) to a zero based MR"}, 71 {IRDMA_AE_AMP_MWBIND_TO_MW_STAG, 72 "Attempt to bind an Memory Window (MW) using MW memory key (L-Key/R-Key) instead of MR memory key (L-Key/R-Key)"}, 73 {IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS, "Memory Window (MW) bind error: Invalid rights"}, 74 {IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS, "Memory Window (MW) bind error: Invalid bounds"}, 75 {IRDMA_AE_AMP_MWBIND_TO_INVALID_PARENT, 76 "Memory Window (MW) bind error: Invalid parent MR"}, 77 {IRDMA_AE_AMP_MWBIND_BIND_DISABLED, 78 "Memory Window (MW) bind error: Disabled bind support"}, 79 {IRDMA_AE_PRIV_OPERATION_DENIED, 80 "Denying a privileged operation on a non-privileged QP"}, 81 {IRDMA_AE_AMP_INVALIDATE_TYPE1_MW, "Memory Window (MW) error: Invalidate type 1 MW"}, 82 {IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW, 83 "Memory Window (MW) bind error: Zero-based addressing for type 1 MW"}, 84 {IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG, 85 "Fastreg error: Invalid host page size config"}, 86 {IRDMA_AE_AMP_MWBIND_WRONG_TYPE, "MB bind error: Wrong Memory Window (MW) type"}, 87 {IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH, 88 "Fastreg error: Invalid request to change physical MR to virtual or vice versa"}, 89 {IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG, 90 "Userspace Direct Access (UDA) QP xmit error: Packet length exceeds the QP MTU"}, 91 {IRDMA_AE_UDA_XMIT_BAD_PD, 92 "Userspace Direct Access (UDA) QP xmit error: Attempt to access a different PD"}, 93 {IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT, 94 "Userspace Direct Access (UDA) QP xmit error: Too short packet length"}, 95 {IRDMA_AE_UDA_L4LEN_INVALID, 96 "Userspace Direct Access (UDA) error: Invalid packet length field"}, 97 {IRDMA_AE_BAD_CLOSE, 98 "iWARP error: Data is received when QP state is closing"}, 99 {IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE, 100 "iWARP error: FIN is received when xmit data is pending"}, 101 {IRDMA_AE_CQ_OPERATION_ERROR, "CQ overflow"}, 102 {IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO, 103 "QP error: Attempted RDMA Read when the outbound RDMA Read queue depth is zero"}, 104 {IRDMA_AE_STAG_ZERO_INVALID, 105 "Zero invalid memory key (L-Key/R-Key) on inbound RDMA R/W"}, 106 {IRDMA_AE_IB_RREQ_AND_Q1_FULL, 107 "QP error: Received RDMA Read request when the inbound RDMA Read queue is full"}, 108 {IRDMA_AE_IB_INVALID_REQUEST, 109 "QP error: Invalid operation detected by the remote peer"}, 110 {IRDMA_AE_WQE_UNEXPECTED_OPCODE, 111 "QP error: Invalid opcode in SQ WQE"}, 112 {IRDMA_AE_WQE_INVALID_PARAMETER, 113 "QP error: Invalid parameter in a WQE"}, 114 {IRDMA_AE_WQE_INVALID_FRAG_DATA, 115 "QP error: Invalid fragment in a WQE"}, 116 {IRDMA_AE_IB_REMOTE_ACCESS_ERROR, 117 "RoCEv2 error: Remote access error"}, 118 {IRDMA_AE_IB_REMOTE_OP_ERROR, 119 "RoCEv2 error: Remote operation error"}, 120 {IRDMA_AE_WQE_LSMM_TOO_LONG, "iWARP error: Connection error"}, 121 {IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN, 122 "iWARP error: Invalid message sequence number"}, 123 {IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER, 124 "iWARP error: Inbound message is too long for the available buffer"}, 125 {IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION, "iWARP error: Invalid DDP protocol version"}, 126 {IRDMA_AE_DDP_UBE_INVALID_MO, "Received message with too large offset"}, 127 {IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE, 128 "iWARP error: Inbound Send message when no receive buffer is available"}, 129 {IRDMA_AE_DDP_UBE_INVALID_QN, "iWARP error: Invalid QP number in inbound packet"}, 130 {IRDMA_AE_DDP_NO_L_BIT, 131 "iWARP error: Last bit not set in an inbound packet which completes RDMA Read"}, 132 {IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION, "iWARP error: Invalid RDMAP protocol version"}, 133 {IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE, "QP error: Invalid opcode"}, 134 {IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST, "Inbound Read request when QP isn't enabled for RDMA Read"}, 135 {IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP, 136 "Inbound RDMA Read response or RDMA Write when QP isn't enabled for RDMA R/W"}, 137 {IRDMA_AE_ROCE_RSP_LENGTH_ERROR, "RoCEv2 error: Received packet with incorrect length field"}, 138 {IRDMA_AE_ROCE_EMPTY_MCG, "RoCEv2 error: Multicast group has no valid members"}, 139 {IRDMA_AE_ROCE_BAD_MC_IP_ADDR, "RoCEv2 error: Multicast IP address doesn't match"}, 140 {IRDMA_AE_ROCE_BAD_MC_QPID, "RoCEv2 error: Multicast packet QP number isn't 0xffffff"}, 141 {IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH, "RoCEv2 error: Multicast packet protocol mismatch"}, 142 {IRDMA_AE_INVALID_ARP_ENTRY, "Invalid ARP entry"}, 143 {IRDMA_AE_INVALID_TCP_OPTION_RCVD, "iWARP error: Invalid TCP option"}, 144 {IRDMA_AE_STALE_ARP_ENTRY, "Stale ARP entry"}, 145 {IRDMA_AE_INVALID_AH_ENTRY, "Invalid AH entry"}, 146 {IRDMA_AE_LLP_CLOSE_COMPLETE, 147 "iWARP event: Graceful close complete"}, 148 {IRDMA_AE_LLP_CONNECTION_RESET, 149 "iWARP event: Received a TCP packet with a RST bit set"}, 150 {IRDMA_AE_LLP_FIN_RECEIVED, 151 "iWARP event: Received a TCP packet with a FIN bit set"}, 152 {IRDMA_AE_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH, 153 "iWARP error: Unable to close a gap in the TCP sequence"}, 154 {IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR, "Received an ICRC error"}, 155 {IRDMA_AE_LLP_SEGMENT_TOO_SMALL, 156 "iWARP error: Received a packet with insufficient space for protocol headers"}, 157 {IRDMA_AE_LLP_SYN_RECEIVED, 158 "iWARP event: Received a TCP packet with a SYN bit set"}, 159 {IRDMA_AE_LLP_TERMINATE_RECEIVED, 160 "iWARP error: Received a terminate message"}, 161 {IRDMA_AE_LLP_TOO_MANY_RETRIES, "Connection error: The max number of retries has been reached"}, 162 {IRDMA_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES, 163 "Connection error: The max number of keepalive retries has been reached"}, 164 {IRDMA_AE_LLP_DOUBT_REACHABILITY, 165 "Connection error: Doubt reachability (usually occurs after the max number of retries has been reached)"}, 166 {IRDMA_AE_LLP_CONNECTION_ESTABLISHED, 167 "iWARP event: Connection established"}, 168 {IRDMA_AE_LLP_TOO_MANY_RNRS, "RoCEv2: Too many RNR NACKs"}, 169 {IRDMA_AE_RESOURCE_EXHAUSTION, 170 "QP error: Resource exhaustion"}, 171 {IRDMA_AE_RESET_SENT, 172 "Reset sent (as requested via Modify QP)"}, 173 {IRDMA_AE_TERMINATE_SENT, 174 "Terminate sent (as requested via Modify QP)"}, 175 {IRDMA_AE_RESET_NOT_SENT, 176 "Reset not sent (but requested via Modify QP)"}, 177 {IRDMA_AE_LCE_QP_CATASTROPHIC, 178 "QP error: HW transaction resulted in catastrophic error"}, 179 {IRDMA_AE_LCE_FUNCTION_CATASTROPHIC, 180 "PCIe function error: HW transaction resulted in catastrophic error"}, 181 {IRDMA_AE_LCE_CQ_CATASTROPHIC, 182 "CQ error: HW transaction resulted in catastrophic error"}, 183 {IRDMA_AE_QP_SUSPEND_COMPLETE, "QP event: Suspend complete"}, 184 }; 185 186 /** 187 * irdma_get_ae_desc - returns AE description 188 * @ae_id: the AE number 189 */ 190 const char * 191 irdma_get_ae_desc(u16 ae_id) 192 { 193 const char *desc = ""; 194 int i; 195 196 for (i = 0; i < ARRAY_SIZE(ae_desc_list); i++) { 197 if (ae_desc_list[i].id == ae_id) { 198 desc = ae_desc_list[i].desc; 199 break; 200 } 201 } 202 return desc; 203 } 204 205 /** 206 * irdma_arp_table -manage arp table 207 * @rf: RDMA PCI function 208 * @ip_addr: ip address for device 209 * @mac_addr: mac address ptr 210 * @action: modify, delete or add/update 211 */ 212 int 213 irdma_arp_table(struct irdma_pci_f *rf, u32 *ip_addr, const u8 *mac_addr, 214 u32 action) 215 { 216 unsigned long flags; 217 int arp_index; 218 u32 ip[4] = {}; 219 220 memcpy(ip, ip_addr, sizeof(ip)); 221 222 spin_lock_irqsave(&rf->arp_lock, flags); 223 for (arp_index = 0; (u32)arp_index < rf->arp_table_size; arp_index++) { 224 if (!memcmp(rf->arp_table[arp_index].ip_addr, ip, sizeof(ip)) && 225 !rf->arp_table[arp_index].delete_pending) 226 break; 227 } 228 229 switch (action) { 230 case IRDMA_ARP_ADD_UPDATE: /* ARP Add or Update */ 231 if (arp_index == rf->arp_table_size) { 232 if (irdma_alloc_rsrc(rf, rf->allocated_arps, 233 rf->arp_table_size, 234 (u32 *)&arp_index, 235 &rf->next_arp_index)) { 236 arp_index = -1; 237 break; 238 } 239 atomic_set(&rf->arp_table[arp_index].refcnt, 0); 240 } 241 242 memcpy(rf->arp_table[arp_index].ip_addr, ip, 243 sizeof(rf->arp_table[arp_index].ip_addr)); 244 ether_addr_copy(rf->arp_table[arp_index].mac_addr, mac_addr); 245 break; 246 case IRDMA_ARP_RESOLVE: 247 if (arp_index == rf->arp_table_size) 248 arp_index = -1; 249 break; 250 case IRDMA_ARP_DELETE: 251 if (arp_index == rf->arp_table_size) { 252 arp_index = -1; 253 break; 254 } 255 256 if (!atomic_read(&rf->arp_table[arp_index].refcnt)) { 257 memset(rf->arp_table[arp_index].ip_addr, 0, 258 sizeof(rf->arp_table[arp_index].ip_addr)); 259 eth_zero_addr(rf->arp_table[arp_index].mac_addr); 260 irdma_free_rsrc(rf, rf->allocated_arps, arp_index); 261 rf->arp_table[arp_index].delete_pending = false; 262 } else { 263 rf->arp_table[arp_index].delete_pending = true; 264 arp_index = -1; /* prevent immediate CQP ARP index deletion */ 265 } 266 break; 267 default: 268 arp_index = -1; 269 break; 270 } 271 272 spin_unlock_irqrestore(&rf->arp_lock, flags); 273 return arp_index; 274 } 275 276 static int 277 irdma_get_arp(struct irdma_pci_f *rf, u16 arp_index) 278 { 279 unsigned long flags; 280 u32 ip_zero[4] = {}; 281 282 if (arp_index >= rf->arp_table_size) 283 return -EINVAL; 284 285 spin_lock_irqsave(&rf->arp_lock, flags); 286 if (!memcmp(rf->arp_table[arp_index].ip_addr, ip_zero, sizeof(ip_zero))) { 287 spin_unlock_irqrestore(&rf->arp_lock, flags); 288 return -EINVAL; 289 } 290 if (!atomic_read(&rf->arp_table[arp_index].refcnt)) 291 atomic_set(&rf->arp_table[arp_index].refcnt, 1); 292 else 293 atomic_inc(&rf->arp_table[arp_index].refcnt); 294 spin_unlock_irqrestore(&rf->arp_lock, flags); 295 296 return 0; 297 } 298 299 static void 300 irdma_put_arp(struct irdma_pci_f *rf, u16 arp_index) 301 { 302 unsigned long flags; 303 304 if (arp_index >= rf->arp_table_size) 305 return; 306 spin_lock_irqsave(&rf->arp_lock, flags); 307 if (!atomic_dec_and_test(&rf->arp_table[arp_index].refcnt)) { 308 spin_unlock_irqrestore(&rf->arp_lock, flags); 309 return; 310 } 311 312 if (rf->arp_table[arp_index].delete_pending) { 313 u32 ip_addr[4]; 314 315 memcpy(ip_addr, rf->arp_table[arp_index].ip_addr, 316 sizeof(ip_addr)); 317 memset(rf->arp_table[arp_index].ip_addr, 0, 318 sizeof(rf->arp_table[arp_index].ip_addr)); 319 eth_zero_addr(rf->arp_table[arp_index].mac_addr); 320 spin_unlock_irqrestore(&rf->arp_lock, flags); 321 irdma_arp_cqp_op(rf, arp_index, NULL, IRDMA_ARP_DELETE); 322 rf->arp_table[arp_index].delete_pending = false; 323 irdma_free_rsrc(rf, rf->allocated_arps, arp_index); 324 } else { 325 spin_unlock_irqrestore(&rf->arp_lock, flags); 326 } 327 } 328 329 /** 330 * irdma_add_arp - add a new arp entry if needed and resolve it 331 * @rf: RDMA function 332 * @ip: IP address 333 * @mac: MAC address 334 */ 335 int 336 irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, const u8 *mac) 337 { 338 irdma_manage_arp_cache(rf, mac, ip, IRDMA_ARP_ADD_UPDATE); 339 340 return irdma_arp_table(rf, ip, NULL, IRDMA_ARP_RESOLVE); 341 } 342 343 /** 344 * irdma_netdevice_event - system notifier for netdev events 345 * @notifier: not used 346 * @event: event for notifier 347 * @ptr: netdev 348 */ 349 int 350 irdma_netdevice_event(struct notifier_block *notifier, unsigned long event, 351 void *ptr) 352 { 353 struct irdma_device *iwdev; 354 struct ifnet *netdev = netdev_notifier_info_to_ifp(ptr); 355 356 iwdev = container_of(notifier, struct irdma_device, nb_netdevice_event); 357 if (iwdev->netdev != netdev) 358 return NOTIFY_DONE; 359 360 iwdev->iw_status = 1; 361 switch (event) { 362 case NETDEV_DOWN: 363 iwdev->iw_status = 0; 364 /* fallthrough */ 365 case NETDEV_UP: 366 irdma_port_ibevent(iwdev); 367 break; 368 default: 369 break; 370 } 371 372 return NOTIFY_DONE; 373 } 374 375 void 376 irdma_unregister_notifiers(struct irdma_device *iwdev) 377 { 378 unregister_netdevice_notifier(&iwdev->nb_netdevice_event); 379 } 380 381 int 382 irdma_register_notifiers(struct irdma_device *iwdev) 383 { 384 int ret; 385 386 iwdev->nb_netdevice_event.notifier_call = irdma_netdevice_event; 387 ret = register_netdevice_notifier(&iwdev->nb_netdevice_event); 388 if (ret) { 389 irdma_dev_err(&iwdev->ibdev, "register_netdevice_notifier failed\n"); 390 return ret; 391 } 392 return ret; 393 } 394 /** 395 * irdma_alloc_and_get_cqp_request - get cqp struct 396 * @cqp: device cqp ptr 397 * @wait: cqp to be used in wait mode 398 */ 399 struct irdma_cqp_request * 400 irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp, 401 bool wait) 402 { 403 struct irdma_cqp_request *cqp_request = NULL; 404 unsigned long flags; 405 406 spin_lock_irqsave(&cqp->req_lock, flags); 407 if (!list_empty(&cqp->cqp_avail_reqs)) { 408 cqp_request = list_entry(cqp->cqp_avail_reqs.next, 409 struct irdma_cqp_request, list); 410 list_del_init(&cqp_request->list); 411 } 412 spin_unlock_irqrestore(&cqp->req_lock, flags); 413 if (!cqp_request) { 414 cqp_request = kzalloc(sizeof(*cqp_request), GFP_ATOMIC); 415 if (cqp_request) { 416 cqp_request->dynamic = true; 417 if (wait) 418 init_waitqueue_head(&cqp_request->waitq); 419 } 420 } 421 if (!cqp_request) { 422 irdma_debug(cqp->sc_cqp.dev, IRDMA_DEBUG_ERR, "CQP Request Fail: No Memory"); 423 return NULL; 424 } 425 426 cqp_request->waiting = wait; 427 atomic_set(&cqp_request->refcnt, 1); 428 memset(&cqp_request->compl_info, 0, sizeof(cqp_request->compl_info)); 429 430 memset(&cqp_request->info, 0, sizeof(cqp_request->info)); 431 432 return cqp_request; 433 } 434 435 /** 436 * irdma_get_cqp_request - increase refcount for cqp_request 437 * @cqp_request: pointer to cqp_request instance 438 */ 439 static inline void 440 irdma_get_cqp_request(struct irdma_cqp_request *cqp_request) 441 { 442 atomic_inc(&cqp_request->refcnt); 443 } 444 445 /** 446 * irdma_free_cqp_request - free cqp request 447 * @cqp: cqp ptr 448 * @cqp_request: to be put back in cqp list 449 */ 450 void 451 irdma_free_cqp_request(struct irdma_cqp *cqp, 452 struct irdma_cqp_request *cqp_request) 453 { 454 unsigned long flags; 455 456 if (cqp_request->dynamic) { 457 kfree(cqp_request); 458 } else { 459 WRITE_ONCE(cqp_request->request_done, false); 460 cqp_request->callback_fcn = NULL; 461 cqp_request->waiting = false; 462 463 spin_lock_irqsave(&cqp->req_lock, flags); 464 list_add_tail(&cqp_request->list, &cqp->cqp_avail_reqs); 465 spin_unlock_irqrestore(&cqp->req_lock, flags); 466 } 467 wake_up(&cqp->remove_wq); 468 } 469 470 /** 471 * irdma_put_cqp_request - dec ref count and free if 0 472 * @cqp: cqp ptr 473 * @cqp_request: to be put back in cqp list 474 */ 475 void 476 irdma_put_cqp_request(struct irdma_cqp *cqp, 477 struct irdma_cqp_request *cqp_request) 478 { 479 if (atomic_dec_and_test(&cqp_request->refcnt)) 480 irdma_free_cqp_request(cqp, cqp_request); 481 } 482 483 /** 484 * irdma_free_pending_cqp_request -free pending cqp request objs 485 * @cqp: cqp ptr 486 * @cqp_request: to be put back in cqp list 487 */ 488 static void 489 irdma_free_pending_cqp_request(struct irdma_cqp *cqp, 490 struct irdma_cqp_request *cqp_request) 491 { 492 cqp_request->compl_info.error = true; 493 WRITE_ONCE(cqp_request->request_done, true); 494 495 if (cqp_request->waiting) 496 wake_up(&cqp_request->waitq); 497 wait_event_timeout(cqp->remove_wq, 498 atomic_read(&cqp_request->refcnt) == 1, 1000); 499 irdma_put_cqp_request(cqp, cqp_request); 500 } 501 502 /** 503 * irdma_cleanup_pending_cqp_op - clean-up cqp with no 504 * completions 505 * @rf: RDMA PCI function 506 */ 507 void 508 irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf) 509 { 510 struct irdma_sc_dev *dev = &rf->sc_dev; 511 struct irdma_cqp *cqp = &rf->cqp; 512 struct irdma_cqp_request *cqp_request = NULL; 513 struct cqp_cmds_info *pcmdinfo = NULL; 514 u32 i, pending_work, wqe_idx; 515 516 pending_work = IRDMA_RING_USED_QUANTA(cqp->sc_cqp.sq_ring); 517 wqe_idx = IRDMA_RING_CURRENT_TAIL(cqp->sc_cqp.sq_ring); 518 for (i = 0; i < pending_work; i++) { 519 cqp_request = (struct irdma_cqp_request *)(uintptr_t) 520 cqp->scratch_array[wqe_idx]; 521 if (cqp_request) 522 irdma_free_pending_cqp_request(cqp, cqp_request); 523 wqe_idx = (wqe_idx + 1) % IRDMA_RING_SIZE(cqp->sc_cqp.sq_ring); 524 } 525 526 while (!list_empty(&dev->cqp_cmd_head)) { 527 pcmdinfo = irdma_remove_cqp_head(dev); 528 cqp_request = 529 container_of(pcmdinfo, struct irdma_cqp_request, info); 530 if (cqp_request) 531 irdma_free_pending_cqp_request(cqp, cqp_request); 532 } 533 } 534 535 /** 536 * irdma_wait_event - wait for completion 537 * @rf: RDMA PCI function 538 * @cqp_request: cqp request to wait 539 */ 540 static int 541 irdma_wait_event(struct irdma_pci_f *rf, 542 struct irdma_cqp_request *cqp_request) 543 { 544 struct irdma_cqp_timeout cqp_timeout = {0}; 545 bool cqp_error = false; 546 int err_code = 0; 547 548 cqp_timeout.compl_cqp_cmds = atomic64_read(&rf->sc_dev.cqp->completed_ops); 549 do { 550 int wait_time_ms = rf->sc_dev.hw_attrs.max_cqp_compl_wait_time_ms; 551 552 irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); 553 if (wait_event_timeout(cqp_request->waitq, 554 READ_ONCE(cqp_request->request_done), 555 msecs_to_jiffies(wait_time_ms))) 556 break; 557 if (cqp_request->info.cqp_cmd_exec_status) { 558 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_CQP, 559 "%s (%d) cqp op error status reported: %d, %d %x %x\n", 560 irdma_cqp_cmd_names[cqp_request->info.cqp_cmd], 561 cqp_request->info.cqp_cmd, 562 cqp_request->info.cqp_cmd_exec_status, 563 cqp_request->compl_info.error, 564 cqp_request->compl_info.maj_err_code, 565 cqp_request->compl_info.min_err_code); 566 break; 567 } 568 569 irdma_check_cqp_progress(&cqp_timeout, &rf->sc_dev); 570 571 if (cqp_timeout.count < CQP_TIMEOUT_THRESHOLD) 572 continue; 573 574 if (!rf->reset) { 575 rf->reset = true; 576 rf->gen_ops.request_reset(rf); 577 } 578 return -ETIMEDOUT; 579 } while (1); 580 581 cqp_error = cqp_request->compl_info.error; 582 if (cqp_error) { 583 err_code = -EIO; 584 if (cqp_request->compl_info.maj_err_code == 0xFFFF) { 585 if (cqp_request->compl_info.min_err_code == 0x8002) { 586 err_code = -EBUSY; 587 } else if (cqp_request->compl_info.min_err_code == 0x8029) { 588 if (!rf->reset) { 589 rf->reset = true; 590 rf->gen_ops.request_reset(rf); 591 } 592 } 593 } 594 } 595 596 return err_code; 597 } 598 599 static const char *const irdma_cqp_cmd_names[IRDMA_MAX_CQP_OPS] = { 600 [IRDMA_OP_CEQ_DESTROY] = "Destroy CEQ Cmd", 601 [IRDMA_OP_AEQ_DESTROY] = "Destroy AEQ Cmd", 602 [IRDMA_OP_DELETE_ARP_CACHE_ENTRY] = "Delete ARP Cache Cmd", 603 [IRDMA_OP_MANAGE_APBVT_ENTRY] = "Manage APBV Table Entry Cmd", 604 [IRDMA_OP_CEQ_CREATE] = "CEQ Create Cmd", 605 [IRDMA_OP_AEQ_CREATE] = "AEQ Create Cmd", 606 [IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY] = "Manage Quad Hash Table Entry Cmd", 607 [IRDMA_OP_QP_MODIFY] = "Modify QP Cmd", 608 [IRDMA_OP_QP_UPLOAD_CONTEXT] = "Upload Context Cmd", 609 [IRDMA_OP_CQ_CREATE] = "Create CQ Cmd", 610 [IRDMA_OP_CQ_DESTROY] = "Destroy CQ Cmd", 611 [IRDMA_OP_QP_CREATE] = "Create QP Cmd", 612 [IRDMA_OP_QP_DESTROY] = "Destroy QP Cmd", 613 [IRDMA_OP_ALLOC_STAG] = "Allocate STag Cmd", 614 [IRDMA_OP_MR_REG_NON_SHARED] = "Register Non-Shared MR Cmd", 615 [IRDMA_OP_DEALLOC_STAG] = "Deallocate STag Cmd", 616 [IRDMA_OP_MW_ALLOC] = "Allocate Memory Window Cmd", 617 [IRDMA_OP_QP_FLUSH_WQES] = "Flush QP Cmd", 618 [IRDMA_OP_ADD_ARP_CACHE_ENTRY] = "Add ARP Cache Cmd", 619 [IRDMA_OP_MANAGE_PUSH_PAGE] = "Manage Push Page Cmd", 620 [IRDMA_OP_UPDATE_PE_SDS] = "Update PE SDs Cmd", 621 [IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE] = "Manage HMC PM Function Table Cmd", 622 [IRDMA_OP_SUSPEND] = "Suspend QP Cmd", 623 [IRDMA_OP_RESUME] = "Resume QP Cmd", 624 [IRDMA_OP_QUERY_FPM_VAL] = "Query FPM Values Cmd", 625 [IRDMA_OP_COMMIT_FPM_VAL] = "Commit FPM Values Cmd", 626 [IRDMA_OP_AH_CREATE] = "Create Address Handle Cmd", 627 [IRDMA_OP_AH_MODIFY] = "Modify Address Handle Cmd", 628 [IRDMA_OP_AH_DESTROY] = "Destroy Address Handle Cmd", 629 [IRDMA_OP_MC_CREATE] = "Create Multicast Group Cmd", 630 [IRDMA_OP_MC_DESTROY] = "Destroy Multicast Group Cmd", 631 [IRDMA_OP_MC_MODIFY] = "Modify Multicast Group Cmd", 632 [IRDMA_OP_STATS_ALLOCATE] = "Add Statistics Instance Cmd", 633 [IRDMA_OP_STATS_FREE] = "Free Statistics Instance Cmd", 634 [IRDMA_OP_STATS_GATHER] = "Gather Statistics Cmd", 635 [IRDMA_OP_WS_ADD_NODE] = "Add Work Scheduler Node Cmd", 636 [IRDMA_OP_WS_MODIFY_NODE] = "Modify Work Scheduler Node Cmd", 637 [IRDMA_OP_WS_DELETE_NODE] = "Delete Work Scheduler Node Cmd", 638 [IRDMA_OP_WS_FAILOVER_START] = "Failover Start Cmd", 639 [IRDMA_OP_WS_FAILOVER_COMPLETE] = "Failover Complete Cmd", 640 [IRDMA_OP_SET_UP_MAP] = "Set UP-UP Mapping Cmd", 641 [IRDMA_OP_GEN_AE] = "Generate AE Cmd", 642 [IRDMA_OP_QUERY_RDMA_FEATURES] = "RDMA Get Features Cmd", 643 [IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY] = "Allocate Local MAC Entry Cmd", 644 [IRDMA_OP_ADD_LOCAL_MAC_ENTRY] = "Add Local MAC Entry Cmd", 645 [IRDMA_OP_DELETE_LOCAL_MAC_ENTRY] = "Delete Local MAC Entry Cmd", 646 [IRDMA_OP_CQ_MODIFY] = "CQ Modify Cmd", 647 }; 648 649 static const struct irdma_cqp_err_info irdma_noncrit_err_list[] = { 650 {0xffff, 0x8002, "Invalid State"}, 651 {0xffff, 0x8006, "Flush No Wqe Pending"}, 652 {0xffff, 0x8007, "Modify QP Bad Close"}, 653 {0xffff, 0x8009, "LLP Closed"}, 654 {0xffff, 0x800a, "Reset Not Sent"}, 655 {0xffff, 0x0200, "Failover Pending"}, 656 }; 657 658 /** 659 * irdma_cqp_crit_err - check if CQP error is critical 660 * @dev: pointer to dev structure 661 * @cqp_cmd: code for last CQP operation 662 * @maj_err_code: major error code 663 * @min_err_code: minot error code 664 */ 665 bool 666 irdma_cqp_crit_err(struct irdma_sc_dev *dev, u8 cqp_cmd, 667 u16 maj_err_code, u16 min_err_code) 668 { 669 int i; 670 671 for (i = 0; i < ARRAY_SIZE(irdma_noncrit_err_list); ++i) { 672 if (maj_err_code == irdma_noncrit_err_list[i].maj && 673 min_err_code == irdma_noncrit_err_list[i].min) { 674 irdma_debug(dev, IRDMA_DEBUG_CQP, 675 "[%s Error][%s] maj=0x%x min=0x%x\n", 676 irdma_noncrit_err_list[i].desc, 677 irdma_cqp_cmd_names[cqp_cmd], maj_err_code, 678 min_err_code); 679 return false; 680 } 681 } 682 return true; 683 } 684 685 /** 686 * irdma_handle_cqp_op - process cqp command 687 * @rf: RDMA PCI function 688 * @cqp_request: cqp request to process 689 */ 690 int 691 irdma_handle_cqp_op(struct irdma_pci_f *rf, 692 struct irdma_cqp_request *cqp_request) 693 { 694 struct irdma_sc_dev *dev = &rf->sc_dev; 695 struct cqp_cmds_info *info = &cqp_request->info; 696 int status; 697 bool put_cqp_request = true; 698 699 if (rf->reset) 700 return (info->create ? -EBUSY : 0); 701 702 irdma_get_cqp_request(cqp_request); 703 status = irdma_process_cqp_cmd(dev, info); 704 if (status) 705 goto err; 706 707 if (cqp_request->waiting) { 708 put_cqp_request = false; 709 status = irdma_wait_event(rf, cqp_request); 710 if (status) 711 goto err; 712 } 713 714 return 0; 715 716 err: 717 if (irdma_cqp_crit_err(dev, info->cqp_cmd, 718 cqp_request->compl_info.maj_err_code, 719 cqp_request->compl_info.min_err_code)) { 720 int qpn = -1; 721 722 if (info->cqp_cmd == IRDMA_OP_QP_CREATE) 723 qpn = cqp_request->info.in.u.qp_create.qp->qp_uk.qp_id; 724 else if (info->cqp_cmd == IRDMA_OP_QP_MODIFY) 725 qpn = cqp_request->info.in.u.qp_modify.qp->qp_uk.qp_id; 726 else if (info->cqp_cmd == IRDMA_OP_QP_DESTROY) 727 qpn = cqp_request->info.in.u.qp_destroy.qp->qp_uk.qp_id; 728 729 irdma_dev_err(&rf->iwdev->ibdev, 730 "[%s Error] status=%d waiting=%d completion_err=%d maj=0x%x min=0x%x qpn=%d\n", 731 irdma_cqp_cmd_names[info->cqp_cmd], status, 732 cqp_request->waiting, cqp_request->compl_info.error, 733 cqp_request->compl_info.maj_err_code, 734 cqp_request->compl_info.min_err_code, qpn); 735 } 736 737 if (put_cqp_request) 738 irdma_put_cqp_request(&rf->cqp, cqp_request); 739 740 return status; 741 } 742 743 void 744 irdma_qp_add_ref(struct ib_qp *ibqp) 745 { 746 struct irdma_qp *iwqp = to_iwqp(ibqp); 747 748 atomic_inc(&iwqp->refcnt); 749 } 750 751 void 752 irdma_qp_rem_ref(struct ib_qp *ibqp) 753 { 754 struct irdma_qp *iwqp = to_iwqp(ibqp); 755 struct irdma_device *iwdev = iwqp->iwdev; 756 unsigned long flags; 757 758 spin_lock_irqsave(&iwdev->rf->qptable_lock, flags); 759 if (!atomic_dec_and_test(&iwqp->refcnt)) { 760 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); 761 return; 762 } 763 764 iwdev->rf->qp_table[iwqp->ibqp.qp_num] = NULL; 765 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); 766 complete(&iwqp->free_qp); 767 } 768 769 void 770 irdma_cq_add_ref(struct ib_cq *ibcq) 771 { 772 struct irdma_cq *iwcq = to_iwcq(ibcq); 773 774 atomic_inc(&iwcq->refcnt); 775 } 776 777 void 778 irdma_cq_rem_ref(struct ib_cq *ibcq) 779 { 780 struct irdma_cq *iwcq = to_iwcq(ibcq); 781 struct irdma_pci_f *rf = container_of(iwcq->sc_cq.dev, struct irdma_pci_f, sc_dev); 782 unsigned long flags; 783 784 spin_lock_irqsave(&rf->cqtable_lock, flags); 785 if (!atomic_dec_and_test(&iwcq->refcnt)) { 786 spin_unlock_irqrestore(&rf->cqtable_lock, flags); 787 return; 788 } 789 790 WRITE_ONCE(rf->cq_table[iwcq->cq_num], NULL); 791 spin_unlock_irqrestore(&rf->cqtable_lock, flags); 792 complete(&iwcq->free_cq); 793 } 794 795 struct ib_device * 796 to_ibdev(struct irdma_sc_dev *dev) 797 { 798 return &(container_of(dev, struct irdma_pci_f, sc_dev))->iwdev->ibdev; 799 } 800 801 /** 802 * irdma_get_qp - get qp address 803 * @device: iwarp device 804 * @qpn: qp number 805 */ 806 struct ib_qp * 807 irdma_get_qp(struct ib_device *device, int qpn) 808 { 809 struct irdma_device *iwdev = to_iwdev(device); 810 811 if (qpn < IW_FIRST_QPN || qpn >= iwdev->rf->max_qp) 812 return NULL; 813 814 return &iwdev->rf->qp_table[qpn]->ibqp; 815 } 816 817 /** 818 * irdma_remove_cqp_head - return head entry and remove 819 * @dev: device 820 */ 821 void * 822 irdma_remove_cqp_head(struct irdma_sc_dev *dev) 823 { 824 struct list_head *entry; 825 struct list_head *list = &dev->cqp_cmd_head; 826 827 if (list_empty(list)) 828 return NULL; 829 830 entry = list->next; 831 list_del(entry); 832 833 return entry; 834 } 835 836 /** 837 * irdma_cqp_sds_cmd - create cqp command for sd 838 * @dev: hardware control device structure 839 * @sdinfo: information for sd cqp 840 * 841 */ 842 int 843 irdma_cqp_sds_cmd(struct irdma_sc_dev *dev, 844 struct irdma_update_sds_info *sdinfo) 845 { 846 struct irdma_cqp_request *cqp_request; 847 struct cqp_cmds_info *cqp_info; 848 struct irdma_pci_f *rf = dev_to_rf(dev); 849 int status; 850 851 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 852 if (!cqp_request) 853 return -ENOMEM; 854 855 cqp_info = &cqp_request->info; 856 memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo, 857 sizeof(cqp_info->in.u.update_pe_sds.info)); 858 cqp_info->cqp_cmd = IRDMA_OP_UPDATE_PE_SDS; 859 cqp_info->post_sq = 1; 860 cqp_info->in.u.update_pe_sds.dev = dev; 861 cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request; 862 863 status = irdma_handle_cqp_op(rf, cqp_request); 864 irdma_put_cqp_request(&rf->cqp, cqp_request); 865 866 return status; 867 } 868 869 /** 870 * irdma_cqp_qp_suspend_resume - cqp command for suspend/resume 871 * @qp: hardware control qp 872 * @op: suspend or resume 873 */ 874 int 875 irdma_cqp_qp_suspend_resume(struct irdma_sc_qp *qp, u8 op) 876 { 877 struct irdma_sc_dev *dev = qp->dev; 878 struct irdma_cqp_request *cqp_request; 879 struct irdma_sc_cqp *cqp = dev->cqp; 880 struct cqp_cmds_info *cqp_info; 881 struct irdma_pci_f *rf = dev_to_rf(dev); 882 int status; 883 884 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false); 885 if (!cqp_request) 886 return -ENOMEM; 887 888 cqp_info = &cqp_request->info; 889 cqp_info->cqp_cmd = op; 890 cqp_info->in.u.suspend_resume.cqp = cqp; 891 cqp_info->in.u.suspend_resume.qp = qp; 892 cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request; 893 894 status = irdma_handle_cqp_op(rf, cqp_request); 895 irdma_put_cqp_request(&rf->cqp, cqp_request); 896 897 return status; 898 } 899 900 /** 901 * irdma_term_modify_qp - modify qp for term message 902 * @qp: hardware control qp 903 * @next_state: qp's next state 904 * @term: terminate code 905 * @term_len: length 906 */ 907 void 908 irdma_term_modify_qp(struct irdma_sc_qp *qp, u8 next_state, u8 term, 909 u8 term_len) 910 { 911 struct irdma_qp *iwqp; 912 913 iwqp = qp->qp_uk.back_qp; 914 irdma_next_iw_state(iwqp, next_state, 0, term, term_len); 915 }; 916 917 /** 918 * irdma_terminate_done - after terminate is completed 919 * @qp: hardware control qp 920 * @timeout_occurred: indicates if terminate timer expired 921 */ 922 void 923 irdma_terminate_done(struct irdma_sc_qp *qp, int timeout_occurred) 924 { 925 struct irdma_qp *iwqp; 926 u8 hte = 0; 927 bool first_time; 928 unsigned long flags; 929 930 iwqp = qp->qp_uk.back_qp; 931 spin_lock_irqsave(&iwqp->lock, flags); 932 if (iwqp->hte_added) { 933 iwqp->hte_added = 0; 934 hte = 1; 935 } 936 first_time = !(qp->term_flags & IRDMA_TERM_DONE); 937 qp->term_flags |= IRDMA_TERM_DONE; 938 spin_unlock_irqrestore(&iwqp->lock, flags); 939 if (first_time) { 940 if (!timeout_occurred) 941 irdma_terminate_del_timer(qp); 942 943 irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, hte, 0, 0); 944 irdma_cm_disconn(iwqp); 945 } 946 } 947 948 static void 949 irdma_terminate_timeout(struct timer_list *t) 950 { 951 struct irdma_qp *iwqp = timer_container_of(iwqp, t, terminate_timer); 952 struct irdma_sc_qp *qp = &iwqp->sc_qp; 953 954 irdma_terminate_done(qp, 1); 955 irdma_qp_rem_ref(&iwqp->ibqp); 956 } 957 958 /** 959 * irdma_terminate_start_timer - start terminate timeout 960 * @qp: hardware control qp 961 */ 962 void 963 irdma_terminate_start_timer(struct irdma_sc_qp *qp) 964 { 965 struct irdma_qp *iwqp; 966 967 iwqp = qp->qp_uk.back_qp; 968 irdma_qp_add_ref(&iwqp->ibqp); 969 timer_setup(&iwqp->terminate_timer, irdma_terminate_timeout, 0); 970 iwqp->terminate_timer.expires = jiffies + HZ; 971 972 add_timer(&iwqp->terminate_timer); 973 } 974 975 /** 976 * irdma_terminate_del_timer - delete terminate timeout 977 * @qp: hardware control qp 978 */ 979 void 980 irdma_terminate_del_timer(struct irdma_sc_qp *qp) 981 { 982 struct irdma_qp *iwqp; 983 int ret; 984 985 iwqp = qp->qp_uk.back_qp; 986 ret = irdma_del_timer_compat(&iwqp->terminate_timer); 987 if (ret) 988 irdma_qp_rem_ref(&iwqp->ibqp); 989 } 990 991 /** 992 * irdma_cqp_query_fpm_val_cmd - send cqp command for fpm 993 * @dev: function device struct 994 * @val_mem: buffer for fpm 995 * @hmc_fn_id: function id for fpm 996 */ 997 int 998 irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev *dev, 999 struct irdma_dma_mem *val_mem, u16 hmc_fn_id) 1000 { 1001 struct irdma_cqp_request *cqp_request; 1002 struct cqp_cmds_info *cqp_info; 1003 struct irdma_pci_f *rf = dev_to_rf(dev); 1004 int status; 1005 1006 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1007 if (!cqp_request) 1008 return -ENOMEM; 1009 1010 cqp_info = &cqp_request->info; 1011 cqp_request->param = NULL; 1012 cqp_info->in.u.query_fpm_val.cqp = dev->cqp; 1013 cqp_info->in.u.query_fpm_val.fpm_val_pa = val_mem->pa; 1014 cqp_info->in.u.query_fpm_val.fpm_val_va = val_mem->va; 1015 cqp_info->in.u.query_fpm_val.hmc_fn_id = hmc_fn_id; 1016 cqp_info->cqp_cmd = IRDMA_OP_QUERY_FPM_VAL; 1017 cqp_info->post_sq = 1; 1018 cqp_info->in.u.query_fpm_val.scratch = (uintptr_t)cqp_request; 1019 1020 status = irdma_handle_cqp_op(rf, cqp_request); 1021 irdma_put_cqp_request(&rf->cqp, cqp_request); 1022 1023 return status; 1024 } 1025 1026 /** 1027 * irdma_cqp_commit_fpm_val_cmd - commit fpm values in hw 1028 * @dev: hardware control device structure 1029 * @val_mem: buffer with fpm values 1030 * @hmc_fn_id: function id for fpm 1031 */ 1032 int 1033 irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev *dev, 1034 struct irdma_dma_mem *val_mem, u16 hmc_fn_id) 1035 { 1036 struct irdma_cqp_request *cqp_request; 1037 struct cqp_cmds_info *cqp_info; 1038 struct irdma_pci_f *rf = dev_to_rf(dev); 1039 int status; 1040 1041 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1042 if (!cqp_request) 1043 return -ENOMEM; 1044 1045 cqp_info = &cqp_request->info; 1046 cqp_request->param = NULL; 1047 cqp_info->in.u.commit_fpm_val.cqp = dev->cqp; 1048 cqp_info->in.u.commit_fpm_val.fpm_val_pa = val_mem->pa; 1049 cqp_info->in.u.commit_fpm_val.fpm_val_va = val_mem->va; 1050 cqp_info->in.u.commit_fpm_val.hmc_fn_id = hmc_fn_id; 1051 cqp_info->cqp_cmd = IRDMA_OP_COMMIT_FPM_VAL; 1052 cqp_info->post_sq = 1; 1053 cqp_info->in.u.commit_fpm_val.scratch = (uintptr_t)cqp_request; 1054 1055 status = irdma_handle_cqp_op(rf, cqp_request); 1056 irdma_put_cqp_request(&rf->cqp, cqp_request); 1057 1058 return status; 1059 } 1060 1061 /** 1062 * irdma_cqp_cq_create_cmd - create a cq for the cqp 1063 * @dev: device pointer 1064 * @cq: pointer to created cq 1065 */ 1066 int 1067 irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq) 1068 { 1069 struct irdma_pci_f *rf = dev_to_rf(dev); 1070 struct irdma_cqp *iwcqp = &rf->cqp; 1071 struct irdma_cqp_request *cqp_request; 1072 struct cqp_cmds_info *cqp_info; 1073 int status; 1074 1075 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true); 1076 if (!cqp_request) 1077 return -ENOMEM; 1078 1079 cqp_info = &cqp_request->info; 1080 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE; 1081 cqp_info->post_sq = 1; 1082 cqp_info->in.u.cq_create.cq = cq; 1083 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request; 1084 cqp_info->create = true; 1085 1086 status = irdma_handle_cqp_op(rf, cqp_request); 1087 irdma_put_cqp_request(iwcqp, cqp_request); 1088 1089 return status; 1090 } 1091 1092 /** 1093 * irdma_cqp_qp_create_cmd - create a qp for the cqp 1094 * @dev: device pointer 1095 * @qp: pointer to created qp 1096 */ 1097 int 1098 irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp) 1099 { 1100 struct irdma_pci_f *rf = dev_to_rf(dev); 1101 struct irdma_cqp *iwcqp = &rf->cqp; 1102 struct irdma_cqp_request *cqp_request; 1103 struct cqp_cmds_info *cqp_info; 1104 struct irdma_create_qp_info *qp_info; 1105 int status; 1106 1107 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true); 1108 if (!cqp_request) 1109 return -ENOMEM; 1110 1111 cqp_info = &cqp_request->info; 1112 qp_info = &cqp_request->info.in.u.qp_create.info; 1113 qp_info->cq_num_valid = true; 1114 qp_info->next_iwarp_state = IRDMA_QP_STATE_RTS; 1115 cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE; 1116 cqp_info->post_sq = 1; 1117 cqp_info->in.u.qp_create.qp = qp; 1118 cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request; 1119 cqp_info->create = true; 1120 1121 status = irdma_handle_cqp_op(rf, cqp_request); 1122 irdma_put_cqp_request(iwcqp, cqp_request); 1123 1124 return status; 1125 } 1126 1127 /** 1128 * irdma_dealloc_push_page - free a push page for qp 1129 * @rf: RDMA PCI function 1130 * @iwqp: QP pointer 1131 */ 1132 void 1133 irdma_dealloc_push_page(struct irdma_pci_f *rf, 1134 struct irdma_qp *iwqp) 1135 { 1136 struct irdma_sc_qp *qp = &iwqp->sc_qp; 1137 struct irdma_cqp_request *cqp_request; 1138 struct cqp_cmds_info *cqp_info; 1139 int status; 1140 1141 if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) 1142 return; 1143 1144 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false); 1145 if (!cqp_request) 1146 return; 1147 1148 cqp_info = &cqp_request->info; 1149 cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE; 1150 cqp_info->post_sq = 1; 1151 cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx; 1152 cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle; 1153 cqp_info->in.u.manage_push_page.info.free_page = 1; 1154 cqp_info->in.u.manage_push_page.info.push_page_type = 0; 1155 cqp_info->in.u.manage_push_page.cqp = &rf->cqp.sc_cqp; 1156 cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request; 1157 1158 status = irdma_handle_cqp_op(rf, cqp_request); 1159 if (!status) 1160 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX; 1161 irdma_put_cqp_request(&rf->cqp, cqp_request); 1162 } 1163 1164 /** 1165 * irdma_cq_wq_destroy - send cq destroy cqp 1166 * @rf: RDMA PCI function 1167 * @cq: hardware control cq 1168 */ 1169 void 1170 irdma_cq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_cq *cq) 1171 { 1172 struct irdma_cqp_request *cqp_request; 1173 struct cqp_cmds_info *cqp_info; 1174 1175 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1176 if (!cqp_request) 1177 return; 1178 1179 cqp_info = &cqp_request->info; 1180 cqp_info->cqp_cmd = IRDMA_OP_CQ_DESTROY; 1181 cqp_info->post_sq = 1; 1182 cqp_info->in.u.cq_destroy.cq = cq; 1183 cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request; 1184 1185 irdma_handle_cqp_op(rf, cqp_request); 1186 irdma_put_cqp_request(&rf->cqp, cqp_request); 1187 } 1188 1189 /** 1190 * irdma_hw_modify_qp_callback - handle state for modifyQPs that don't wait 1191 * @cqp_request: modify QP completion 1192 */ 1193 static void 1194 irdma_hw_modify_qp_callback(struct irdma_cqp_request *cqp_request) 1195 { 1196 struct cqp_cmds_info *cqp_info; 1197 struct irdma_qp *iwqp; 1198 1199 cqp_info = &cqp_request->info; 1200 iwqp = cqp_info->in.u.qp_modify.qp->qp_uk.back_qp; 1201 atomic_dec(&iwqp->hw_mod_qp_pend); 1202 wake_up(&iwqp->mod_qp_waitq); 1203 } 1204 1205 /** 1206 * irdma_hw_modify_qp - setup cqp for modify qp 1207 * @iwdev: RDMA device 1208 * @iwqp: qp ptr (user or kernel) 1209 * @info: info for modify qp 1210 * @wait: flag to wait or not for modify qp completion 1211 */ 1212 int 1213 irdma_hw_modify_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp, 1214 struct irdma_modify_qp_info *info, bool wait) 1215 { 1216 int status; 1217 struct irdma_pci_f *rf = iwdev->rf; 1218 struct irdma_cqp_request *cqp_request; 1219 struct cqp_cmds_info *cqp_info; 1220 struct irdma_modify_qp_info *m_info; 1221 1222 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait); 1223 if (!cqp_request) 1224 return -ENOMEM; 1225 1226 if (!wait) { 1227 cqp_request->callback_fcn = irdma_hw_modify_qp_callback; 1228 atomic_inc(&iwqp->hw_mod_qp_pend); 1229 } 1230 cqp_info = &cqp_request->info; 1231 m_info = &cqp_info->in.u.qp_modify.info; 1232 memcpy(m_info, info, sizeof(*m_info)); 1233 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY; 1234 cqp_info->post_sq = 1; 1235 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp; 1236 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request; 1237 cqp_info->create = false; 1238 status = irdma_handle_cqp_op(rf, cqp_request); 1239 irdma_put_cqp_request(&rf->cqp, cqp_request); 1240 if (status) { 1241 if (rdma_protocol_roce(&iwdev->ibdev, 1)) 1242 return status; 1243 1244 switch (m_info->next_iwarp_state) { 1245 struct irdma_gen_ae_info ae_info; 1246 1247 case IRDMA_QP_STATE_RTS: 1248 case IRDMA_QP_STATE_IDLE: 1249 case IRDMA_QP_STATE_TERMINATE: 1250 case IRDMA_QP_STATE_CLOSING: 1251 if (info->curr_iwarp_state == IRDMA_QP_STATE_IDLE) 1252 irdma_send_reset(iwqp->cm_node); 1253 else 1254 iwqp->sc_qp.term_flags = IRDMA_TERM_DONE; 1255 if (!wait) { 1256 ae_info.ae_code = IRDMA_AE_BAD_CLOSE; 1257 ae_info.ae_src = 0; 1258 irdma_gen_ae(rf, &iwqp->sc_qp, &ae_info, false); 1259 } else { 1260 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, 1261 wait); 1262 if (!cqp_request) 1263 return -ENOMEM; 1264 1265 cqp_info = &cqp_request->info; 1266 m_info = &cqp_info->in.u.qp_modify.info; 1267 memcpy(m_info, info, sizeof(*m_info)); 1268 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY; 1269 cqp_info->post_sq = 1; 1270 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp; 1271 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request; 1272 m_info->next_iwarp_state = IRDMA_QP_STATE_ERROR; 1273 m_info->reset_tcp_conn = true; 1274 irdma_handle_cqp_op(rf, cqp_request); 1275 irdma_put_cqp_request(&rf->cqp, cqp_request); 1276 } 1277 break; 1278 case IRDMA_QP_STATE_ERROR: 1279 default: 1280 break; 1281 } 1282 } 1283 1284 return status; 1285 } 1286 1287 /** 1288 * irdma_cqp_cq_destroy_cmd - destroy the cqp cq 1289 * @dev: device pointer 1290 * @cq: pointer to cq 1291 */ 1292 void 1293 irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq) 1294 { 1295 struct irdma_pci_f *rf = dev_to_rf(dev); 1296 1297 irdma_cq_wq_destroy(rf, cq); 1298 } 1299 1300 /** 1301 * irdma_cqp_qp_destroy_cmd - destroy the cqp 1302 * @dev: device pointer 1303 * @qp: pointer to qp 1304 */ 1305 int 1306 irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp) 1307 { 1308 struct irdma_pci_f *rf = dev_to_rf(dev); 1309 struct irdma_cqp *iwcqp = &rf->cqp; 1310 struct irdma_cqp_request *cqp_request; 1311 struct cqp_cmds_info *cqp_info; 1312 int status; 1313 1314 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true); 1315 if (!cqp_request) 1316 return -ENOMEM; 1317 1318 cqp_info = &cqp_request->info; 1319 cqp_info->cqp_cmd = IRDMA_OP_QP_DESTROY; 1320 cqp_info->post_sq = 1; 1321 cqp_info->in.u.qp_destroy.qp = qp; 1322 cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request; 1323 cqp_info->in.u.qp_destroy.remove_hash_idx = true; 1324 1325 status = irdma_handle_cqp_op(rf, cqp_request); 1326 irdma_put_cqp_request(&rf->cqp, cqp_request); 1327 1328 return status; 1329 } 1330 1331 /** 1332 * irdma_ieq_mpa_crc_ae - generate AE for crc error 1333 * @dev: hardware control device structure 1334 * @qp: hardware control qp 1335 */ 1336 void 1337 irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp) 1338 { 1339 struct irdma_gen_ae_info info = {0}; 1340 struct irdma_pci_f *rf = dev_to_rf(dev); 1341 1342 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_AEQ, "Generate MPA CRC AE\n"); 1343 info.ae_code = IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR; 1344 info.ae_src = IRDMA_AE_SOURCE_RQ; 1345 irdma_gen_ae(rf, qp, &info, false); 1346 } 1347 1348 /** 1349 * irdma_ieq_get_qp - get qp based on quad in puda buffer 1350 * @dev: hardware control device structure 1351 * @buf: receive puda buffer on exception q 1352 */ 1353 struct irdma_sc_qp * 1354 irdma_ieq_get_qp(struct irdma_sc_dev *dev, 1355 struct irdma_puda_buf *buf) 1356 { 1357 struct irdma_qp *iwqp; 1358 struct irdma_cm_node *cm_node; 1359 struct irdma_device *iwdev = buf->vsi->back_vsi; 1360 u32 loc_addr[4] = {0}; 1361 u32 rem_addr[4] = {0}; 1362 u16 loc_port, rem_port; 1363 struct ip6_hdr *ip6h; 1364 struct ip *iph = (struct ip *)buf->iph; 1365 struct tcphdr *tcph = (struct tcphdr *)buf->tcph; 1366 1367 if (iph->ip_v == 4) { 1368 loc_addr[0] = ntohl(iph->ip_dst.s_addr); 1369 rem_addr[0] = ntohl(iph->ip_src.s_addr); 1370 } else { 1371 ip6h = (struct ip6_hdr *)buf->iph; 1372 irdma_copy_ip_ntohl(loc_addr, ip6h->ip6_dst.__u6_addr.__u6_addr32); 1373 irdma_copy_ip_ntohl(rem_addr, ip6h->ip6_src.__u6_addr.__u6_addr32); 1374 } 1375 loc_port = ntohs(tcph->th_dport); 1376 rem_port = ntohs(tcph->th_sport); 1377 cm_node = irdma_find_node(&iwdev->cm_core, rem_port, rem_addr, loc_port, 1378 loc_addr, buf->vlan_valid ? buf->vlan_id : 0xFFFF); 1379 if (!cm_node) 1380 return NULL; 1381 1382 iwqp = cm_node->iwqp; 1383 irdma_rem_ref_cmnode(cm_node); 1384 1385 return &iwqp->sc_qp; 1386 } 1387 1388 /** 1389 * irdma_send_ieq_ack - ACKs for duplicate or OOO partials FPDUs 1390 * @qp: qp ptr 1391 */ 1392 void 1393 irdma_send_ieq_ack(struct irdma_sc_qp *qp) 1394 { 1395 struct irdma_cm_node *cm_node = ((struct irdma_qp *)qp->qp_uk.back_qp)->cm_node; 1396 struct irdma_puda_buf *buf = qp->pfpdu.lastrcv_buf; 1397 struct tcphdr *tcph = (struct tcphdr *)buf->tcph; 1398 1399 cm_node->tcp_cntxt.rcv_nxt = qp->pfpdu.nextseqnum; 1400 cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->th_ack); 1401 1402 irdma_send_ack(cm_node); 1403 } 1404 1405 /** 1406 * irdma_puda_ieq_get_ah_info - get AH info from IEQ buffer 1407 * @qp: qp pointer 1408 * @ah_info: AH info pointer 1409 */ 1410 void 1411 irdma_puda_ieq_get_ah_info(struct irdma_sc_qp *qp, 1412 struct irdma_ah_info *ah_info) 1413 { 1414 struct irdma_puda_buf *buf = qp->pfpdu.ah_buf; 1415 struct ip *iph; 1416 struct ip6_hdr *ip6h; 1417 1418 memset(ah_info, 0, sizeof(*ah_info)); 1419 ah_info->do_lpbk = true; 1420 ah_info->vlan_tag = buf->vlan_id; 1421 ah_info->insert_vlan_tag = buf->vlan_valid; 1422 ah_info->ipv4_valid = buf->ipv4; 1423 ah_info->vsi = qp->vsi; 1424 1425 if (buf->smac_valid) 1426 ether_addr_copy(ah_info->mac_addr, buf->smac); 1427 1428 if (buf->ipv4) { 1429 ah_info->ipv4_valid = true; 1430 iph = (struct ip *)buf->iph; 1431 ah_info->hop_ttl = iph->ip_ttl; 1432 ah_info->tc_tos = iph->ip_tos; 1433 ah_info->dest_ip_addr[0] = ntohl(iph->ip_dst.s_addr); 1434 ah_info->src_ip_addr[0] = ntohl(iph->ip_src.s_addr); 1435 } else { 1436 ip6h = (struct ip6_hdr *)buf->iph; 1437 ah_info->hop_ttl = ip6h->ip6_hops; 1438 ah_info->tc_tos = ip6h->ip6_vfc; 1439 irdma_copy_ip_ntohl(ah_info->dest_ip_addr, 1440 ip6h->ip6_dst.__u6_addr.__u6_addr32); 1441 irdma_copy_ip_ntohl(ah_info->src_ip_addr, 1442 ip6h->ip6_src.__u6_addr.__u6_addr32); 1443 } 1444 1445 ah_info->dst_arpindex = irdma_arp_table(dev_to_rf(qp->dev), 1446 ah_info->dest_ip_addr, 1447 NULL, IRDMA_ARP_RESOLVE); 1448 } 1449 1450 /** 1451 * irdma_gen1_ieq_update_tcpip_info - update tcpip in the buffer 1452 * @buf: puda to update 1453 * @len: length of buffer 1454 * @seqnum: seq number for tcp 1455 */ 1456 static void 1457 irdma_gen1_ieq_update_tcpip_info(struct irdma_puda_buf *buf, 1458 u16 len, u32 seqnum) 1459 { 1460 struct tcphdr *tcph; 1461 struct ip *iph; 1462 u16 iphlen; 1463 u16 pktsize; 1464 u8 *addr = buf->mem.va; 1465 1466 iphlen = (buf->ipv4) ? 20 : 40; 1467 iph = (struct ip *)(addr + buf->maclen); 1468 tcph = (struct tcphdr *)(addr + buf->maclen + iphlen); 1469 pktsize = len + buf->tcphlen + iphlen; 1470 iph->ip_len = htons(pktsize); 1471 tcph->th_seq = htonl(seqnum); 1472 } 1473 1474 /** 1475 * irdma_ieq_update_tcpip_info - update tcpip in the buffer 1476 * @buf: puda to update 1477 * @len: length of buffer 1478 * @seqnum: seq number for tcp 1479 */ 1480 void 1481 irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len, 1482 u32 seqnum) 1483 { 1484 struct tcphdr *tcph; 1485 u8 *addr; 1486 1487 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 1488 return irdma_gen1_ieq_update_tcpip_info(buf, len, seqnum); 1489 1490 addr = buf->mem.va; 1491 tcph = (struct tcphdr *)addr; 1492 tcph->th_seq = htonl(seqnum); 1493 } 1494 1495 /** 1496 * irdma_gen1_puda_get_tcpip_info - get tcpip info from puda 1497 * buffer 1498 * @info: to get information 1499 * @buf: puda buffer 1500 */ 1501 static int 1502 irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info, 1503 struct irdma_puda_buf *buf) 1504 { 1505 struct ip *iph; 1506 struct ip6_hdr *ip6h; 1507 struct tcphdr *tcph; 1508 u16 iphlen; 1509 u16 pkt_len; 1510 u8 *mem = buf->mem.va; 1511 struct ether_header *ethh = buf->mem.va; 1512 1513 if (ethh->ether_type == htons(0x8100)) { 1514 info->vlan_valid = true; 1515 buf->vlan_id = ntohs(((struct ether_vlan_header *)ethh)->evl_tag) & 1516 EVL_VLID_MASK; 1517 } 1518 1519 buf->maclen = (info->vlan_valid) ? 18 : 14; 1520 iphlen = (info->l3proto) ? 40 : 20; 1521 buf->ipv4 = (info->l3proto) ? false : true; 1522 buf->iph = mem + buf->maclen; 1523 iph = (struct ip *)buf->iph; 1524 buf->tcph = buf->iph + iphlen; 1525 tcph = (struct tcphdr *)buf->tcph; 1526 1527 if (buf->ipv4) { 1528 pkt_len = ntohs(iph->ip_len); 1529 } else { 1530 ip6h = (struct ip6_hdr *)buf->iph; 1531 pkt_len = ntohs(ip6h->ip6_plen) + iphlen; 1532 } 1533 1534 buf->totallen = pkt_len + buf->maclen; 1535 1536 if (info->payload_len < buf->totallen) { 1537 irdma_debug(buf->vsi->dev, IRDMA_DEBUG_ERR, 1538 "payload_len = 0x%x totallen expected0x%x\n", 1539 info->payload_len, buf->totallen); 1540 return -EINVAL; 1541 } 1542 1543 buf->tcphlen = tcph->th_off << 2; 1544 buf->datalen = pkt_len - iphlen - buf->tcphlen; 1545 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL; 1546 buf->hdrlen = buf->maclen + iphlen + buf->tcphlen; 1547 buf->seqnum = ntohl(tcph->th_seq); 1548 1549 return 0; 1550 } 1551 1552 /** 1553 * irdma_puda_get_tcpip_info - get tcpip info from puda buffer 1554 * @info: to get information 1555 * @buf: puda buffer 1556 */ 1557 int 1558 irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info, 1559 struct irdma_puda_buf *buf) 1560 { 1561 struct tcphdr *tcph; 1562 u32 pkt_len; 1563 u8 *mem; 1564 1565 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 1566 return irdma_gen1_puda_get_tcpip_info(info, buf); 1567 1568 mem = buf->mem.va; 1569 buf->vlan_valid = info->vlan_valid; 1570 if (info->vlan_valid) 1571 buf->vlan_id = info->vlan; 1572 1573 buf->ipv4 = info->ipv4; 1574 if (buf->ipv4) 1575 buf->iph = mem + IRDMA_IPV4_PAD; 1576 else 1577 buf->iph = mem; 1578 1579 buf->tcph = mem + IRDMA_TCP_OFFSET; 1580 tcph = (struct tcphdr *)buf->tcph; 1581 pkt_len = info->payload_len; 1582 buf->totallen = pkt_len; 1583 buf->tcphlen = tcph->th_off << 2; 1584 buf->datalen = pkt_len - IRDMA_TCP_OFFSET - buf->tcphlen; 1585 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL; 1586 buf->hdrlen = IRDMA_TCP_OFFSET + buf->tcphlen; 1587 buf->seqnum = ntohl(tcph->th_seq); 1588 1589 if (info->smac_valid) { 1590 ether_addr_copy(buf->smac, info->smac); 1591 buf->smac_valid = true; 1592 } 1593 1594 return 0; 1595 } 1596 1597 /** 1598 * irdma_hw_stats_timeout - Stats timer-handler which updates all HW stats 1599 * @t: timer_list pointer 1600 */ 1601 static void 1602 irdma_hw_stats_timeout(struct timer_list *t) 1603 { 1604 struct irdma_vsi_pestat *pf_devstat = 1605 timer_container_of(pf_devstat, t, stats_timer); 1606 struct irdma_sc_vsi *sc_vsi = pf_devstat->vsi; 1607 1608 if (sc_vsi->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 1609 irdma_cqp_gather_stats_cmd(sc_vsi->dev, sc_vsi->pestat, false); 1610 1611 mod_timer(&pf_devstat->stats_timer, 1612 jiffies + msecs_to_jiffies(STATS_TIMER_DELAY)); 1613 } 1614 1615 /** 1616 * irdma_hw_stats_start_timer - Start periodic stats timer 1617 * @vsi: vsi structure pointer 1618 */ 1619 void 1620 irdma_hw_stats_start_timer(struct irdma_sc_vsi *vsi) 1621 { 1622 struct irdma_vsi_pestat *devstat = vsi->pestat; 1623 1624 timer_setup(&devstat->stats_timer, irdma_hw_stats_timeout, 0); 1625 mod_timer(&devstat->stats_timer, 1626 jiffies + msecs_to_jiffies(STATS_TIMER_DELAY)); 1627 } 1628 1629 /** 1630 * irdma_hw_stats_stop_timer - Delete periodic stats timer 1631 * @vsi: pointer to vsi structure 1632 */ 1633 void 1634 irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi) 1635 { 1636 struct irdma_vsi_pestat *devstat = vsi->pestat; 1637 1638 del_timer_sync(&devstat->stats_timer); 1639 } 1640 1641 /** 1642 * irdma_process_cqp_stats - Checking for wrap and update stats 1643 * @cqp_request: cqp_request structure pointer 1644 */ 1645 static void 1646 irdma_process_cqp_stats(struct irdma_cqp_request *cqp_request) 1647 { 1648 struct irdma_vsi_pestat *pestat = cqp_request->param; 1649 1650 sc_vsi_update_stats(pestat->vsi); 1651 } 1652 1653 /** 1654 * irdma_cqp_gather_stats_cmd - Gather stats 1655 * @dev: pointer to device structure 1656 * @pestat: pointer to stats info 1657 * @wait: flag to wait or not wait for stats 1658 */ 1659 int 1660 irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev, 1661 struct irdma_vsi_pestat *pestat, bool wait) 1662 { 1663 1664 struct irdma_pci_f *rf = dev_to_rf(dev); 1665 struct irdma_cqp *iwcqp = &rf->cqp; 1666 struct irdma_cqp_request *cqp_request; 1667 struct cqp_cmds_info *cqp_info; 1668 int status; 1669 1670 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait); 1671 if (!cqp_request) 1672 return -ENOMEM; 1673 1674 cqp_info = &cqp_request->info; 1675 cqp_info->cqp_cmd = IRDMA_OP_STATS_GATHER; 1676 cqp_info->post_sq = 1; 1677 cqp_info->in.u.stats_gather.info = pestat->gather_info; 1678 cqp_info->in.u.stats_gather.scratch = (uintptr_t)cqp_request; 1679 cqp_info->in.u.stats_gather.cqp = &rf->cqp.sc_cqp; 1680 cqp_request->param = pestat; 1681 if (!wait) 1682 cqp_request->callback_fcn = irdma_process_cqp_stats; 1683 status = irdma_handle_cqp_op(rf, cqp_request); 1684 if (wait) 1685 sc_vsi_update_stats(pestat->vsi); 1686 irdma_put_cqp_request(&rf->cqp, cqp_request); 1687 1688 return status; 1689 } 1690 1691 /** 1692 * irdma_cqp_ceq_cmd - Create/Destroy CEQ's after CEQ 0 1693 * @dev: pointer to device info 1694 * @sc_ceq: pointer to ceq structure 1695 * @op: Create or Destroy 1696 */ 1697 int 1698 irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_ceq *sc_ceq, 1699 u8 op) 1700 { 1701 struct irdma_cqp_request *cqp_request; 1702 struct cqp_cmds_info *cqp_info; 1703 struct irdma_pci_f *rf = dev_to_rf(dev); 1704 int status; 1705 1706 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1707 if (!cqp_request) 1708 return -ENOMEM; 1709 1710 cqp_info = &cqp_request->info; 1711 cqp_info->post_sq = 1; 1712 cqp_info->cqp_cmd = op; 1713 cqp_info->in.u.ceq_create.ceq = sc_ceq; 1714 cqp_info->in.u.ceq_create.scratch = (uintptr_t)cqp_request; 1715 1716 status = irdma_handle_cqp_op(rf, cqp_request); 1717 irdma_put_cqp_request(&rf->cqp, cqp_request); 1718 1719 return status; 1720 } 1721 1722 /** 1723 * irdma_cqp_aeq_cmd - Create/Destroy AEQ 1724 * @dev: pointer to device info 1725 * @sc_aeq: pointer to aeq structure 1726 * @op: Create or Destroy 1727 */ 1728 int 1729 irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_aeq *sc_aeq, 1730 u8 op) 1731 { 1732 struct irdma_cqp_request *cqp_request; 1733 struct cqp_cmds_info *cqp_info; 1734 struct irdma_pci_f *rf = dev_to_rf(dev); 1735 int status; 1736 1737 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1738 if (!cqp_request) 1739 return -ENOMEM; 1740 1741 cqp_info = &cqp_request->info; 1742 cqp_info->post_sq = 1; 1743 cqp_info->cqp_cmd = op; 1744 cqp_info->in.u.aeq_create.aeq = sc_aeq; 1745 cqp_info->in.u.aeq_create.scratch = (uintptr_t)cqp_request; 1746 1747 status = irdma_handle_cqp_op(rf, cqp_request); 1748 irdma_put_cqp_request(&rf->cqp, cqp_request); 1749 1750 return status; 1751 } 1752 1753 /** 1754 * irdma_cqp_ws_node_cmd - Add/modify/delete ws node 1755 * @dev: pointer to device structure 1756 * @cmd: Add, modify or delete 1757 * @node_info: pointer to ws node info 1758 */ 1759 int 1760 irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd, 1761 struct irdma_ws_node_info *node_info) 1762 { 1763 struct irdma_pci_f *rf = dev_to_rf(dev); 1764 struct irdma_cqp *iwcqp = &rf->cqp; 1765 struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp; 1766 struct irdma_cqp_request *cqp_request; 1767 struct cqp_cmds_info *cqp_info; 1768 int status; 1769 bool poll; 1770 1771 if (!rf->sc_dev.ceq_valid) 1772 poll = true; 1773 else 1774 poll = false; 1775 1776 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, !poll); 1777 if (!cqp_request) 1778 return -ENOMEM; 1779 1780 cqp_info = &cqp_request->info; 1781 cqp_info->cqp_cmd = cmd; 1782 cqp_info->post_sq = 1; 1783 cqp_info->in.u.ws_node.info = *node_info; 1784 cqp_info->in.u.ws_node.cqp = cqp; 1785 cqp_info->in.u.ws_node.scratch = (uintptr_t)cqp_request; 1786 cqp_info->create = true; 1787 status = irdma_handle_cqp_op(rf, cqp_request); 1788 if (status) 1789 goto exit; 1790 1791 if (poll) { 1792 struct irdma_ccq_cqe_info compl_info; 1793 1794 status = irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_WORK_SCHED_NODE, 1795 &compl_info); 1796 node_info->qs_handle = compl_info.op_ret_val; 1797 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, 1798 "opcode=%d, compl_info.retval=%d\n", 1799 compl_info.op_code, compl_info.op_ret_val); 1800 } else { 1801 node_info->qs_handle = cqp_request->compl_info.op_ret_val; 1802 } 1803 1804 exit: 1805 irdma_put_cqp_request(&rf->cqp, cqp_request); 1806 1807 return status; 1808 } 1809 1810 /** 1811 * irdma_ah_do_cqp - perform an AH cqp operation 1812 * @rf: RDMA PCI function 1813 * @sc_ah: address handle 1814 * @cmd: AH operation 1815 * @wait: wait if true 1816 * @callback_fcn: Callback function on CQP op completion 1817 * @cb_param: parameter for callback function 1818 * 1819 * returns errno 1820 */ 1821 static int 1822 irdma_ah_do_cqp(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd, 1823 bool wait, 1824 void (*callback_fcn) (struct irdma_cqp_request *), 1825 void *cb_param) 1826 { 1827 struct irdma_cqp_request *cqp_request; 1828 struct cqp_cmds_info *cqp_info; 1829 int status; 1830 1831 if (cmd != IRDMA_OP_AH_CREATE && cmd != IRDMA_OP_AH_DESTROY) 1832 return -EINVAL; 1833 1834 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait); 1835 if (!cqp_request) 1836 return -ENOMEM; 1837 1838 cqp_info = &cqp_request->info; 1839 cqp_info->cqp_cmd = cmd; 1840 cqp_info->post_sq = 1; 1841 if (cmd == IRDMA_OP_AH_CREATE) { 1842 if (!wait) 1843 irdma_get_cqp_request(cqp_request); 1844 sc_ah->ah_info.cqp_request = cqp_request; 1845 1846 cqp_info->in.u.ah_create.info = sc_ah->ah_info; 1847 cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request; 1848 cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp; 1849 cqp_info->create = true; 1850 } else if (cmd == IRDMA_OP_AH_DESTROY) { 1851 cqp_info->in.u.ah_destroy.info = sc_ah->ah_info; 1852 cqp_info->in.u.ah_destroy.scratch = (uintptr_t)cqp_request; 1853 cqp_info->in.u.ah_destroy.cqp = &rf->cqp.sc_cqp; 1854 } 1855 1856 if (!wait) { 1857 cqp_request->callback_fcn = callback_fcn; 1858 cqp_request->param = cb_param; 1859 } 1860 status = irdma_handle_cqp_op(rf, cqp_request); 1861 irdma_put_cqp_request(&rf->cqp, cqp_request); 1862 1863 if (status) 1864 return -ENOMEM; 1865 1866 if (wait) 1867 sc_ah->ah_info.ah_valid = (cmd != IRDMA_OP_AH_DESTROY); 1868 1869 return 0; 1870 } 1871 1872 int 1873 irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd, 1874 bool wait, 1875 void (*callback_fcn) (struct irdma_cqp_request *), 1876 void *cb_param) 1877 { 1878 int status; 1879 1880 if (cmd == IRDMA_OP_AH_CREATE) { 1881 status = irdma_get_arp(rf, sc_ah->ah_info.dst_arpindex); 1882 if (status) { 1883 irdma_dev_err(&rf->iwdev->ibdev, "%s get_arp failed for index = %d\n", 1884 __func__, sc_ah->ah_info.dst_arpindex); 1885 1886 return -EINVAL; 1887 } 1888 status = irdma_ah_do_cqp(rf, sc_ah, cmd, wait, callback_fcn, 1889 cb_param); 1890 if (status) 1891 irdma_put_arp(rf, sc_ah->ah_info.dst_arpindex); 1892 } else { 1893 status = irdma_ah_do_cqp(rf, sc_ah, cmd, wait, callback_fcn, 1894 cb_param); 1895 if (cmd == IRDMA_OP_AH_DESTROY) 1896 irdma_put_arp(rf, sc_ah->ah_info.dst_arpindex); 1897 } 1898 1899 return status; 1900 } 1901 1902 /** 1903 * irdma_ieq_ah_cb - callback after creation of AH for IEQ 1904 * @cqp_request: pointer to cqp_request of create AH 1905 */ 1906 static void 1907 irdma_ieq_ah_cb(struct irdma_cqp_request *cqp_request) 1908 { 1909 struct irdma_sc_qp *qp = cqp_request->param; 1910 struct irdma_sc_ah *sc_ah = qp->pfpdu.ah; 1911 unsigned long flags; 1912 1913 spin_lock_irqsave(&qp->pfpdu.lock, flags); 1914 if (!cqp_request->compl_info.op_ret_val) { 1915 sc_ah->ah_info.ah_valid = true; 1916 irdma_ieq_process_fpdus(qp, qp->vsi->ieq); 1917 } else { 1918 sc_ah->ah_info.ah_valid = false; 1919 irdma_ieq_cleanup_qp(qp->vsi->ieq, qp); 1920 } 1921 spin_unlock_irqrestore(&qp->pfpdu.lock, flags); 1922 if (!cqp_request->waiting) 1923 irdma_put_cqp_request(sc_ah->dev->cqp->back_cqp, 1924 cqp_request); 1925 } 1926 1927 /** 1928 * irdma_ilq_ah_cb - callback after creation of AH for ILQ 1929 * @cqp_request: pointer to cqp_request of create AH 1930 */ 1931 static void 1932 irdma_ilq_ah_cb(struct irdma_cqp_request *cqp_request) 1933 { 1934 struct irdma_cm_node *cm_node = cqp_request->param; 1935 struct irdma_sc_ah *sc_ah = cm_node->ah; 1936 1937 sc_ah->ah_info.ah_valid = !cqp_request->compl_info.op_ret_val; 1938 irdma_add_conn_est_qh(cm_node); 1939 if (!cqp_request->waiting) 1940 irdma_put_cqp_request(sc_ah->dev->cqp->back_cqp, 1941 cqp_request); 1942 } 1943 1944 /** 1945 * irdma_puda_create_ah - create AH for ILQ/IEQ qp's 1946 * @dev: device pointer 1947 * @ah_info: Address handle info 1948 * @wait: When true will wait for operation to complete 1949 * @type: ILQ/IEQ 1950 * @cb_param: Callback param when not waiting 1951 * @ah_ret: Returned pointer to address handle if created 1952 * 1953 */ 1954 int 1955 irdma_puda_create_ah(struct irdma_sc_dev *dev, 1956 struct irdma_ah_info *ah_info, bool wait, 1957 enum puda_rsrc_type type, void *cb_param, 1958 struct irdma_sc_ah **ah_ret) 1959 { 1960 struct irdma_sc_ah *ah; 1961 struct irdma_pci_f *rf = dev_to_rf(dev); 1962 int err; 1963 1964 ah = kzalloc(sizeof(*ah), GFP_ATOMIC); 1965 *ah_ret = ah; 1966 if (!ah) 1967 return -ENOMEM; 1968 1969 err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah, 1970 &ah_info->ah_idx, &rf->next_ah); 1971 if (err) 1972 goto err_free; 1973 1974 ah->dev = dev; 1975 ah->ah_info = *ah_info; 1976 1977 if (type == IRDMA_PUDA_RSRC_TYPE_ILQ) 1978 err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait, 1979 irdma_ilq_ah_cb, cb_param); 1980 else 1981 err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait, 1982 irdma_ieq_ah_cb, cb_param); 1983 1984 if (err) 1985 goto error; 1986 return 0; 1987 1988 error: 1989 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx); 1990 err_free: 1991 kfree(ah); 1992 *ah_ret = NULL; 1993 return -ENOMEM; 1994 } 1995 1996 /** 1997 * irdma_puda_free_ah - free a puda address handle 1998 * @dev: device pointer 1999 * @ah: The address handle to free 2000 */ 2001 void 2002 irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah) 2003 { 2004 struct irdma_pci_f *rf = dev_to_rf(dev); 2005 2006 if (!ah) 2007 return; 2008 2009 if (ah->ah_info.ah_valid) { 2010 irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_DESTROY, false, NULL, NULL); 2011 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx); 2012 } 2013 2014 kfree(ah); 2015 } 2016 2017 /** 2018 * irdma_prm_add_pble_mem - add moemory to pble resources 2019 * @pprm: pble resource manager 2020 * @pchunk: chunk of memory to add 2021 */ 2022 int 2023 irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm, 2024 struct irdma_chunk *pchunk) 2025 { 2026 u64 sizeofbitmap; 2027 2028 if (pchunk->size & 0xfff) 2029 return -EINVAL; 2030 2031 sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift; 2032 2033 pchunk->bitmapbuf = bitmap_zalloc(sizeofbitmap, GFP_KERNEL); 2034 if (!pchunk->bitmapbuf) 2035 return -ENOMEM; 2036 2037 pchunk->sizeofbitmap = sizeofbitmap; 2038 /* each pble is 8 bytes hence shift by 3 */ 2039 pprm->total_pble_alloc += pchunk->size >> 3; 2040 pprm->free_pble_cnt += pchunk->size >> 3; 2041 2042 return 0; 2043 } 2044 2045 /** 2046 * irdma_prm_get_pbles - get pble's from prm 2047 * @pprm: pble resource manager 2048 * @chunkinfo: nformation about chunk where pble's were acquired 2049 * @mem_size: size of pble memory needed 2050 * @vaddr: returns virtual address of pble memory 2051 * @fpm_addr: returns fpm address of pble memory 2052 */ 2053 int 2054 irdma_prm_get_pbles(struct irdma_pble_prm *pprm, 2055 struct irdma_pble_chunkinfo *chunkinfo, u64 mem_size, 2056 u64 **vaddr, u64 *fpm_addr) 2057 { 2058 u64 bits_needed; 2059 u64 bit_idx = PBLE_INVALID_IDX; 2060 struct irdma_chunk *pchunk = NULL; 2061 struct list_head *chunk_entry = (&pprm->clist)->next; 2062 u32 offset; 2063 unsigned long flags; 2064 2065 *vaddr = NULL; 2066 *fpm_addr = 0; 2067 2068 bits_needed = DIV_ROUND_UP_ULL(mem_size, BIT_ULL(pprm->pble_shift)); 2069 2070 spin_lock_irqsave(&pprm->prm_lock, flags); 2071 while (chunk_entry != &pprm->clist) { 2072 pchunk = (struct irdma_chunk *)chunk_entry; 2073 bit_idx = bitmap_find_next_zero_area(pchunk->bitmapbuf, 2074 pchunk->sizeofbitmap, 0, 2075 bits_needed, 0); 2076 if (bit_idx < pchunk->sizeofbitmap) 2077 break; 2078 2079 /* list.next used macro */ 2080 chunk_entry = (&pchunk->list)->next; 2081 } 2082 2083 if (!pchunk || bit_idx >= pchunk->sizeofbitmap) { 2084 spin_unlock_irqrestore(&pprm->prm_lock, flags); 2085 return -ENOMEM; 2086 } 2087 2088 bitmap_set(pchunk->bitmapbuf, bit_idx, bits_needed); 2089 offset = bit_idx << pprm->pble_shift; 2090 *vaddr = (u64 *)((u8 *)pchunk->vaddr + offset); 2091 *fpm_addr = pchunk->fpm_addr + offset; 2092 2093 chunkinfo->pchunk = pchunk; 2094 chunkinfo->bit_idx = bit_idx; 2095 chunkinfo->bits_used = bits_needed; 2096 /* 3 is sizeof pble divide */ 2097 pprm->free_pble_cnt -= chunkinfo->bits_used << (pprm->pble_shift - 3); 2098 spin_unlock_irqrestore(&pprm->prm_lock, flags); 2099 2100 return 0; 2101 } 2102 2103 /** 2104 * irdma_prm_return_pbles - return pbles back to prm 2105 * @pprm: pble resource manager 2106 * @chunkinfo: chunk where pble's were acquired and to be freed 2107 */ 2108 void 2109 irdma_prm_return_pbles(struct irdma_pble_prm *pprm, 2110 struct irdma_pble_chunkinfo *chunkinfo) 2111 { 2112 unsigned long flags; 2113 2114 spin_lock_irqsave(&pprm->prm_lock, flags); 2115 pprm->free_pble_cnt += chunkinfo->bits_used << (pprm->pble_shift - 3); 2116 bitmap_clear(chunkinfo->pchunk->bitmapbuf, chunkinfo->bit_idx, 2117 chunkinfo->bits_used); 2118 spin_unlock_irqrestore(&pprm->prm_lock, flags); 2119 } 2120 2121 int 2122 irdma_map_vm_page_list(struct irdma_hw *hw, void *va, dma_addr_t * pg_dma, 2123 u32 pg_cnt) 2124 { 2125 struct page *vm_page; 2126 int i; 2127 u8 *addr; 2128 2129 addr = (u8 *)(uintptr_t)va; 2130 for (i = 0; i < pg_cnt; i++) { 2131 vm_page = vmalloc_to_page(addr); 2132 if (!vm_page) 2133 goto err; 2134 2135 pg_dma[i] = dma_map_page(hw_to_dev(hw), vm_page, 0, PAGE_SIZE, DMA_BIDIRECTIONAL); 2136 if (dma_mapping_error(hw_to_dev(hw), pg_dma[i])) 2137 goto err; 2138 2139 addr += PAGE_SIZE; 2140 } 2141 2142 return 0; 2143 2144 err: 2145 irdma_unmap_vm_page_list(hw, pg_dma, i); 2146 return -ENOMEM; 2147 } 2148 2149 void 2150 irdma_unmap_vm_page_list(struct irdma_hw *hw, dma_addr_t * pg_dma, u32 pg_cnt) 2151 { 2152 int i; 2153 2154 for (i = 0; i < pg_cnt; i++) 2155 dma_unmap_page(hw_to_dev(hw), pg_dma[i], PAGE_SIZE, DMA_BIDIRECTIONAL); 2156 } 2157 2158 /** 2159 * irdma_pble_free_paged_mem - free virtual paged memory 2160 * @chunk: chunk to free with paged memory 2161 */ 2162 void 2163 irdma_pble_free_paged_mem(struct irdma_chunk *chunk) 2164 { 2165 if (!chunk->pg_cnt) 2166 goto done; 2167 2168 irdma_unmap_vm_page_list(chunk->dev->hw, chunk->dmainfo.dmaaddrs, 2169 chunk->pg_cnt); 2170 2171 done: 2172 kfree(chunk->dmainfo.dmaaddrs); 2173 chunk->dmainfo.dmaaddrs = NULL; 2174 vfree(chunk->vaddr); 2175 chunk->vaddr = NULL; 2176 chunk->type = 0; 2177 } 2178 2179 /** 2180 * irdma_pble_get_paged_mem -allocate paged memory for pbles 2181 * @chunk: chunk to add for paged memory 2182 * @pg_cnt: number of pages needed 2183 */ 2184 int 2185 irdma_pble_get_paged_mem(struct irdma_chunk *chunk, u32 pg_cnt) 2186 { 2187 u32 size; 2188 void *va; 2189 2190 chunk->dmainfo.dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL); 2191 if (!chunk->dmainfo.dmaaddrs) 2192 return -ENOMEM; 2193 2194 size = PAGE_SIZE * pg_cnt; 2195 va = vmalloc(size); 2196 if (!va) 2197 goto err; 2198 2199 if (irdma_map_vm_page_list(chunk->dev->hw, va, chunk->dmainfo.dmaaddrs, 2200 pg_cnt)) { 2201 vfree(va); 2202 goto err; 2203 } 2204 chunk->vaddr = va; 2205 chunk->size = size; 2206 chunk->pg_cnt = pg_cnt; 2207 chunk->type = PBLE_SD_PAGED; 2208 2209 return 0; 2210 err: 2211 kfree(chunk->dmainfo.dmaaddrs); 2212 chunk->dmainfo.dmaaddrs = NULL; 2213 2214 return -ENOMEM; 2215 } 2216 2217 /** 2218 * irdma_alloc_ws_node_id - Allocate a tx scheduler node ID 2219 * @dev: device pointer 2220 */ 2221 u16 2222 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev) 2223 { 2224 struct irdma_pci_f *rf = dev_to_rf(dev); 2225 u32 next = 1; 2226 u32 node_id; 2227 2228 if (irdma_alloc_rsrc(rf, rf->allocated_ws_nodes, rf->max_ws_node_id, 2229 &node_id, &next)) 2230 return IRDMA_WS_NODE_INVALID; 2231 2232 return (u16)node_id; 2233 } 2234 2235 /** 2236 * irdma_free_ws_node_id - Free a tx scheduler node ID 2237 * @dev: device pointer 2238 * @node_id: Work scheduler node ID 2239 */ 2240 void 2241 irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id) 2242 { 2243 struct irdma_pci_f *rf = dev_to_rf(dev); 2244 2245 irdma_free_rsrc(rf, rf->allocated_ws_nodes, (u32)node_id); 2246 } 2247 2248 /** 2249 * irdma_modify_qp_to_err - Modify a QP to error 2250 * @sc_qp: qp structure 2251 */ 2252 void 2253 irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp) 2254 { 2255 struct irdma_qp *qp = sc_qp->qp_uk.back_qp; 2256 struct ib_qp_attr attr; 2257 2258 if (qp->iwdev->rf->reset) 2259 return; 2260 attr.qp_state = IB_QPS_ERR; 2261 2262 if (rdma_protocol_roce(qp->ibqp.device, 1)) 2263 irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL); 2264 else 2265 irdma_modify_qp(&qp->ibqp, &attr, IB_QP_STATE, NULL); 2266 } 2267 2268 void 2269 irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event) 2270 { 2271 struct ib_event ibevent; 2272 2273 if (!iwqp->ibqp.event_handler) 2274 return; 2275 2276 switch (event) { 2277 case IRDMA_QP_EVENT_CATASTROPHIC: 2278 ibevent.event = IB_EVENT_QP_FATAL; 2279 break; 2280 case IRDMA_QP_EVENT_ACCESS_ERR: 2281 ibevent.event = IB_EVENT_QP_ACCESS_ERR; 2282 break; 2283 case IRDMA_QP_EVENT_REQ_ERR: 2284 ibevent.event = IB_EVENT_QP_REQ_ERR; 2285 break; 2286 } 2287 ibevent.device = iwqp->ibqp.device; 2288 ibevent.element.qp = &iwqp->ibqp; 2289 iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context); 2290 } 2291 2292 static void 2293 clear_qp_ctx_addr(__le64 * ctx) 2294 { 2295 u64 tmp; 2296 2297 get_64bit_val(ctx, 272, &tmp); 2298 tmp &= GENMASK_ULL(63, 58); 2299 set_64bit_val(ctx, 272, tmp); 2300 2301 get_64bit_val(ctx, 296, &tmp); 2302 tmp &= GENMASK_ULL(7, 0); 2303 set_64bit_val(ctx, 296, tmp); 2304 2305 get_64bit_val(ctx, 312, &tmp); 2306 tmp &= GENMASK_ULL(7, 0); 2307 set_64bit_val(ctx, 312, tmp); 2308 2309 set_64bit_val(ctx, 368, 0); 2310 } 2311 2312 /** 2313 * irdma_upload_qp_context - upload raw QP context 2314 * @rf: RDMA PCI function 2315 * @qpn: QP ID 2316 * @qp_type: QP Type 2317 * @freeze: freeze QP 2318 * @raw: raw context flag 2319 */ 2320 int 2321 irdma_upload_qp_context(struct irdma_pci_f *rf, u32 qpn, 2322 u8 qp_type, bool freeze, bool raw) 2323 { 2324 struct irdma_dma_mem dma_mem; 2325 struct irdma_sc_dev *dev; 2326 struct irdma_cqp *iwcqp; 2327 struct irdma_cqp_request *cqp_request; 2328 struct cqp_cmds_info *cqp_info; 2329 struct irdma_upload_context_info *info; 2330 int ret; 2331 u32 *ctx; 2332 2333 dev = &rf->sc_dev; 2334 iwcqp = &rf->cqp; 2335 2336 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true); 2337 if (!cqp_request) { 2338 irdma_debug((dev), IRDMA_DEBUG_QP, "Could not get CQP req for QP [%u]\n", qpn); 2339 return -EINVAL; 2340 } 2341 cqp_info = &cqp_request->info; 2342 info = &cqp_info->in.u.qp_upload_context.info; 2343 cqp_info->cqp_cmd = IRDMA_OP_QP_UPLOAD_CONTEXT; 2344 cqp_info->post_sq = 1; 2345 cqp_info->in.u.qp_upload_context.dev = dev; 2346 cqp_info->in.u.qp_upload_context.scratch = (uintptr_t)cqp_request; 2347 2348 dma_mem.size = PAGE_SIZE; 2349 dma_mem.va = irdma_allocate_dma_mem(dev->hw, &dma_mem, dma_mem.size, PAGE_SIZE); 2350 if (!dma_mem.va) { 2351 irdma_put_cqp_request(&rf->cqp, cqp_request); 2352 irdma_debug((dev), IRDMA_DEBUG_QP, "Could not allocate buffer for QP [%u]\n", qpn); 2353 return -ENOMEM; 2354 } 2355 2356 ctx = dma_mem.va; 2357 info->buf_pa = dma_mem.pa; 2358 info->raw_format = raw; 2359 info->freeze_qp = freeze; 2360 info->qp_type = qp_type; /* 1 is iWARP and 2 UDA */ 2361 info->qp_id = qpn; 2362 ret = irdma_handle_cqp_op(rf, cqp_request); 2363 if (ret) 2364 goto error; 2365 irdma_debug((dev), IRDMA_DEBUG_QP, "PRINT CONTXT QP [%u]\n", info->qp_id); 2366 { 2367 u32 i, j; 2368 2369 clear_qp_ctx_addr(dma_mem.va); 2370 for (i = 0, j = 0; i < 32; i++, j += 4) 2371 irdma_debug((dev), IRDMA_DEBUG_QP, 2372 "[%u] %u:\t [%08X %08x %08X %08X]\n", 2373 info->qp_id, (j * 4), ctx[j], ctx[j + 1], 2374 ctx[j + 2], ctx[j + 3]); 2375 } 2376 error: 2377 irdma_put_cqp_request(iwcqp, cqp_request); 2378 irdma_free_dma_mem(dev->hw, &dma_mem); 2379 2380 return ret; 2381 } 2382 2383 static bool 2384 qp_has_unpolled_cqes(struct irdma_qp *iwqp, struct irdma_cq *iwcq) 2385 { 2386 struct irdma_cq_uk *cq = &iwcq->sc_cq.cq_uk; 2387 struct irdma_qp_uk *qp = &iwqp->sc_qp.qp_uk; 2388 u32 cq_head = IRDMA_RING_CURRENT_HEAD(cq->cq_ring); 2389 u64 qword3, comp_ctx; 2390 __le64 *cqe; 2391 u8 polarity, cq_polarity; 2392 2393 cq_polarity = cq->polarity; 2394 do { 2395 if (cq->avoid_mem_cflct) 2396 cqe = ((struct irdma_extended_cqe *)(cq->cq_base))[cq_head].buf; 2397 else 2398 cqe = cq->cq_base[cq_head].buf; 2399 get_64bit_val(cqe, IRDMA_BYTE_24, &qword3); 2400 polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3); 2401 2402 if (polarity != cq_polarity) 2403 break; 2404 2405 /* Ensure CQE contents are read after valid bit is checked */ 2406 rmb(); 2407 2408 get_64bit_val(cqe, IRDMA_BYTE_8, &comp_ctx); 2409 if ((struct irdma_qp_uk *)(irdma_uintptr) comp_ctx == qp) 2410 return true; 2411 2412 cq_head = (cq_head + 1) % cq->cq_ring.size; 2413 if (!cq_head) 2414 cq_polarity ^= 1; 2415 } while (true); 2416 2417 return false; 2418 } 2419 2420 void 2421 irdma_remove_cmpls_list(struct irdma_cq *iwcq) 2422 { 2423 struct irdma_cmpl_gen *cmpl_node; 2424 struct list_head *tmp_node, *list_node; 2425 2426 list_for_each_safe(list_node, tmp_node, &iwcq->cmpl_generated) { 2427 cmpl_node = list_entry(list_node, struct irdma_cmpl_gen, list); 2428 list_del(&cmpl_node->list); 2429 kfree(cmpl_node); 2430 } 2431 } 2432 2433 int 2434 irdma_generated_cmpls(struct irdma_cq *iwcq, struct irdma_cq_poll_info *cq_poll_info) 2435 { 2436 struct irdma_cmpl_gen *cmpl; 2437 2438 if (list_empty(&iwcq->cmpl_generated)) 2439 return -ENOENT; 2440 cmpl = list_first_entry_or_null(&iwcq->cmpl_generated, struct irdma_cmpl_gen, list); 2441 list_del(&cmpl->list); 2442 memcpy(cq_poll_info, &cmpl->cpi, sizeof(*cq_poll_info)); 2443 kfree(cmpl); 2444 2445 irdma_debug(iwcq->sc_cq.dev, IRDMA_DEBUG_VERBS, 2446 "%s: Poll artificially generated completion for QP 0x%X, op %u, wr_id=0x%lx\n", 2447 __func__, cq_poll_info->qp_id, cq_poll_info->op_type, 2448 cq_poll_info->wr_id); 2449 2450 return 0; 2451 } 2452 2453 /** 2454 * irdma_set_cpi_common_values - fill in values for polling info struct 2455 * @cpi: resulting structure of cq_poll_info type 2456 * @qp: QPair 2457 * @qp_num: id of the QP 2458 */ 2459 static void 2460 irdma_set_cpi_common_values(struct irdma_cq_poll_info *cpi, 2461 struct irdma_qp_uk *qp, u32 qp_num) 2462 { 2463 cpi->comp_status = IRDMA_COMPL_STATUS_FLUSHED; 2464 cpi->error = 1; 2465 cpi->major_err = IRDMA_FLUSH_MAJOR_ERR; 2466 cpi->minor_err = FLUSH_GENERAL_ERR; 2467 cpi->qp_handle = (irdma_qp_handle) (uintptr_t)qp; 2468 cpi->qp_id = qp_num; 2469 } 2470 2471 static inline void 2472 irdma_comp_handler(struct irdma_cq *cq) 2473 { 2474 struct irdma_device *iwdev = to_iwdev(cq->ibcq.device); 2475 struct irdma_ceq *ceq = &iwdev->rf->ceqlist[cq->sc_cq.ceq_id]; 2476 unsigned long flags; 2477 2478 if (!cq->ibcq.comp_handler) 2479 return; 2480 2481 if (atomic_read(&cq->armed)) { 2482 spin_lock_irqsave(&ceq->ce_lock, flags); 2483 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); 2484 spin_unlock_irqrestore(&ceq->ce_lock, flags); 2485 } 2486 } 2487 2488 /** 2489 * irdma_generate_flush_completions - generate completion from WRs 2490 * @iwqp: pointer to QP 2491 */ 2492 void 2493 irdma_generate_flush_completions(struct irdma_qp *iwqp) 2494 { 2495 struct irdma_qp_uk *qp = &iwqp->sc_qp.qp_uk; 2496 struct irdma_ring *sq_ring = &qp->sq_ring; 2497 struct irdma_ring *rq_ring = &qp->rq_ring; 2498 struct irdma_cmpl_gen *cmpl; 2499 __le64 *sw_wqe; 2500 u64 wqe_qword; 2501 u32 wqe_idx; 2502 bool compl_generated = false; 2503 unsigned long flags1; 2504 2505 spin_lock_irqsave(&iwqp->iwscq->lock, flags1); 2506 if (!qp_has_unpolled_cqes(iwqp, iwqp->iwscq)) { 2507 unsigned long flags2; 2508 2509 spin_lock_irqsave(&iwqp->lock, flags2); 2510 while (IRDMA_RING_MORE_WORK(*sq_ring)) { 2511 cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC); 2512 if (!cmpl) { 2513 spin_unlock_irqrestore(&iwqp->lock, flags2); 2514 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); 2515 return; 2516 } 2517 2518 wqe_idx = sq_ring->tail; 2519 irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id); 2520 2521 cmpl->cpi.wr_id = qp->sq_wrtrk_array[wqe_idx].wrid; 2522 cmpl->cpi.signaled = qp->sq_wrtrk_array[wqe_idx].signaled; 2523 sw_wqe = qp->sq_base[wqe_idx].elem; 2524 get_64bit_val(sw_wqe, IRDMA_BYTE_24, &wqe_qword); 2525 cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, wqe_qword); 2526 cmpl->cpi.q_type = IRDMA_CQE_QTYPE_SQ; 2527 /* remove the SQ WR by moving SQ tail */ 2528 IRDMA_RING_SET_TAIL(*sq_ring, 2529 sq_ring->tail + qp->sq_wrtrk_array[sq_ring->tail].quanta); 2530 2531 if (cmpl->cpi.op_type == IRDMAQP_OP_NOP) { 2532 kfree(cmpl); 2533 continue; 2534 } 2535 irdma_debug(iwqp->sc_qp.dev, IRDMA_DEBUG_DEV, 2536 "%s: adding wr_id = 0x%lx SQ Completion to list qp_id=%d\n", 2537 __func__, cmpl->cpi.wr_id, qp->qp_id); 2538 list_add_tail(&cmpl->list, &iwqp->iwscq->cmpl_generated); 2539 compl_generated = true; 2540 } 2541 spin_unlock_irqrestore(&iwqp->lock, flags2); 2542 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); 2543 if (compl_generated) { 2544 irdma_comp_handler(iwqp->iwscq); 2545 compl_generated = false; 2546 } 2547 } else { 2548 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); 2549 irdma_sched_qp_flush_work(iwqp); 2550 } 2551 2552 spin_lock_irqsave(&iwqp->iwrcq->lock, flags1); 2553 if (!qp_has_unpolled_cqes(iwqp, iwqp->iwrcq)) { 2554 unsigned long flags2; 2555 2556 spin_lock_irqsave(&iwqp->lock, flags2); 2557 while (IRDMA_RING_MORE_WORK(*rq_ring)) { 2558 cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC); 2559 if (!cmpl) { 2560 spin_unlock_irqrestore(&iwqp->lock, flags2); 2561 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); 2562 return; 2563 } 2564 2565 wqe_idx = rq_ring->tail; 2566 irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id); 2567 2568 cmpl->cpi.wr_id = qp->rq_wrid_array[wqe_idx]; 2569 cmpl->cpi.signaled = 1; 2570 cmpl->cpi.op_type = IRDMA_OP_TYPE_REC; 2571 cmpl->cpi.q_type = IRDMA_CQE_QTYPE_RQ; 2572 /* remove the RQ WR by moving RQ tail */ 2573 IRDMA_RING_SET_TAIL(*rq_ring, rq_ring->tail + 1); 2574 irdma_debug(iwqp->sc_qp.dev, IRDMA_DEBUG_DEV, 2575 "%s: adding wr_id = 0x%lx RQ Completion to list qp_id=%d, wqe_idx=%d\n", 2576 __func__, cmpl->cpi.wr_id, qp->qp_id, 2577 wqe_idx); 2578 2579 list_add_tail(&cmpl->list, &iwqp->iwrcq->cmpl_generated); 2580 2581 compl_generated = true; 2582 } 2583 spin_unlock_irqrestore(&iwqp->lock, flags2); 2584 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); 2585 if (compl_generated) 2586 irdma_comp_handler(iwqp->iwrcq); 2587 } else { 2588 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); 2589 irdma_sched_qp_flush_work(iwqp); 2590 } 2591 } 2592 2593 /** 2594 * irdma_udqp_qs_change - change qs for UD QP in a worker thread 2595 * @iwqp: QP pointer 2596 * @user_prio: new user priority value 2597 * @qs_change: when false, only user priority changes, QS handle do not need to change 2598 */ 2599 static void 2600 irdma_udqp_qs_change(struct irdma_qp *iwqp, u8 user_prio, bool qs_change) 2601 { 2602 irdma_qp_rem_qos(&iwqp->sc_qp); 2603 if (qs_change) 2604 iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi, iwqp->ctx_info.user_pri); 2605 2606 iwqp->ctx_info.user_pri = user_prio; 2607 iwqp->sc_qp.user_pri = user_prio; 2608 2609 if (qs_change) 2610 if (iwqp->sc_qp.dev->ws_add(iwqp->sc_qp.vsi, user_prio)) 2611 irdma_dev_warn(&iwqp->iwdev->ibdev, 2612 "WS add failed during %s, qp_id: %x user_pri: %x", 2613 __func__, iwqp->ibqp.qp_num, user_prio); 2614 irdma_qp_add_qos(&iwqp->sc_qp); 2615 } 2616 2617 void 2618 irdma_udqp_qs_worker(struct work_struct *work) 2619 { 2620 struct irdma_udqs_work *udqs_work = container_of(work, struct irdma_udqs_work, work); 2621 2622 irdma_udqp_qs_change(udqs_work->iwqp, udqs_work->user_prio, udqs_work->qs_change); 2623 if (udqs_work->qs_change) 2624 irdma_cqp_qp_suspend_resume(&udqs_work->iwqp->sc_qp, IRDMA_OP_RESUME); 2625 irdma_qp_rem_ref(&udqs_work->iwqp->ibqp); 2626 kfree(udqs_work); 2627 } 2628 2629 void 2630 irdma_chk_free_stag(struct irdma_pci_f *rf) 2631 { 2632 struct irdma_cqp_request *cqp_request; 2633 struct cqp_cmds_info *cqp_info; 2634 struct irdma_dealloc_stag_info *info; 2635 2636 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 2637 if (!cqp_request) 2638 return; 2639 2640 cqp_info = &cqp_request->info; 2641 info = &cqp_info->in.u.dealloc_stag.info; 2642 info->stag_idx = RS_64_1(rf->chk_stag, IRDMA_CQPSQ_STAG_IDX_S); 2643 cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG; 2644 cqp_info->post_sq = 1; 2645 cqp_info->in.u.dealloc_stag.dev = &rf->sc_dev; 2646 cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request; 2647 irdma_handle_cqp_op(rf, cqp_request); 2648 irdma_put_cqp_request(&rf->cqp, cqp_request); 2649 } 2650 2651 void 2652 cqp_poll_worker(struct work_struct *work) 2653 { 2654 struct delayed_work *dwork = to_delayed_work(work); 2655 struct irdma_pci_f *rf = container_of(dwork, struct irdma_pci_f, dwork_cqp_poll); 2656 struct irdma_mr iwmr = {}; 2657 struct irdma_pd *iwpd; 2658 2659 iwpd = kzalloc(sizeof(*iwpd), GFP_KERNEL); 2660 if (!iwpd) 2661 return; 2662 iwmr.stag = rf->chk_stag; 2663 iwmr.ibmw.type = IB_MW_TYPE_1; 2664 iwmr.ibmr.pd = &iwpd->ibpd; 2665 if (irdma_hw_alloc_mw(rf->iwdev, &iwmr)) 2666 goto exit; 2667 irdma_chk_free_stag(rf); 2668 2669 mod_delayed_work(rf->iwdev->cleanup_wq, &rf->dwork_cqp_poll, 2670 msecs_to_jiffies(3000)); 2671 exit: 2672 kfree(iwpd); 2673 } 2674