1 /*- 2 * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 3 * 4 * Copyright (c) 2015 - 2023 Intel Corporation 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenFabrics.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #include "osdep.h" 36 #include "irdma_hmc.h" 37 #include "irdma_defs.h" 38 #include "irdma_type.h" 39 #include "irdma_ws.h" 40 #include "irdma_protos.h" 41 42 /** 43 * irdma_qp_from_entry - Given entry, get to the qp structure 44 * @entry: Points to list of qp structure 45 */ 46 static struct irdma_sc_qp * 47 irdma_qp_from_entry(struct list_head *entry) 48 { 49 if (!entry) 50 return NULL; 51 52 return (struct irdma_sc_qp *)((char *)entry - 53 offsetof(struct irdma_sc_qp, list)); 54 } 55 56 /** 57 * irdma_get_qp_from_list - get next qp from a list 58 * @head: Listhead of qp's 59 * @qp: current qp 60 */ 61 struct irdma_sc_qp * 62 irdma_get_qp_from_list(struct list_head *head, 63 struct irdma_sc_qp *qp) 64 { 65 struct list_head *lastentry; 66 struct list_head *entry = NULL; 67 68 if (list_empty(head)) 69 return NULL; 70 71 if (!qp) { 72 entry = (head)->next; 73 } else { 74 lastentry = &qp->list; 75 entry = (lastentry)->next; 76 if (entry == head) 77 return NULL; 78 } 79 80 return irdma_qp_from_entry(entry); 81 } 82 83 /** 84 * irdma_sc_suspend_resume_qps - suspend/resume all qp's on VSI 85 * @vsi: the VSI struct pointer 86 * @op: Set to IRDMA_OP_RESUME or IRDMA_OP_SUSPEND 87 */ 88 void 89 irdma_sc_suspend_resume_qps(struct irdma_sc_vsi *vsi, u8 op) 90 { 91 struct irdma_sc_qp *qp = NULL; 92 u8 i; 93 94 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) { 95 mutex_lock(&vsi->qos[i].qos_mutex); 96 qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp); 97 while (qp) { 98 if (op == IRDMA_OP_RESUME) { 99 if (!qp->dev->ws_add(vsi, i)) { 100 qp->qs_handle = 101 vsi->qos[qp->user_pri].qs_handle; 102 irdma_cqp_qp_suspend_resume(qp, op); 103 } else { 104 irdma_cqp_qp_suspend_resume(qp, op); 105 irdma_modify_qp_to_err(qp); 106 } 107 } else if (op == IRDMA_OP_SUSPEND) { 108 /* issue cqp suspend command */ 109 if (!irdma_cqp_qp_suspend_resume(qp, op)) 110 atomic_inc(&vsi->qp_suspend_reqs); 111 } 112 qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp); 113 } 114 mutex_unlock(&vsi->qos[i].qos_mutex); 115 } 116 } 117 118 static void 119 irdma_set_qos_info(struct irdma_sc_vsi *vsi, struct irdma_l2params *l2p) 120 { 121 u8 i; 122 123 vsi->qos_rel_bw = l2p->vsi_rel_bw; 124 vsi->qos_prio_type = l2p->vsi_prio_type; 125 vsi->dscp_mode = l2p->dscp_mode; 126 if (l2p->dscp_mode) { 127 irdma_memcpy(vsi->dscp_map, l2p->dscp_map, sizeof(vsi->dscp_map)); 128 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) 129 l2p->up2tc[i] = i; 130 } 131 for (i = 0; i < IRDMA_MAX_TRAFFIC_CLASS; i++) 132 vsi->tc_print_warning[i] = true; 133 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) { 134 if (vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 135 vsi->qos[i].qs_handle = l2p->qs_handle_list[i]; 136 if (vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_2) 137 irdma_init_config_check(&vsi->cfg_check[i], 138 l2p->up2tc[i], 139 l2p->qs_handle_list[i]); 140 vsi->qos[i].traffic_class = l2p->up2tc[i]; 141 vsi->qos[i].rel_bw = 142 l2p->tc_info[vsi->qos[i].traffic_class].rel_bw; 143 vsi->qos[i].prio_type = 144 l2p->tc_info[vsi->qos[i].traffic_class].prio_type; 145 vsi->qos[i].valid = false; 146 } 147 } 148 149 /** 150 * irdma_change_l2params - given the new l2 parameters, change all qp 151 * @vsi: RDMA VSI pointer 152 * @l2params: New parameters from l2 153 */ 154 void 155 irdma_change_l2params(struct irdma_sc_vsi *vsi, 156 struct irdma_l2params *l2params) 157 { 158 if (l2params->tc_changed) { 159 vsi->tc_change_pending = false; 160 irdma_set_qos_info(vsi, l2params); 161 irdma_sc_suspend_resume_qps(vsi, IRDMA_OP_RESUME); 162 } 163 if (l2params->mtu_changed) { 164 vsi->mtu = l2params->mtu; 165 if (vsi->ieq) 166 irdma_reinitialize_ieq(vsi); 167 } 168 } 169 170 /** 171 * irdma_qp_rem_qos - remove qp from qos lists during destroy qp 172 * @qp: qp to be removed from qos 173 */ 174 void 175 irdma_qp_rem_qos(struct irdma_sc_qp *qp) 176 { 177 struct irdma_sc_vsi *vsi = qp->vsi; 178 179 irdma_debug(qp->dev, IRDMA_DEBUG_DCB, 180 "DCB: Remove qp[%d] UP[%d] qset[%d] on_qoslist[%d]\n", 181 qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle, 182 qp->on_qoslist); 183 mutex_lock(&vsi->qos[qp->user_pri].qos_mutex); 184 if (qp->on_qoslist) { 185 qp->on_qoslist = false; 186 list_del(&qp->list); 187 } 188 mutex_unlock(&vsi->qos[qp->user_pri].qos_mutex); 189 } 190 191 /** 192 * irdma_qp_add_qos - called during setctx for qp to be added to qos 193 * @qp: qp to be added to qos 194 */ 195 void 196 irdma_qp_add_qos(struct irdma_sc_qp *qp) 197 { 198 struct irdma_sc_vsi *vsi = qp->vsi; 199 200 irdma_debug(qp->dev, IRDMA_DEBUG_DCB, 201 "DCB: Add qp[%d] UP[%d] qset[%d] on_qoslist[%d]\n", 202 qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle, 203 qp->on_qoslist); 204 mutex_lock(&vsi->qos[qp->user_pri].qos_mutex); 205 if (!qp->on_qoslist) { 206 list_add(&qp->list, &vsi->qos[qp->user_pri].qplist); 207 qp->on_qoslist = true; 208 qp->qs_handle = vsi->qos[qp->user_pri].qs_handle; 209 } 210 mutex_unlock(&vsi->qos[qp->user_pri].qos_mutex); 211 } 212 213 /** 214 * irdma_sc_pd_init - initialize sc pd struct 215 * @dev: sc device struct 216 * @pd: sc pd ptr 217 * @pd_id: pd_id for allocated pd 218 * @abi_ver: User/Kernel ABI version 219 */ 220 void 221 irdma_sc_pd_init(struct irdma_sc_dev *dev, struct irdma_sc_pd *pd, u32 pd_id, 222 int abi_ver) 223 { 224 pd->pd_id = pd_id; 225 pd->abi_ver = abi_ver; 226 pd->dev = dev; 227 } 228 229 /** 230 * irdma_sc_add_arp_cache_entry - cqp wqe add arp cache entry 231 * @cqp: struct for cqp hw 232 * @info: arp entry information 233 * @scratch: u64 saved to be used during cqp completion 234 * @post_sq: flag for cqp db to ring 235 */ 236 static int 237 irdma_sc_add_arp_cache_entry(struct irdma_sc_cqp *cqp, 238 struct irdma_add_arp_cache_entry_info *info, 239 u64 scratch, bool post_sq) 240 { 241 __le64 *wqe; 242 u64 hdr; 243 244 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 245 if (!wqe) 246 return -ENOSPC; 247 set_64bit_val(wqe, IRDMA_BYTE_8, info->reach_max); 248 249 set_64bit_val(wqe, IRDMA_BYTE_16, irdma_mac_to_u64(info->mac_addr)); 250 251 hdr = info->arp_index | 252 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_ARP) | 253 FIELD_PREP(IRDMA_CQPSQ_MAT_PERMANENT, info->permanent) | 254 FIELD_PREP(IRDMA_CQPSQ_MAT_ENTRYVALID, true) | 255 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 256 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 257 258 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 259 260 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "ARP_CACHE_ENTRY WQE", wqe, 261 IRDMA_CQP_WQE_SIZE * 8); 262 if (post_sq) 263 irdma_sc_cqp_post_sq(cqp); 264 265 return 0; 266 } 267 268 /** 269 * irdma_sc_del_arp_cache_entry - dele arp cache entry 270 * @cqp: struct for cqp hw 271 * @scratch: u64 saved to be used during cqp completion 272 * @arp_index: arp index to delete arp entry 273 * @post_sq: flag for cqp db to ring 274 */ 275 static int 276 irdma_sc_del_arp_cache_entry(struct irdma_sc_cqp *cqp, u64 scratch, 277 u16 arp_index, bool post_sq) 278 { 279 __le64 *wqe; 280 u64 hdr; 281 282 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 283 if (!wqe) 284 return -ENOSPC; 285 286 hdr = arp_index | 287 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_ARP) | 288 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 289 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 290 291 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 292 293 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "ARP_CACHE_DEL_ENTRY WQE", 294 wqe, IRDMA_CQP_WQE_SIZE * 8); 295 if (post_sq) 296 irdma_sc_cqp_post_sq(cqp); 297 298 return 0; 299 } 300 301 /** 302 * irdma_sc_manage_apbvt_entry - for adding and deleting apbvt entries 303 * @cqp: struct for cqp hw 304 * @info: info for apbvt entry to add or delete 305 * @scratch: u64 saved to be used during cqp completion 306 * @post_sq: flag for cqp db to ring 307 */ 308 static int 309 irdma_sc_manage_apbvt_entry(struct irdma_sc_cqp *cqp, 310 struct irdma_apbvt_info *info, 311 u64 scratch, bool post_sq) 312 { 313 __le64 *wqe; 314 u64 hdr; 315 316 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 317 if (!wqe) 318 return -ENOSPC; 319 320 set_64bit_val(wqe, IRDMA_BYTE_16, info->port); 321 322 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_APBVT) | 323 FIELD_PREP(IRDMA_CQPSQ_MAPT_ADDPORT, info->add) | 324 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 325 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 326 327 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 328 329 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "MANAGE_APBVT WQE", wqe, 330 IRDMA_CQP_WQE_SIZE * 8); 331 if (post_sq) 332 irdma_sc_cqp_post_sq(cqp); 333 334 return 0; 335 } 336 337 /** 338 * irdma_sc_manage_qhash_table_entry - manage quad hash entries 339 * @cqp: struct for cqp hw 340 * @info: info for quad hash to manage 341 * @scratch: u64 saved to be used during cqp completion 342 * @post_sq: flag for cqp db to ring 343 * 344 * This is called before connection establishment is started. 345 * For passive connections, when listener is created, it will 346 * call with entry type of IRDMA_QHASH_TYPE_TCP_SYN with local 347 * ip address and tcp port. When SYN is received (passive 348 * connections) or sent (active connections), this routine is 349 * called with entry type of IRDMA_QHASH_TYPE_TCP_ESTABLISHED 350 * and quad is passed in info. 351 * 352 * When iwarp connection is done and its state moves to RTS, the 353 * quad hash entry in the hardware will point to iwarp's qp 354 * number and requires no calls from the driver. 355 */ 356 static int 357 irdma_sc_manage_qhash_table_entry(struct irdma_sc_cqp *cqp, 358 struct irdma_qhash_table_info *info, 359 u64 scratch, bool post_sq) 360 { 361 __le64 *wqe; 362 u64 qw1 = 0; 363 u64 qw2 = 0; 364 u64 temp; 365 struct irdma_sc_vsi *vsi = info->vsi; 366 367 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 368 if (!wqe) 369 return -ENOSPC; 370 set_64bit_val(wqe, IRDMA_BYTE_0, irdma_mac_to_u64(info->mac_addr)); 371 372 qw1 = FIELD_PREP(IRDMA_CQPSQ_QHASH_QPN, info->qp_num) | 373 FIELD_PREP(IRDMA_CQPSQ_QHASH_DEST_PORT, info->dest_port); 374 if (info->ipv4_valid) { 375 set_64bit_val(wqe, IRDMA_BYTE_48, 376 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->dest_ip[0])); 377 } else { 378 set_64bit_val(wqe, IRDMA_BYTE_56, 379 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR0, info->dest_ip[0]) | 380 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR1, info->dest_ip[1])); 381 382 set_64bit_val(wqe, IRDMA_BYTE_48, 383 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR2, info->dest_ip[2]) | 384 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->dest_ip[3])); 385 } 386 qw2 = FIELD_PREP(IRDMA_CQPSQ_QHASH_QS_HANDLE, 387 vsi->qos[info->user_pri].qs_handle); 388 if (info->vlan_valid) 389 qw2 |= FIELD_PREP(IRDMA_CQPSQ_QHASH_VLANID, info->vlan_id); 390 set_64bit_val(wqe, IRDMA_BYTE_16, qw2); 391 if (info->entry_type == IRDMA_QHASH_TYPE_TCP_ESTABLISHED) { 392 qw1 |= FIELD_PREP(IRDMA_CQPSQ_QHASH_SRC_PORT, info->src_port); 393 if (!info->ipv4_valid) { 394 set_64bit_val(wqe, IRDMA_BYTE_40, 395 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR0, info->src_ip[0]) | 396 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR1, info->src_ip[1])); 397 set_64bit_val(wqe, IRDMA_BYTE_32, 398 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR2, info->src_ip[2]) | 399 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->src_ip[3])); 400 } else { 401 set_64bit_val(wqe, IRDMA_BYTE_32, 402 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->src_ip[0])); 403 } 404 } 405 406 set_64bit_val(wqe, IRDMA_BYTE_8, qw1); 407 temp = FIELD_PREP(IRDMA_CQPSQ_QHASH_WQEVALID, cqp->polarity) | 408 FIELD_PREP(IRDMA_CQPSQ_QHASH_OPCODE, 409 IRDMA_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY) | 410 FIELD_PREP(IRDMA_CQPSQ_QHASH_MANAGE, info->manage) | 411 FIELD_PREP(IRDMA_CQPSQ_QHASH_IPV4VALID, info->ipv4_valid) | 412 FIELD_PREP(IRDMA_CQPSQ_QHASH_VLANVALID, info->vlan_valid) | 413 FIELD_PREP(IRDMA_CQPSQ_QHASH_ENTRYTYPE, info->entry_type); 414 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 415 416 set_64bit_val(wqe, IRDMA_BYTE_24, temp); 417 418 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "MANAGE_QHASH WQE", wqe, 419 IRDMA_CQP_WQE_SIZE * 8); 420 if (post_sq) 421 irdma_sc_cqp_post_sq(cqp); 422 423 return 0; 424 } 425 426 /** 427 * irdma_sc_qp_init - initialize qp 428 * @qp: sc qp 429 * @info: initialization qp info 430 */ 431 int 432 irdma_sc_qp_init(struct irdma_sc_qp *qp, struct irdma_qp_init_info *info) 433 { 434 int ret_code; 435 u32 pble_obj_cnt; 436 u16 wqe_size; 437 438 if (info->qp_uk_init_info.max_sq_frag_cnt > 439 info->pd->dev->hw_attrs.uk_attrs.max_hw_wq_frags || 440 info->qp_uk_init_info.max_rq_frag_cnt > 441 info->pd->dev->hw_attrs.uk_attrs.max_hw_wq_frags) 442 return -EINVAL; 443 444 qp->dev = info->pd->dev; 445 qp->vsi = info->vsi; 446 qp->ieq_qp = info->vsi->exception_lan_q; 447 qp->sq_pa = info->sq_pa; 448 qp->rq_pa = info->rq_pa; 449 qp->hw_host_ctx_pa = info->host_ctx_pa; 450 qp->q2_pa = info->q2_pa; 451 qp->shadow_area_pa = info->shadow_area_pa; 452 qp->q2_buf = info->q2; 453 qp->pd = info->pd; 454 qp->hw_host_ctx = info->host_ctx; 455 info->qp_uk_init_info.wqe_alloc_db = qp->pd->dev->wqe_alloc_db; 456 ret_code = irdma_uk_qp_init(&qp->qp_uk, &info->qp_uk_init_info); 457 if (ret_code) 458 return ret_code; 459 460 qp->virtual_map = info->virtual_map; 461 pble_obj_cnt = info->pd->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; 462 463 if ((info->virtual_map && info->sq_pa >= pble_obj_cnt) || 464 (info->virtual_map && info->rq_pa >= pble_obj_cnt)) 465 return -EINVAL; 466 467 qp->llp_stream_handle = (void *)(-1); 468 qp->hw_sq_size = irdma_get_encoded_wqe_size(qp->qp_uk.sq_ring.size, 469 IRDMA_QUEUE_TYPE_SQ_RQ); 470 irdma_debug(qp->dev, IRDMA_DEBUG_WQE, 471 "hw_sq_size[%04d] sq_ring.size[%04d]\n", qp->hw_sq_size, 472 qp->qp_uk.sq_ring.size); 473 if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) 474 wqe_size = IRDMA_WQE_SIZE_128; 475 else 476 ret_code = irdma_fragcnt_to_wqesize_rq(qp->qp_uk.max_rq_frag_cnt, 477 &wqe_size); 478 if (ret_code) 479 return ret_code; 480 481 qp->hw_rq_size = 482 irdma_get_encoded_wqe_size(qp->qp_uk.rq_size * 483 (wqe_size / IRDMA_QP_WQE_MIN_SIZE), 484 IRDMA_QUEUE_TYPE_SQ_RQ); 485 irdma_debug(qp->dev, IRDMA_DEBUG_WQE, 486 "hw_rq_size[%04d] qp_uk.rq_size[%04d] wqe_size[%04d]\n", 487 qp->hw_rq_size, qp->qp_uk.rq_size, wqe_size); 488 489 qp->sq_tph_val = info->sq_tph_val; 490 qp->rq_tph_val = info->rq_tph_val; 491 qp->sq_tph_en = info->sq_tph_en; 492 qp->rq_tph_en = info->rq_tph_en; 493 qp->rcv_tph_en = info->rcv_tph_en; 494 qp->xmit_tph_en = info->xmit_tph_en; 495 qp->qp_uk.first_sq_wq = info->qp_uk_init_info.first_sq_wq; 496 qp->qs_handle = qp->vsi->qos[qp->user_pri].qs_handle; 497 498 return 0; 499 } 500 501 /** 502 * irdma_sc_qp_create - create qp 503 * @qp: sc qp 504 * @info: qp create info 505 * @scratch: u64 saved to be used during cqp completion 506 * @post_sq: flag for cqp db to ring 507 */ 508 int 509 irdma_sc_qp_create(struct irdma_sc_qp *qp, struct irdma_create_qp_info *info, 510 u64 scratch, bool post_sq) 511 { 512 struct irdma_sc_cqp *cqp; 513 __le64 *wqe; 514 u64 hdr; 515 516 cqp = qp->dev->cqp; 517 if (qp->qp_uk.qp_id < cqp->dev->hw_attrs.min_hw_qp_id || 518 qp->qp_uk.qp_id > (cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt - 1)) 519 return -EINVAL; 520 521 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 522 if (!wqe) 523 return -ENOSPC; 524 525 set_64bit_val(wqe, IRDMA_BYTE_16, qp->hw_host_ctx_pa); 526 set_64bit_val(wqe, IRDMA_BYTE_40, qp->shadow_area_pa); 527 528 hdr = qp->qp_uk.qp_id | 529 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_QP) | 530 FIELD_PREP(IRDMA_CQPSQ_QP_ORDVALID, info->ord_valid) | 531 FIELD_PREP(IRDMA_CQPSQ_QP_TOECTXVALID, info->tcp_ctx_valid) | 532 FIELD_PREP(IRDMA_CQPSQ_QP_MACVALID, info->mac_valid) | 533 FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp->qp_uk.qp_type) | 534 FIELD_PREP(IRDMA_CQPSQ_QP_VQ, qp->virtual_map) | 535 FIELD_PREP(IRDMA_CQPSQ_QP_FORCELOOPBACK, info->force_lpb) | 536 FIELD_PREP(IRDMA_CQPSQ_QP_CQNUMVALID, info->cq_num_valid) | 537 FIELD_PREP(IRDMA_CQPSQ_QP_ARPTABIDXVALID, 538 info->arp_cache_idx_valid) | 539 FIELD_PREP(IRDMA_CQPSQ_QP_NEXTIWSTATE, info->next_iwarp_state) | 540 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 541 542 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 543 544 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 545 546 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "QP_CREATE WQE", wqe, 547 IRDMA_CQP_WQE_SIZE * 8); 548 if (post_sq) 549 irdma_sc_cqp_post_sq(cqp); 550 551 return 0; 552 } 553 554 /** 555 * irdma_sc_qp_modify - modify qp cqp wqe 556 * @qp: sc qp 557 * @info: modify qp info 558 * @scratch: u64 saved to be used during cqp completion 559 * @post_sq: flag for cqp db to ring 560 */ 561 int 562 irdma_sc_qp_modify(struct irdma_sc_qp *qp, struct irdma_modify_qp_info *info, 563 u64 scratch, bool post_sq) 564 { 565 __le64 *wqe; 566 struct irdma_sc_cqp *cqp; 567 u64 hdr; 568 u8 term_actions = 0; 569 u8 term_len = 0; 570 571 cqp = qp->dev->cqp; 572 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 573 if (!wqe) 574 return -ENOSPC; 575 576 if (info->next_iwarp_state == IRDMA_QP_STATE_TERMINATE) { 577 if (info->dont_send_fin) 578 term_actions += IRDMAQP_TERM_SEND_TERM_ONLY; 579 if (info->dont_send_term) 580 term_actions += IRDMAQP_TERM_SEND_FIN_ONLY; 581 if (term_actions == IRDMAQP_TERM_SEND_TERM_AND_FIN || 582 term_actions == IRDMAQP_TERM_SEND_TERM_ONLY) 583 term_len = info->termlen; 584 } 585 586 set_64bit_val(wqe, IRDMA_BYTE_8, 587 FIELD_PREP(IRDMA_CQPSQ_QP_NEWMSS, info->new_mss) | 588 FIELD_PREP(IRDMA_CQPSQ_QP_TERMLEN, term_len)); 589 set_64bit_val(wqe, IRDMA_BYTE_16, qp->hw_host_ctx_pa); 590 set_64bit_val(wqe, IRDMA_BYTE_40, qp->shadow_area_pa); 591 592 hdr = qp->qp_uk.qp_id | 593 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MODIFY_QP) | 594 FIELD_PREP(IRDMA_CQPSQ_QP_ORDVALID, info->ord_valid) | 595 FIELD_PREP(IRDMA_CQPSQ_QP_TOECTXVALID, info->tcp_ctx_valid) | 596 FIELD_PREP(IRDMA_CQPSQ_QP_CACHEDVARVALID, 597 info->cached_var_valid) | 598 FIELD_PREP(IRDMA_CQPSQ_QP_VQ, qp->virtual_map) | 599 FIELD_PREP(IRDMA_CQPSQ_QP_FORCELOOPBACK, info->force_lpb) | 600 FIELD_PREP(IRDMA_CQPSQ_QP_CQNUMVALID, info->cq_num_valid) | 601 FIELD_PREP(IRDMA_CQPSQ_QP_MACVALID, info->mac_valid) | 602 FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp->qp_uk.qp_type) | 603 FIELD_PREP(IRDMA_CQPSQ_QP_MSSCHANGE, info->mss_change) | 604 FIELD_PREP(IRDMA_CQPSQ_QP_REMOVEHASHENTRY, 605 info->remove_hash_idx) | 606 FIELD_PREP(IRDMA_CQPSQ_QP_TERMACT, term_actions) | 607 FIELD_PREP(IRDMA_CQPSQ_QP_RESETCON, info->reset_tcp_conn) | 608 FIELD_PREP(IRDMA_CQPSQ_QP_ARPTABIDXVALID, 609 info->arp_cache_idx_valid) | 610 FIELD_PREP(IRDMA_CQPSQ_QP_NEXTIWSTATE, info->next_iwarp_state) | 611 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 612 613 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 614 615 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 616 617 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "QP_MODIFY WQE", wqe, 618 IRDMA_CQP_WQE_SIZE * 8); 619 if (post_sq) 620 irdma_sc_cqp_post_sq(cqp); 621 622 return 0; 623 } 624 625 /** 626 * irdma_sc_qp_destroy - cqp destroy qp 627 * @qp: sc qp 628 * @scratch: u64 saved to be used during cqp completion 629 * @remove_hash_idx: flag if to remove hash idx 630 * @ignore_mw_bnd: memory window bind flag 631 * @post_sq: flag for cqp db to ring 632 */ 633 int 634 irdma_sc_qp_destroy(struct irdma_sc_qp *qp, u64 scratch, 635 bool remove_hash_idx, bool ignore_mw_bnd, bool post_sq) 636 { 637 __le64 *wqe; 638 struct irdma_sc_cqp *cqp; 639 u64 hdr; 640 641 cqp = qp->dev->cqp; 642 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 643 if (!wqe) 644 return -ENOSPC; 645 646 set_64bit_val(wqe, IRDMA_BYTE_16, qp->hw_host_ctx_pa); 647 set_64bit_val(wqe, IRDMA_BYTE_40, qp->shadow_area_pa); 648 649 hdr = qp->qp_uk.qp_id | 650 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_QP) | 651 FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp->qp_uk.qp_type) | 652 FIELD_PREP(IRDMA_CQPSQ_QP_IGNOREMWBOUND, ignore_mw_bnd) | 653 FIELD_PREP(IRDMA_CQPSQ_QP_REMOVEHASHENTRY, remove_hash_idx) | 654 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 655 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 656 657 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 658 659 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "QP_DESTROY WQE", wqe, 660 IRDMA_CQP_WQE_SIZE * 8); 661 if (post_sq) 662 irdma_sc_cqp_post_sq(cqp); 663 664 return 0; 665 } 666 667 /** 668 * irdma_sc_get_encoded_ird_size - 669 * @ird_size: IRD size 670 * The ird from the connection is rounded to a supported HW setting and then encoded 671 * for ird_size field of qp_ctx. Consumers are expected to provide valid ird size based 672 * on hardware attributes. IRD size defaults to a value of 4 in case of invalid input 673 */ 674 static u8 irdma_sc_get_encoded_ird_size(u16 ird_size) { 675 switch (ird_size ? 676 roundup_pow_of_two(2 * ird_size) : 4) { 677 case 256: 678 return IRDMA_IRD_HW_SIZE_256; 679 case 128: 680 return IRDMA_IRD_HW_SIZE_128; 681 case 64: 682 case 32: 683 return IRDMA_IRD_HW_SIZE_64; 684 case 16: 685 case 8: 686 return IRDMA_IRD_HW_SIZE_16; 687 case 4: 688 default: 689 break; 690 } 691 692 return IRDMA_IRD_HW_SIZE_4; 693 } 694 695 /** 696 * irdma_sc_qp_setctx_roce - set qp's context 697 * @qp: sc qp 698 * @qp_ctx: context ptr 699 * @info: ctx info 700 */ 701 void 702 irdma_sc_qp_setctx_roce(struct irdma_sc_qp *qp, __le64 * qp_ctx, 703 struct irdma_qp_host_ctx_info *info) 704 { 705 struct irdma_roce_offload_info *roce_info; 706 struct irdma_udp_offload_info *udp; 707 u8 push_mode_en; 708 u32 push_idx; 709 710 roce_info = info->roce_info; 711 udp = info->udp_info; 712 713 qp->user_pri = info->user_pri; 714 if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) { 715 push_mode_en = 0; 716 push_idx = 0; 717 } else { 718 push_mode_en = 1; 719 push_idx = qp->push_idx; 720 } 721 set_64bit_val(qp_ctx, IRDMA_BYTE_0, 722 FIELD_PREP(IRDMAQPC_RQWQESIZE, qp->qp_uk.rq_wqe_size) | 723 FIELD_PREP(IRDMAQPC_RCVTPHEN, qp->rcv_tph_en) | 724 FIELD_PREP(IRDMAQPC_XMITTPHEN, qp->xmit_tph_en) | 725 FIELD_PREP(IRDMAQPC_RQTPHEN, qp->rq_tph_en) | 726 FIELD_PREP(IRDMAQPC_SQTPHEN, qp->sq_tph_en) | 727 FIELD_PREP(IRDMAQPC_PPIDX, push_idx) | 728 FIELD_PREP(IRDMAQPC_PMENA, push_mode_en) | 729 FIELD_PREP(IRDMAQPC_PDIDXHI, roce_info->pd_id >> 16) | 730 FIELD_PREP(IRDMAQPC_DC_TCP_EN, roce_info->dctcp_en) | 731 FIELD_PREP(IRDMAQPC_ERR_RQ_IDX_VALID, roce_info->err_rq_idx_valid) | 732 FIELD_PREP(IRDMAQPC_ISQP1, roce_info->is_qp1) | 733 FIELD_PREP(IRDMAQPC_ROCE_TVER, roce_info->roce_tver) | 734 FIELD_PREP(IRDMAQPC_IPV4, udp->ipv4) | 735 FIELD_PREP(IRDMAQPC_INSERTVLANTAG, udp->insert_vlan_tag)); 736 set_64bit_val(qp_ctx, IRDMA_BYTE_8, qp->sq_pa); 737 set_64bit_val(qp_ctx, IRDMA_BYTE_16, qp->rq_pa); 738 if (roce_info->dcqcn_en || roce_info->dctcp_en) { 739 udp->tos &= ~ECN_CODE_PT_MASK; 740 udp->tos |= ECN_CODE_PT_VAL; 741 } 742 743 set_64bit_val(qp_ctx, IRDMA_BYTE_24, 744 FIELD_PREP(IRDMAQPC_RQSIZE, qp->hw_rq_size) | 745 FIELD_PREP(IRDMAQPC_SQSIZE, qp->hw_sq_size) | 746 FIELD_PREP(IRDMAQPC_TTL, udp->ttl) | FIELD_PREP(IRDMAQPC_TOS, udp->tos) | 747 FIELD_PREP(IRDMAQPC_SRCPORTNUM, udp->src_port) | 748 FIELD_PREP(IRDMAQPC_DESTPORTNUM, udp->dst_port)); 749 set_64bit_val(qp_ctx, IRDMA_BYTE_32, 750 FIELD_PREP(IRDMAQPC_DESTIPADDR2, udp->dest_ip_addr[2]) | 751 FIELD_PREP(IRDMAQPC_DESTIPADDR3, udp->dest_ip_addr[3])); 752 set_64bit_val(qp_ctx, IRDMA_BYTE_40, 753 FIELD_PREP(IRDMAQPC_DESTIPADDR0, udp->dest_ip_addr[0]) | 754 FIELD_PREP(IRDMAQPC_DESTIPADDR1, udp->dest_ip_addr[1])); 755 set_64bit_val(qp_ctx, IRDMA_BYTE_48, 756 FIELD_PREP(IRDMAQPC_SNDMSS, udp->snd_mss) | 757 FIELD_PREP(IRDMAQPC_VLANTAG, udp->vlan_tag) | 758 FIELD_PREP(IRDMAQPC_ARPIDX, udp->arp_idx)); 759 set_64bit_val(qp_ctx, IRDMA_BYTE_56, 760 FIELD_PREP(IRDMAQPC_PKEY, roce_info->p_key) | 761 FIELD_PREP(IRDMAQPC_PDIDX, roce_info->pd_id) | 762 FIELD_PREP(IRDMAQPC_ACKCREDITS, roce_info->ack_credits) | 763 FIELD_PREP(IRDMAQPC_FLOWLABEL, udp->flow_label)); 764 set_64bit_val(qp_ctx, IRDMA_BYTE_64, 765 FIELD_PREP(IRDMAQPC_QKEY, roce_info->qkey) | 766 FIELD_PREP(IRDMAQPC_DESTQP, roce_info->dest_qp)); 767 set_64bit_val(qp_ctx, IRDMA_BYTE_80, 768 FIELD_PREP(IRDMAQPC_PSNNXT, udp->psn_nxt) | 769 FIELD_PREP(IRDMAQPC_LSN, udp->lsn)); 770 set_64bit_val(qp_ctx, IRDMA_BYTE_88, 771 FIELD_PREP(IRDMAQPC_EPSN, udp->epsn)); 772 set_64bit_val(qp_ctx, IRDMA_BYTE_96, 773 FIELD_PREP(IRDMAQPC_PSNMAX, udp->psn_max) | 774 FIELD_PREP(IRDMAQPC_PSNUNA, udp->psn_una)); 775 set_64bit_val(qp_ctx, IRDMA_BYTE_112, 776 FIELD_PREP(IRDMAQPC_CWNDROCE, udp->cwnd)); 777 set_64bit_val(qp_ctx, IRDMA_BYTE_128, 778 FIELD_PREP(IRDMAQPC_ERR_RQ_IDX, roce_info->err_rq_idx) | 779 FIELD_PREP(IRDMAQPC_RNRNAK_THRESH, udp->rnr_nak_thresh) | 780 FIELD_PREP(IRDMAQPC_REXMIT_THRESH, udp->rexmit_thresh) | 781 FIELD_PREP(IRDMAQPC_RTOMIN, roce_info->rtomin)); 782 set_64bit_val(qp_ctx, IRDMA_BYTE_136, 783 FIELD_PREP(IRDMAQPC_TXCQNUM, info->send_cq_num) | 784 FIELD_PREP(IRDMAQPC_RXCQNUM, info->rcv_cq_num)); 785 set_64bit_val(qp_ctx, IRDMA_BYTE_144, 786 FIELD_PREP(IRDMAQPC_STAT_INDEX, info->stats_idx)); 787 set_64bit_val(qp_ctx, IRDMA_BYTE_152, 788 FIELD_PREP(IRDMAQPC_MACADDRESS, irdma_mac_to_u64(roce_info->mac_addr))); 789 set_64bit_val(qp_ctx, IRDMA_BYTE_160, 790 FIELD_PREP(IRDMAQPC_ORDSIZE, roce_info->ord_size) | 791 FIELD_PREP(IRDMAQPC_IRDSIZE, irdma_sc_get_encoded_ird_size(roce_info->ird_size)) | 792 FIELD_PREP(IRDMAQPC_WRRDRSPOK, roce_info->wr_rdresp_en) | 793 FIELD_PREP(IRDMAQPC_RDOK, roce_info->rd_en) | 794 FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info->stats_idx_valid) | 795 FIELD_PREP(IRDMAQPC_BINDEN, roce_info->bind_en) | 796 FIELD_PREP(IRDMAQPC_FASTREGEN, roce_info->fast_reg_en) | 797 FIELD_PREP(IRDMAQPC_DCQCNENABLE, roce_info->dcqcn_en) | 798 FIELD_PREP(IRDMAQPC_RCVNOICRC, roce_info->rcv_no_icrc) | 799 FIELD_PREP(IRDMAQPC_FW_CC_ENABLE, roce_info->fw_cc_enable) | 800 FIELD_PREP(IRDMAQPC_UDPRIVCQENABLE, roce_info->udprivcq_en) | 801 FIELD_PREP(IRDMAQPC_PRIVEN, roce_info->priv_mode_en) | 802 FIELD_PREP(IRDMAQPC_TIMELYENABLE, roce_info->timely_en)); 803 set_64bit_val(qp_ctx, IRDMA_BYTE_168, 804 FIELD_PREP(IRDMAQPC_QPCOMPCTX, info->qp_compl_ctx)); 805 set_64bit_val(qp_ctx, IRDMA_BYTE_176, 806 FIELD_PREP(IRDMAQPC_SQTPHVAL, qp->sq_tph_val) | 807 FIELD_PREP(IRDMAQPC_RQTPHVAL, qp->rq_tph_val) | 808 FIELD_PREP(IRDMAQPC_QSHANDLE, qp->qs_handle)); 809 set_64bit_val(qp_ctx, IRDMA_BYTE_184, 810 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR3, udp->local_ipaddr[3]) | 811 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR2, udp->local_ipaddr[2])); 812 set_64bit_val(qp_ctx, IRDMA_BYTE_192, 813 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR1, udp->local_ipaddr[1]) | 814 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR0, udp->local_ipaddr[0])); 815 set_64bit_val(qp_ctx, IRDMA_BYTE_200, 816 FIELD_PREP(IRDMAQPC_THIGH, roce_info->t_high) | 817 FIELD_PREP(IRDMAQPC_TLOW, roce_info->t_low)); 818 set_64bit_val(qp_ctx, IRDMA_BYTE_208, 819 FIELD_PREP(IRDMAQPC_REMENDPOINTIDX, info->rem_endpoint_idx)); 820 821 irdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, "QP_HOST CTX WQE", qp_ctx, 822 IRDMA_QP_CTX_SIZE); 823 } 824 825 /* 826 * irdma_sc_alloc_local_mac_entry - allocate a mac entry @cqp: struct for cqp hw @scratch: u64 saved to be used during 827 * cqp completion @post_sq: flag for cqp db to ring 828 */ 829 static int 830 irdma_sc_alloc_local_mac_entry(struct irdma_sc_cqp *cqp, u64 scratch, 831 bool post_sq) 832 { 833 __le64 *wqe; 834 u64 hdr; 835 836 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 837 if (!wqe) 838 return -ENOSPC; 839 840 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, 841 IRDMA_CQP_OP_ALLOCATE_LOC_MAC_TABLE_ENTRY) | 842 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 843 844 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 845 846 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 847 848 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "ALLOCATE_LOCAL_MAC WQE", 849 wqe, IRDMA_CQP_WQE_SIZE * 8); 850 851 if (post_sq) 852 irdma_sc_cqp_post_sq(cqp); 853 return 0; 854 } 855 856 /** 857 * irdma_sc_add_local_mac_entry - add mac enry 858 * @cqp: struct for cqp hw 859 * @info:mac addr info 860 * @scratch: u64 saved to be used during cqp completion 861 * @post_sq: flag for cqp db to ring 862 */ 863 static int 864 irdma_sc_add_local_mac_entry(struct irdma_sc_cqp *cqp, 865 struct irdma_local_mac_entry_info *info, 866 u64 scratch, bool post_sq) 867 { 868 __le64 *wqe; 869 u64 header; 870 871 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 872 if (!wqe) 873 return -ENOSPC; 874 875 set_64bit_val(wqe, IRDMA_BYTE_32, irdma_mac_to_u64(info->mac_addr)); 876 877 header = FIELD_PREP(IRDMA_CQPSQ_MLM_TABLEIDX, info->entry_idx) | 878 FIELD_PREP(IRDMA_CQPSQ_OPCODE, 879 IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE) | 880 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 881 882 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 883 884 set_64bit_val(wqe, IRDMA_BYTE_24, header); 885 886 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "ADD_LOCAL_MAC WQE", wqe, 887 IRDMA_CQP_WQE_SIZE * 8); 888 889 if (post_sq) 890 irdma_sc_cqp_post_sq(cqp); 891 return 0; 892 } 893 894 /** 895 * irdma_sc_del_local_mac_entry - cqp wqe to dele local mac 896 * @cqp: struct for cqp hw 897 * @scratch: u64 saved to be used during cqp completion 898 * @entry_idx: index of mac entry 899 * @ignore_ref_count: to force mac adde delete 900 * @post_sq: flag for cqp db to ring 901 */ 902 static int 903 irdma_sc_del_local_mac_entry(struct irdma_sc_cqp *cqp, u64 scratch, 904 u16 entry_idx, u8 ignore_ref_count, 905 bool post_sq) 906 { 907 __le64 *wqe; 908 u64 header; 909 910 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 911 if (!wqe) 912 return -ENOSPC; 913 header = FIELD_PREP(IRDMA_CQPSQ_MLM_TABLEIDX, entry_idx) | 914 FIELD_PREP(IRDMA_CQPSQ_OPCODE, 915 IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE) | 916 FIELD_PREP(IRDMA_CQPSQ_MLM_FREEENTRY, 1) | 917 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) | 918 FIELD_PREP(IRDMA_CQPSQ_MLM_IGNORE_REF_CNT, ignore_ref_count); 919 920 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 921 922 set_64bit_val(wqe, IRDMA_BYTE_24, header); 923 924 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "DEL_LOCAL_MAC_IPADDR WQE", 925 wqe, IRDMA_CQP_WQE_SIZE * 8); 926 927 if (post_sq) 928 irdma_sc_cqp_post_sq(cqp); 929 return 0; 930 } 931 932 /** 933 * irdma_sc_qp_setctx - set qp's context 934 * @qp: sc qp 935 * @qp_ctx: context ptr 936 * @info: ctx info 937 */ 938 void 939 irdma_sc_qp_setctx(struct irdma_sc_qp *qp, __le64 * qp_ctx, 940 struct irdma_qp_host_ctx_info *info) 941 { 942 struct irdma_iwarp_offload_info *iw; 943 struct irdma_tcp_offload_info *tcp; 944 struct irdma_sc_dev *dev; 945 u8 push_mode_en; 946 u32 push_idx; 947 u64 qw0, qw3, qw7 = 0, qw16 = 0; 948 u64 mac = 0; 949 950 iw = info->iwarp_info; 951 tcp = info->tcp_info; 952 dev = qp->dev; 953 if (iw->rcv_mark_en) { 954 qp->pfpdu.marker_len = 4; 955 qp->pfpdu.rcv_start_seq = tcp->rcv_nxt; 956 } 957 qp->user_pri = info->user_pri; 958 if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) { 959 push_mode_en = 0; 960 push_idx = 0; 961 } else { 962 push_mode_en = 1; 963 push_idx = qp->push_idx; 964 } 965 qw0 = FIELD_PREP(IRDMAQPC_RQWQESIZE, qp->qp_uk.rq_wqe_size) | 966 FIELD_PREP(IRDMAQPC_RCVTPHEN, qp->rcv_tph_en) | 967 FIELD_PREP(IRDMAQPC_XMITTPHEN, qp->xmit_tph_en) | 968 FIELD_PREP(IRDMAQPC_RQTPHEN, qp->rq_tph_en) | 969 FIELD_PREP(IRDMAQPC_SQTPHEN, qp->sq_tph_en) | 970 FIELD_PREP(IRDMAQPC_PPIDX, push_idx) | 971 FIELD_PREP(IRDMAQPC_PMENA, push_mode_en); 972 973 set_64bit_val(qp_ctx, IRDMA_BYTE_8, qp->sq_pa); 974 set_64bit_val(qp_ctx, IRDMA_BYTE_16, qp->rq_pa); 975 976 qw3 = FIELD_PREP(IRDMAQPC_RQSIZE, qp->hw_rq_size) | 977 FIELD_PREP(IRDMAQPC_SQSIZE, qp->hw_sq_size); 978 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 979 qw3 |= FIELD_PREP(IRDMAQPC_GEN1_SRCMACADDRIDX, 980 qp->src_mac_addr_idx); 981 set_64bit_val(qp_ctx, IRDMA_BYTE_136, 982 FIELD_PREP(IRDMAQPC_TXCQNUM, info->send_cq_num) | 983 FIELD_PREP(IRDMAQPC_RXCQNUM, info->rcv_cq_num)); 984 set_64bit_val(qp_ctx, IRDMA_BYTE_168, 985 FIELD_PREP(IRDMAQPC_QPCOMPCTX, info->qp_compl_ctx)); 986 set_64bit_val(qp_ctx, IRDMA_BYTE_176, 987 FIELD_PREP(IRDMAQPC_SQTPHVAL, qp->sq_tph_val) | 988 FIELD_PREP(IRDMAQPC_RQTPHVAL, qp->rq_tph_val) | 989 FIELD_PREP(IRDMAQPC_QSHANDLE, qp->qs_handle) | 990 FIELD_PREP(IRDMAQPC_EXCEPTION_LAN_QUEUE, qp->ieq_qp)); 991 if (info->iwarp_info_valid) { 992 qw0 |= FIELD_PREP(IRDMAQPC_DDP_VER, iw->ddp_ver) | 993 FIELD_PREP(IRDMAQPC_RDMAP_VER, iw->rdmap_ver) | 994 FIELD_PREP(IRDMAQPC_DC_TCP_EN, iw->dctcp_en) | 995 FIELD_PREP(IRDMAQPC_ECN_EN, iw->ecn_en) | 996 FIELD_PREP(IRDMAQPC_IBRDENABLE, iw->ib_rd_en) | 997 FIELD_PREP(IRDMAQPC_PDIDXHI, iw->pd_id >> 16) | 998 FIELD_PREP(IRDMAQPC_ERR_RQ_IDX_VALID, 999 iw->err_rq_idx_valid); 1000 qw7 |= FIELD_PREP(IRDMAQPC_PDIDX, iw->pd_id); 1001 qw16 |= FIELD_PREP(IRDMAQPC_ERR_RQ_IDX, iw->err_rq_idx) | 1002 FIELD_PREP(IRDMAQPC_RTOMIN, iw->rtomin); 1003 set_64bit_val(qp_ctx, IRDMA_BYTE_144, 1004 FIELD_PREP(IRDMAQPC_Q2ADDR, qp->q2_pa >> 8) | 1005 FIELD_PREP(IRDMAQPC_STAT_INDEX, info->stats_idx)); 1006 1007 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 1008 mac = FIELD_PREP(IRDMAQPC_MACADDRESS, 1009 irdma_mac_to_u64(iw->mac_addr)); 1010 1011 set_64bit_val(qp_ctx, IRDMA_BYTE_152, 1012 mac | FIELD_PREP(IRDMAQPC_LASTBYTESENT, iw->last_byte_sent)); 1013 set_64bit_val(qp_ctx, IRDMA_BYTE_160, 1014 FIELD_PREP(IRDMAQPC_ORDSIZE, iw->ord_size) | 1015 FIELD_PREP(IRDMAQPC_IRDSIZE, irdma_sc_get_encoded_ird_size(iw->ird_size)) | 1016 FIELD_PREP(IRDMAQPC_WRRDRSPOK, iw->wr_rdresp_en) | 1017 FIELD_PREP(IRDMAQPC_RDOK, iw->rd_en) | 1018 FIELD_PREP(IRDMAQPC_SNDMARKERS, iw->snd_mark_en) | 1019 FIELD_PREP(IRDMAQPC_BINDEN, iw->bind_en) | 1020 FIELD_PREP(IRDMAQPC_FASTREGEN, iw->fast_reg_en) | 1021 FIELD_PREP(IRDMAQPC_PRIVEN, iw->priv_mode_en) | 1022 FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info->stats_idx_valid) | 1023 FIELD_PREP(IRDMAQPC_IWARPMODE, 1) | 1024 FIELD_PREP(IRDMAQPC_RCVMARKERS, iw->rcv_mark_en) | 1025 FIELD_PREP(IRDMAQPC_ALIGNHDRS, iw->align_hdrs) | 1026 FIELD_PREP(IRDMAQPC_RCVNOMPACRC, iw->rcv_no_mpa_crc) | 1027 FIELD_PREP(IRDMAQPC_RCVMARKOFFSET, iw->rcv_mark_offset) | 1028 FIELD_PREP(IRDMAQPC_SNDMARKOFFSET, iw->snd_mark_offset) | 1029 FIELD_PREP(IRDMAQPC_TIMELYENABLE, iw->timely_en)); 1030 } 1031 if (info->tcp_info_valid) { 1032 qw0 |= FIELD_PREP(IRDMAQPC_IPV4, tcp->ipv4) | 1033 FIELD_PREP(IRDMAQPC_NONAGLE, tcp->no_nagle) | 1034 FIELD_PREP(IRDMAQPC_INSERTVLANTAG, 1035 tcp->insert_vlan_tag) | 1036 FIELD_PREP(IRDMAQPC_TIMESTAMP, tcp->time_stamp) | 1037 FIELD_PREP(IRDMAQPC_LIMIT, tcp->cwnd_inc_limit) | 1038 FIELD_PREP(IRDMAQPC_DROPOOOSEG, tcp->drop_ooo_seg) | 1039 FIELD_PREP(IRDMAQPC_DUPACK_THRESH, tcp->dup_ack_thresh); 1040 1041 if (iw->ecn_en || iw->dctcp_en) { 1042 tcp->tos &= ~ECN_CODE_PT_MASK; 1043 tcp->tos |= ECN_CODE_PT_VAL; 1044 } 1045 1046 qw3 |= FIELD_PREP(IRDMAQPC_TTL, tcp->ttl) | 1047 FIELD_PREP(IRDMAQPC_AVOIDSTRETCHACK, tcp->avoid_stretch_ack) | 1048 FIELD_PREP(IRDMAQPC_TOS, tcp->tos) | 1049 FIELD_PREP(IRDMAQPC_SRCPORTNUM, tcp->src_port) | 1050 FIELD_PREP(IRDMAQPC_DESTPORTNUM, tcp->dst_port); 1051 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) { 1052 qw3 |= FIELD_PREP(IRDMAQPC_GEN1_SRCMACADDRIDX, tcp->src_mac_addr_idx); 1053 1054 qp->src_mac_addr_idx = tcp->src_mac_addr_idx; 1055 } 1056 set_64bit_val(qp_ctx, IRDMA_BYTE_32, 1057 FIELD_PREP(IRDMAQPC_DESTIPADDR2, tcp->dest_ip_addr[2]) | 1058 FIELD_PREP(IRDMAQPC_DESTIPADDR3, tcp->dest_ip_addr[3])); 1059 set_64bit_val(qp_ctx, IRDMA_BYTE_40, 1060 FIELD_PREP(IRDMAQPC_DESTIPADDR0, tcp->dest_ip_addr[0]) | 1061 FIELD_PREP(IRDMAQPC_DESTIPADDR1, tcp->dest_ip_addr[1])); 1062 set_64bit_val(qp_ctx, IRDMA_BYTE_48, 1063 FIELD_PREP(IRDMAQPC_SNDMSS, tcp->snd_mss) | 1064 FIELD_PREP(IRDMAQPC_SYN_RST_HANDLING, tcp->syn_rst_handling) | 1065 FIELD_PREP(IRDMAQPC_VLANTAG, tcp->vlan_tag) | 1066 FIELD_PREP(IRDMAQPC_ARPIDX, tcp->arp_idx)); 1067 qw7 |= FIELD_PREP(IRDMAQPC_FLOWLABEL, tcp->flow_label) | 1068 FIELD_PREP(IRDMAQPC_WSCALE, tcp->wscale) | 1069 FIELD_PREP(IRDMAQPC_IGNORE_TCP_OPT, 1070 tcp->ignore_tcp_opt) | 1071 FIELD_PREP(IRDMAQPC_IGNORE_TCP_UNS_OPT, 1072 tcp->ignore_tcp_uns_opt) | 1073 FIELD_PREP(IRDMAQPC_TCPSTATE, tcp->tcp_state) | 1074 FIELD_PREP(IRDMAQPC_RCVSCALE, tcp->rcv_wscale) | 1075 FIELD_PREP(IRDMAQPC_SNDSCALE, tcp->snd_wscale); 1076 set_64bit_val(qp_ctx, IRDMA_BYTE_72, 1077 FIELD_PREP(IRDMAQPC_TIMESTAMP_RECENT, tcp->time_stamp_recent) | 1078 FIELD_PREP(IRDMAQPC_TIMESTAMP_AGE, tcp->time_stamp_age)); 1079 set_64bit_val(qp_ctx, IRDMA_BYTE_80, 1080 FIELD_PREP(IRDMAQPC_SNDNXT, tcp->snd_nxt) | 1081 FIELD_PREP(IRDMAQPC_SNDWND, tcp->snd_wnd)); 1082 set_64bit_val(qp_ctx, IRDMA_BYTE_88, 1083 FIELD_PREP(IRDMAQPC_RCVNXT, tcp->rcv_nxt) | 1084 FIELD_PREP(IRDMAQPC_RCVWND, tcp->rcv_wnd)); 1085 set_64bit_val(qp_ctx, IRDMA_BYTE_96, 1086 FIELD_PREP(IRDMAQPC_SNDMAX, tcp->snd_max) | 1087 FIELD_PREP(IRDMAQPC_SNDUNA, tcp->snd_una)); 1088 set_64bit_val(qp_ctx, IRDMA_BYTE_104, 1089 FIELD_PREP(IRDMAQPC_SRTT, tcp->srtt) | 1090 FIELD_PREP(IRDMAQPC_RTTVAR, tcp->rtt_var)); 1091 set_64bit_val(qp_ctx, IRDMA_BYTE_112, 1092 FIELD_PREP(IRDMAQPC_SSTHRESH, tcp->ss_thresh) | 1093 FIELD_PREP(IRDMAQPC_CWND, tcp->cwnd)); 1094 set_64bit_val(qp_ctx, IRDMA_BYTE_120, 1095 FIELD_PREP(IRDMAQPC_SNDWL1, tcp->snd_wl1) | 1096 FIELD_PREP(IRDMAQPC_SNDWL2, tcp->snd_wl2)); 1097 qw16 |= FIELD_PREP(IRDMAQPC_MAXSNDWND, tcp->max_snd_window) | 1098 FIELD_PREP(IRDMAQPC_REXMIT_THRESH, tcp->rexmit_thresh); 1099 set_64bit_val(qp_ctx, IRDMA_BYTE_184, 1100 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR3, tcp->local_ipaddr[3]) | 1101 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR2, tcp->local_ipaddr[2])); 1102 set_64bit_val(qp_ctx, IRDMA_BYTE_192, 1103 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR1, tcp->local_ipaddr[1]) | 1104 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR0, tcp->local_ipaddr[0])); 1105 set_64bit_val(qp_ctx, IRDMA_BYTE_200, 1106 FIELD_PREP(IRDMAQPC_THIGH, iw->t_high) | 1107 FIELD_PREP(IRDMAQPC_TLOW, iw->t_low)); 1108 set_64bit_val(qp_ctx, IRDMA_BYTE_208, 1109 FIELD_PREP(IRDMAQPC_REMENDPOINTIDX, info->rem_endpoint_idx)); 1110 } 1111 1112 set_64bit_val(qp_ctx, IRDMA_BYTE_0, qw0); 1113 set_64bit_val(qp_ctx, IRDMA_BYTE_24, qw3); 1114 set_64bit_val(qp_ctx, IRDMA_BYTE_56, qw7); 1115 set_64bit_val(qp_ctx, IRDMA_BYTE_128, qw16); 1116 1117 irdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, "QP_HOST CTX", qp_ctx, 1118 IRDMA_QP_CTX_SIZE); 1119 } 1120 1121 /** 1122 * irdma_sc_alloc_stag - mr stag alloc 1123 * @dev: sc device struct 1124 * @info: stag info 1125 * @scratch: u64 saved to be used during cqp completion 1126 * @post_sq: flag for cqp db to ring 1127 */ 1128 static int 1129 irdma_sc_alloc_stag(struct irdma_sc_dev *dev, 1130 struct irdma_allocate_stag_info *info, 1131 u64 scratch, bool post_sq) 1132 { 1133 __le64 *wqe; 1134 struct irdma_sc_cqp *cqp; 1135 u64 hdr; 1136 enum irdma_page_size page_size; 1137 1138 if (!info->total_len && !info->all_memory) 1139 return -EINVAL; 1140 1141 if (info->page_size == 0x40000000) 1142 page_size = IRDMA_PAGE_SIZE_1G; 1143 else if (info->page_size == 0x200000) 1144 page_size = IRDMA_PAGE_SIZE_2M; 1145 else 1146 page_size = IRDMA_PAGE_SIZE_4K; 1147 1148 cqp = dev->cqp; 1149 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 1150 if (!wqe) 1151 return -ENOSPC; 1152 1153 set_64bit_val(wqe, IRDMA_BYTE_8, 1154 FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID) | 1155 FIELD_PREP(IRDMA_CQPSQ_STAG_STAGLEN, info->total_len)); 1156 set_64bit_val(wqe, IRDMA_BYTE_16, 1157 FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->stag_idx)); 1158 set_64bit_val(wqe, IRDMA_BYTE_40, 1159 FIELD_PREP(IRDMA_CQPSQ_STAG_HMCFNIDX, info->hmc_fcn_index)); 1160 1161 if (info->chunk_size) 1162 set_64bit_val(wqe, IRDMA_BYTE_48, 1163 FIELD_PREP(IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX, info->first_pm_pbl_idx)); 1164 1165 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_ALLOC_STAG) | 1166 FIELD_PREP(IRDMA_CQPSQ_STAG_MR, 1) | 1167 FIELD_PREP(IRDMA_CQPSQ_STAG_ARIGHTS, info->access_rights) | 1168 FIELD_PREP(IRDMA_CQPSQ_STAG_LPBLSIZE, info->chunk_size) | 1169 FIELD_PREP(IRDMA_CQPSQ_STAG_HPAGESIZE, page_size) | 1170 FIELD_PREP(IRDMA_CQPSQ_STAG_REMACCENABLED, info->remote_access) | 1171 FIELD_PREP(IRDMA_CQPSQ_STAG_USEHMCFNIDX, info->use_hmc_fcn_index) | 1172 FIELD_PREP(IRDMA_CQPSQ_STAG_USEPFRID, info->use_pf_rid) | 1173 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 1174 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 1175 1176 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 1177 1178 irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "ALLOC_STAG WQE", wqe, 1179 IRDMA_CQP_WQE_SIZE * 8); 1180 if (post_sq) 1181 irdma_sc_cqp_post_sq(cqp); 1182 1183 return 0; 1184 } 1185 1186 /** 1187 * irdma_sc_mr_reg_non_shared - non-shared mr registration 1188 * @dev: sc device struct 1189 * @info: mr info 1190 * @scratch: u64 saved to be used during cqp completion 1191 * @post_sq: flag for cqp db to ring 1192 */ 1193 static int 1194 irdma_sc_mr_reg_non_shared(struct irdma_sc_dev *dev, 1195 struct irdma_reg_ns_stag_info *info, 1196 u64 scratch, bool post_sq) 1197 { 1198 __le64 *wqe; 1199 u64 fbo; 1200 struct irdma_sc_cqp *cqp; 1201 u64 hdr; 1202 u32 pble_obj_cnt; 1203 bool remote_access; 1204 u8 addr_type; 1205 enum irdma_page_size page_size; 1206 1207 if (!info->total_len && !info->all_memory) 1208 return -EINVAL; 1209 1210 if (info->page_size == 0x40000000) 1211 page_size = IRDMA_PAGE_SIZE_1G; 1212 else if (info->page_size == 0x200000) 1213 page_size = IRDMA_PAGE_SIZE_2M; 1214 else if (info->page_size == 0x1000) 1215 page_size = IRDMA_PAGE_SIZE_4K; 1216 else 1217 return -EINVAL; 1218 1219 if (info->access_rights & (IRDMA_ACCESS_FLAGS_REMOTEREAD_ONLY | 1220 IRDMA_ACCESS_FLAGS_REMOTEWRITE_ONLY)) 1221 remote_access = true; 1222 else 1223 remote_access = false; 1224 1225 pble_obj_cnt = dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; 1226 if (info->chunk_size && info->first_pm_pbl_index >= pble_obj_cnt) 1227 return -EINVAL; 1228 1229 cqp = dev->cqp; 1230 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 1231 if (!wqe) 1232 return -ENOSPC; 1233 fbo = info->va & (info->page_size - 1); 1234 1235 set_64bit_val(wqe, IRDMA_BYTE_0, 1236 (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED ? 1237 info->va : fbo)); 1238 set_64bit_val(wqe, IRDMA_BYTE_8, 1239 FIELD_PREP(IRDMA_CQPSQ_STAG_STAGLEN, info->total_len) | 1240 FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID)); 1241 set_64bit_val(wqe, IRDMA_BYTE_16, 1242 FIELD_PREP(IRDMA_CQPSQ_STAG_KEY, info->stag_key) | 1243 FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->stag_idx)); 1244 if (!info->chunk_size) 1245 set_64bit_val(wqe, IRDMA_BYTE_32, info->reg_addr_pa); 1246 else 1247 set_64bit_val(wqe, IRDMA_BYTE_48, 1248 FIELD_PREP(IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX, info->first_pm_pbl_index)); 1249 1250 set_64bit_val(wqe, IRDMA_BYTE_40, info->hmc_fcn_index); 1251 1252 addr_type = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ? 1 : 0; 1253 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_REG_MR) | 1254 FIELD_PREP(IRDMA_CQPSQ_STAG_MR, 1) | 1255 FIELD_PREP(IRDMA_CQPSQ_STAG_LPBLSIZE, info->chunk_size) | 1256 FIELD_PREP(IRDMA_CQPSQ_STAG_HPAGESIZE, page_size) | 1257 FIELD_PREP(IRDMA_CQPSQ_STAG_ARIGHTS, info->access_rights) | 1258 FIELD_PREP(IRDMA_CQPSQ_STAG_REMACCENABLED, remote_access) | 1259 FIELD_PREP(IRDMA_CQPSQ_STAG_VABASEDTO, addr_type) | 1260 FIELD_PREP(IRDMA_CQPSQ_STAG_USEHMCFNIDX, info->use_hmc_fcn_index) | 1261 FIELD_PREP(IRDMA_CQPSQ_STAG_USEPFRID, info->use_pf_rid) | 1262 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 1263 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 1264 1265 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 1266 1267 irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "MR_REG_NS WQE", wqe, 1268 IRDMA_CQP_WQE_SIZE * 8); 1269 if (post_sq) 1270 irdma_sc_cqp_post_sq(cqp); 1271 1272 return 0; 1273 } 1274 1275 /** 1276 * irdma_sc_dealloc_stag - deallocate stag 1277 * @dev: sc device struct 1278 * @info: dealloc stag info 1279 * @scratch: u64 saved to be used during cqp completion 1280 * @post_sq: flag for cqp db to ring 1281 */ 1282 static int 1283 irdma_sc_dealloc_stag(struct irdma_sc_dev *dev, 1284 struct irdma_dealloc_stag_info *info, 1285 u64 scratch, bool post_sq) 1286 { 1287 u64 hdr; 1288 __le64 *wqe; 1289 struct irdma_sc_cqp *cqp; 1290 1291 cqp = dev->cqp; 1292 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 1293 if (!wqe) 1294 return -ENOSPC; 1295 1296 set_64bit_val(wqe, IRDMA_BYTE_8, 1297 FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID)); 1298 set_64bit_val(wqe, IRDMA_BYTE_16, 1299 FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->stag_idx)); 1300 1301 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DEALLOC_STAG) | 1302 FIELD_PREP(IRDMA_CQPSQ_STAG_MR, info->mr) | 1303 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 1304 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 1305 1306 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 1307 1308 irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "DEALLOC_STAG WQE", wqe, 1309 IRDMA_CQP_WQE_SIZE * 8); 1310 if (post_sq) 1311 irdma_sc_cqp_post_sq(cqp); 1312 1313 return 0; 1314 } 1315 1316 /** 1317 * irdma_sc_mw_alloc - mw allocate 1318 * @dev: sc device struct 1319 * @info: memory window allocation information 1320 * @scratch: u64 saved to be used during cqp completion 1321 * @post_sq: flag for cqp db to ring 1322 */ 1323 static int 1324 irdma_sc_mw_alloc(struct irdma_sc_dev *dev, 1325 struct irdma_mw_alloc_info *info, u64 scratch, 1326 bool post_sq) 1327 { 1328 u64 hdr; 1329 struct irdma_sc_cqp *cqp; 1330 __le64 *wqe; 1331 1332 cqp = dev->cqp; 1333 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 1334 if (!wqe) 1335 return -ENOSPC; 1336 1337 set_64bit_val(wqe, IRDMA_BYTE_8, 1338 FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID)); 1339 set_64bit_val(wqe, IRDMA_BYTE_16, 1340 FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->mw_stag_index)); 1341 1342 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_ALLOC_STAG) | 1343 FIELD_PREP(IRDMA_CQPSQ_STAG_MWTYPE, info->mw_wide) | 1344 FIELD_PREP(IRDMA_CQPSQ_STAG_MW1_BIND_DONT_VLDT_KEY, 1345 info->mw1_bind_dont_vldt_key) | 1346 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 1347 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 1348 1349 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 1350 1351 irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "MW_ALLOC WQE", wqe, 1352 IRDMA_CQP_WQE_SIZE * 8); 1353 if (post_sq) 1354 irdma_sc_cqp_post_sq(cqp); 1355 1356 return 0; 1357 } 1358 1359 /** 1360 * irdma_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp 1361 * @qp: sc qp struct 1362 * @info: fast mr info 1363 * @post_sq: flag for cqp db to ring 1364 */ 1365 int 1366 irdma_sc_mr_fast_register(struct irdma_sc_qp *qp, 1367 struct irdma_fast_reg_stag_info *info, 1368 bool post_sq) 1369 { 1370 u64 temp, hdr; 1371 __le64 *wqe; 1372 u32 wqe_idx; 1373 u16 quanta = IRDMA_QP_WQE_MIN_QUANTA; 1374 enum irdma_page_size page_size; 1375 struct irdma_post_sq_info sq_info = {0}; 1376 1377 if (info->page_size == 0x40000000) 1378 page_size = IRDMA_PAGE_SIZE_1G; 1379 else if (info->page_size == 0x200000) 1380 page_size = IRDMA_PAGE_SIZE_2M; 1381 else 1382 page_size = IRDMA_PAGE_SIZE_4K; 1383 1384 sq_info.wr_id = info->wr_id; 1385 sq_info.signaled = info->signaled; 1386 sq_info.push_wqe = info->push_wqe; 1387 1388 wqe = irdma_qp_get_next_send_wqe(&qp->qp_uk, &wqe_idx, &quanta, 0, &sq_info); 1389 if (!wqe) 1390 return -ENOSPC; 1391 1392 qp->qp_uk.sq_wrtrk_array[wqe_idx].signaled = info->signaled; 1393 irdma_debug(qp->dev, IRDMA_DEBUG_MR, 1394 "wr_id[%llxh] wqe_idx[%04d] location[%p]\n", 1395 (unsigned long long)info->wr_id, wqe_idx, 1396 &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid); 1397 1398 temp = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ? 1399 (uintptr_t)info->va : info->fbo; 1400 set_64bit_val(wqe, IRDMA_BYTE_0, temp); 1401 1402 temp = FIELD_GET(IRDMAQPSQ_FIRSTPMPBLIDXHI, 1403 info->first_pm_pbl_index >> 16); 1404 set_64bit_val(wqe, IRDMA_BYTE_8, 1405 FIELD_PREP(IRDMAQPSQ_FIRSTPMPBLIDXHI, temp) | 1406 FIELD_PREP(IRDMAQPSQ_PBLADDR, info->reg_addr_pa >> IRDMA_HW_PAGE_SHIFT)); 1407 set_64bit_val(wqe, IRDMA_BYTE_16, 1408 info->total_len | 1409 FIELD_PREP(IRDMAQPSQ_FIRSTPMPBLIDXLO, info->first_pm_pbl_index)); 1410 1411 hdr = FIELD_PREP(IRDMAQPSQ_STAGKEY, info->stag_key) | 1412 FIELD_PREP(IRDMAQPSQ_STAGINDEX, info->stag_idx) | 1413 FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_FAST_REGISTER) | 1414 FIELD_PREP(IRDMAQPSQ_LPBLSIZE, info->chunk_size) | 1415 FIELD_PREP(IRDMAQPSQ_HPAGESIZE, page_size) | 1416 FIELD_PREP(IRDMAQPSQ_STAGRIGHTS, info->access_rights) | 1417 FIELD_PREP(IRDMAQPSQ_VABASEDTO, info->addr_type) | 1418 FIELD_PREP(IRDMAQPSQ_PUSHWQE, (sq_info.push_wqe ? 1 : 0)) | 1419 FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) | 1420 FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) | 1421 FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | 1422 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity); 1423 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 1424 1425 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 1426 1427 irdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, "FAST_REG WQE", wqe, 1428 IRDMA_QP_WQE_MIN_SIZE); 1429 if (sq_info.push_wqe) 1430 irdma_qp_push_wqe(&qp->qp_uk, wqe, quanta, wqe_idx, post_sq); 1431 else if (post_sq) 1432 irdma_uk_qp_post_wr(&qp->qp_uk); 1433 1434 return 0; 1435 } 1436 1437 /** 1438 * irdma_sc_gen_rts_ae - request AE generated after RTS 1439 * @qp: sc qp struct 1440 */ 1441 static void 1442 irdma_sc_gen_rts_ae(struct irdma_sc_qp *qp) 1443 { 1444 __le64 *wqe; 1445 u64 hdr; 1446 struct irdma_qp_uk *qp_uk; 1447 1448 qp_uk = &qp->qp_uk; 1449 1450 wqe = qp_uk->sq_base[1].elem; 1451 1452 hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) | 1453 FIELD_PREP(IRDMAQPSQ_LOCALFENCE, 1) | 1454 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity); 1455 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 1456 1457 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 1458 irdma_debug_buf(qp->dev, IRDMA_DEBUG_QP, "NOP W/LOCAL FENCE WQE", wqe, 1459 IRDMA_QP_WQE_MIN_SIZE); 1460 1461 wqe = qp_uk->sq_base[2].elem; 1462 hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_GEN_RTS_AE) | 1463 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity); 1464 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 1465 1466 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 1467 irdma_debug_buf(qp->dev, IRDMA_DEBUG_QP, "CONN EST WQE", wqe, 1468 IRDMA_QP_WQE_MIN_SIZE); 1469 } 1470 1471 /** 1472 * irdma_sc_send_lsmm - send last streaming mode message 1473 * @qp: sc qp struct 1474 * @lsmm_buf: buffer with lsmm message 1475 * @size: size of lsmm buffer 1476 * @stag: stag of lsmm buffer 1477 */ 1478 void 1479 irdma_sc_send_lsmm(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size, 1480 irdma_stag stag) 1481 { 1482 __le64 *wqe; 1483 u64 hdr; 1484 struct irdma_qp_uk *qp_uk; 1485 1486 qp_uk = &qp->qp_uk; 1487 wqe = qp_uk->sq_base->elem; 1488 1489 set_64bit_val(wqe, IRDMA_BYTE_0, (uintptr_t)lsmm_buf); 1490 if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) { 1491 set_64bit_val(wqe, IRDMA_BYTE_8, 1492 FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, size) | 1493 FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, stag)); 1494 } else { 1495 set_64bit_val(wqe, IRDMA_BYTE_8, 1496 FIELD_PREP(IRDMAQPSQ_FRAG_LEN, size) | 1497 FIELD_PREP(IRDMAQPSQ_FRAG_STAG, stag) | 1498 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity)); 1499 } 1500 set_64bit_val(wqe, IRDMA_BYTE_16, 0); 1501 1502 hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_SEND) | 1503 FIELD_PREP(IRDMAQPSQ_STREAMMODE, 1) | 1504 FIELD_PREP(IRDMAQPSQ_WAITFORRCVPDU, 1) | 1505 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity); 1506 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 1507 1508 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 1509 1510 irdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, "SEND_LSMM WQE", wqe, 1511 IRDMA_QP_WQE_MIN_SIZE); 1512 1513 if (qp->dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE) 1514 irdma_sc_gen_rts_ae(qp); 1515 } 1516 1517 /** 1518 * irdma_sc_send_rtt - send last read0 or write0 1519 * @qp: sc qp struct 1520 * @read: Do read0 or write0 1521 */ 1522 void 1523 irdma_sc_send_rtt(struct irdma_sc_qp *qp, bool read) 1524 { 1525 __le64 *wqe; 1526 u64 hdr; 1527 struct irdma_qp_uk *qp_uk; 1528 1529 qp_uk = &qp->qp_uk; 1530 wqe = qp_uk->sq_base->elem; 1531 1532 set_64bit_val(wqe, IRDMA_BYTE_0, 0); 1533 set_64bit_val(wqe, IRDMA_BYTE_16, 0); 1534 if (read) { 1535 if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) { 1536 set_64bit_val(wqe, IRDMA_BYTE_8, 1537 FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, 0xabcd)); 1538 } else { 1539 set_64bit_val(wqe, IRDMA_BYTE_8, 1540 (u64)0xabcd | FIELD_PREP(IRDMAQPSQ_VALID, 1541 qp->qp_uk.swqe_polarity)); 1542 } 1543 hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, 0x1234) | 1544 FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_READ) | 1545 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity); 1546 1547 } else { 1548 if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) { 1549 set_64bit_val(wqe, IRDMA_BYTE_8, 0); 1550 } else { 1551 set_64bit_val(wqe, IRDMA_BYTE_8, 1552 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity)); 1553 } 1554 hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_WRITE) | 1555 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity); 1556 } 1557 1558 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 1559 1560 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 1561 1562 irdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, "RTR WQE", wqe, 1563 IRDMA_QP_WQE_MIN_SIZE); 1564 1565 if (qp->dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE) 1566 irdma_sc_gen_rts_ae(qp); 1567 } 1568 1569 /** 1570 * irdma_iwarp_opcode - determine if incoming is rdma layer 1571 * @info: aeq info for the packet 1572 * @pkt: packet for error 1573 */ 1574 static u32 irdma_iwarp_opcode(struct irdma_aeqe_info *info, u8 *pkt){ 1575 BE16 *mpa; 1576 u32 opcode = 0xffffffff; 1577 1578 if (info->q2_data_written) { 1579 mpa = (BE16 *) pkt; 1580 opcode = IRDMA_NTOHS(mpa[1]) & 0xf; 1581 } 1582 1583 return opcode; 1584 } 1585 1586 /** 1587 * irdma_locate_mpa - return pointer to mpa in the pkt 1588 * @pkt: packet with data 1589 */ 1590 static u8 *irdma_locate_mpa(u8 *pkt) { 1591 /* skip over ethernet header */ 1592 pkt += IRDMA_MAC_HLEN; 1593 1594 /* Skip over IP and TCP headers */ 1595 pkt += 4 * (pkt[0] & 0x0f); 1596 pkt += 4 * ((pkt[12] >> 4) & 0x0f); 1597 1598 return pkt; 1599 } 1600 1601 /** 1602 * irdma_bld_termhdr_ctrl - setup terminate hdr control fields 1603 * @qp: sc qp ptr for pkt 1604 * @hdr: term hdr 1605 * @opcode: flush opcode for termhdr 1606 * @layer_etype: error layer + error type 1607 * @err: error cod ein the header 1608 */ 1609 static void 1610 irdma_bld_termhdr_ctrl(struct irdma_sc_qp *qp, 1611 struct irdma_terminate_hdr *hdr, 1612 enum irdma_flush_opcode opcode, 1613 u8 layer_etype, u8 err) 1614 { 1615 qp->flush_code = opcode; 1616 hdr->layer_etype = layer_etype; 1617 hdr->error_code = err; 1618 } 1619 1620 /** 1621 * irdma_bld_termhdr_ddp_rdma - setup ddp and rdma hdrs in terminate hdr 1622 * @pkt: ptr to mpa in offending pkt 1623 * @hdr: term hdr 1624 * @copy_len: offending pkt length to be copied to term hdr 1625 * @is_tagged: DDP tagged or untagged 1626 */ 1627 static void 1628 irdma_bld_termhdr_ddp_rdma(u8 *pkt, struct irdma_terminate_hdr *hdr, 1629 int *copy_len, u8 *is_tagged) 1630 { 1631 u16 ddp_seg_len; 1632 1633 ddp_seg_len = IRDMA_NTOHS(*(BE16 *) pkt); 1634 if (ddp_seg_len) { 1635 *copy_len = 2; 1636 hdr->hdrct = DDP_LEN_FLAG; 1637 if (pkt[2] & 0x80) { 1638 *is_tagged = 1; 1639 if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) { 1640 *copy_len += TERM_DDP_LEN_TAGGED; 1641 hdr->hdrct |= DDP_HDR_FLAG; 1642 } 1643 } else { 1644 if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) { 1645 *copy_len += TERM_DDP_LEN_UNTAGGED; 1646 hdr->hdrct |= DDP_HDR_FLAG; 1647 } 1648 if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN) && 1649 ((pkt[3] & RDMA_OPCODE_M) == RDMA_READ_REQ_OPCODE)) { 1650 *copy_len += TERM_RDMA_LEN; 1651 hdr->hdrct |= RDMA_HDR_FLAG; 1652 } 1653 } 1654 } 1655 } 1656 1657 /** 1658 * irdma_bld_terminate_hdr - build terminate message header 1659 * @qp: qp associated with received terminate AE 1660 * @info: the struct contiaing AE information 1661 */ 1662 static int 1663 irdma_bld_terminate_hdr(struct irdma_sc_qp *qp, 1664 struct irdma_aeqe_info *info) 1665 { 1666 u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET; 1667 int copy_len = 0; 1668 u8 is_tagged = 0; 1669 u32 opcode; 1670 struct irdma_terminate_hdr *termhdr; 1671 1672 termhdr = (struct irdma_terminate_hdr *)qp->q2_buf; 1673 memset(termhdr, 0, Q2_BAD_FRAME_OFFSET); 1674 1675 if (info->q2_data_written) { 1676 pkt = irdma_locate_mpa(pkt); 1677 irdma_bld_termhdr_ddp_rdma(pkt, termhdr, ©_len, &is_tagged); 1678 } 1679 1680 opcode = irdma_iwarp_opcode(info, pkt); 1681 qp->event_type = IRDMA_QP_EVENT_CATASTROPHIC; 1682 qp->sq_flush_code = info->sq; 1683 qp->rq_flush_code = info->rq; 1684 1685 switch (info->ae_id) { 1686 case IRDMA_AE_AMP_UNALLOCATED_STAG: 1687 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; 1688 if (opcode == IRDMA_OP_TYPE_RDMA_WRITE) 1689 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_PROT_ERR, 1690 (LAYER_DDP << 4) | DDP_TAGGED_BUF, 1691 DDP_TAGGED_INV_STAG); 1692 else 1693 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR, 1694 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, 1695 RDMAP_INV_STAG); 1696 break; 1697 case IRDMA_AE_AMP_BOUNDS_VIOLATION: 1698 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; 1699 if (info->q2_data_written) 1700 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_PROT_ERR, 1701 (LAYER_DDP << 4) | DDP_TAGGED_BUF, 1702 DDP_TAGGED_BOUNDS); 1703 else 1704 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR, 1705 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, 1706 RDMAP_INV_BOUNDS); 1707 break; 1708 case IRDMA_AE_AMP_BAD_PD: 1709 switch (opcode) { 1710 case IRDMA_OP_TYPE_RDMA_WRITE: 1711 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_PROT_ERR, 1712 (LAYER_DDP << 4) | DDP_TAGGED_BUF, 1713 DDP_TAGGED_UNASSOC_STAG); 1714 break; 1715 case IRDMA_OP_TYPE_SEND_INV: 1716 case IRDMA_OP_TYPE_SEND_SOL_INV: 1717 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR, 1718 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, 1719 RDMAP_CANT_INV_STAG); 1720 break; 1721 default: 1722 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR, 1723 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, 1724 RDMAP_UNASSOC_STAG); 1725 } 1726 break; 1727 case IRDMA_AE_AMP_INVALID_STAG: 1728 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; 1729 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR, 1730 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, 1731 RDMAP_INV_STAG); 1732 break; 1733 case IRDMA_AE_AMP_BAD_QP: 1734 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_LOC_QP_OP_ERR, 1735 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF, 1736 DDP_UNTAGGED_INV_QN); 1737 break; 1738 case IRDMA_AE_AMP_BAD_STAG_KEY: 1739 case IRDMA_AE_AMP_BAD_STAG_INDEX: 1740 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; 1741 switch (opcode) { 1742 case IRDMA_OP_TYPE_SEND_INV: 1743 case IRDMA_OP_TYPE_SEND_SOL_INV: 1744 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_OP_ERR, 1745 (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, 1746 RDMAP_CANT_INV_STAG); 1747 break; 1748 default: 1749 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR, 1750 (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, 1751 RDMAP_INV_STAG); 1752 } 1753 break; 1754 case IRDMA_AE_AMP_RIGHTS_VIOLATION: 1755 case IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS: 1756 case IRDMA_AE_PRIV_OPERATION_DENIED: 1757 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; 1758 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR, 1759 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, 1760 RDMAP_ACCESS); 1761 break; 1762 case IRDMA_AE_AMP_TO_WRAP: 1763 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; 1764 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR, 1765 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, 1766 RDMAP_TO_WRAP); 1767 break; 1768 case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR: 1769 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR, 1770 (LAYER_MPA << 4) | DDP_LLP, MPA_CRC); 1771 break; 1772 case IRDMA_AE_LLP_SEGMENT_TOO_SMALL: 1773 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_LOC_LEN_ERR, 1774 (LAYER_DDP << 4) | DDP_CATASTROPHIC, 1775 DDP_CATASTROPHIC_LOCAL); 1776 break; 1777 case IRDMA_AE_LCE_QP_CATASTROPHIC: 1778 case IRDMA_AE_DDP_NO_L_BIT: 1779 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_FATAL_ERR, 1780 (LAYER_DDP << 4) | DDP_CATASTROPHIC, 1781 DDP_CATASTROPHIC_LOCAL); 1782 break; 1783 case IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN: 1784 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR, 1785 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF, 1786 DDP_UNTAGGED_INV_MSN_RANGE); 1787 break; 1788 case IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER: 1789 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; 1790 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_LOC_LEN_ERR, 1791 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF, 1792 DDP_UNTAGGED_INV_TOO_LONG); 1793 break; 1794 case IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION: 1795 if (is_tagged) 1796 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR, 1797 (LAYER_DDP << 4) | DDP_TAGGED_BUF, 1798 DDP_TAGGED_INV_DDP_VER); 1799 else 1800 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR, 1801 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF, 1802 DDP_UNTAGGED_INV_DDP_VER); 1803 break; 1804 case IRDMA_AE_DDP_UBE_INVALID_MO: 1805 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR, 1806 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF, 1807 DDP_UNTAGGED_INV_MO); 1808 break; 1809 case IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE: 1810 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_OP_ERR, 1811 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF, 1812 DDP_UNTAGGED_INV_MSN_NO_BUF); 1813 break; 1814 case IRDMA_AE_DDP_UBE_INVALID_QN: 1815 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR, 1816 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF, 1817 DDP_UNTAGGED_INV_QN); 1818 break; 1819 case IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION: 1820 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR, 1821 (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, 1822 RDMAP_INV_RDMAP_VER); 1823 break; 1824 default: 1825 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_FATAL_ERR, 1826 (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, 1827 RDMAP_UNSPECIFIED); 1828 break; 1829 } 1830 1831 if (copy_len) 1832 irdma_memcpy(termhdr + 1, pkt, copy_len); 1833 1834 return sizeof(struct irdma_terminate_hdr) + copy_len; 1835 } 1836 1837 /** 1838 * irdma_terminate_send_fin() - Send fin for terminate message 1839 * @qp: qp associated with received terminate AE 1840 */ 1841 void 1842 irdma_terminate_send_fin(struct irdma_sc_qp *qp) 1843 { 1844 irdma_term_modify_qp(qp, IRDMA_QP_STATE_TERMINATE, 1845 IRDMAQP_TERM_SEND_FIN_ONLY, 0); 1846 } 1847 1848 /** 1849 * irdma_terminate_connection() - Bad AE and send terminate to remote QP 1850 * @qp: qp associated with received terminate AE 1851 * @info: the struct contiaing AE information 1852 */ 1853 void 1854 irdma_terminate_connection(struct irdma_sc_qp *qp, 1855 struct irdma_aeqe_info *info) 1856 { 1857 u8 termlen = 0; 1858 1859 if (qp->term_flags & IRDMA_TERM_SENT) 1860 return; 1861 1862 termlen = irdma_bld_terminate_hdr(qp, info); 1863 irdma_terminate_start_timer(qp); 1864 qp->term_flags |= IRDMA_TERM_SENT; 1865 irdma_term_modify_qp(qp, IRDMA_QP_STATE_TERMINATE, 1866 IRDMAQP_TERM_SEND_TERM_ONLY, termlen); 1867 } 1868 1869 /** 1870 * irdma_terminate_received - handle terminate received AE 1871 * @qp: qp associated with received terminate AE 1872 * @info: the struct contiaing AE information 1873 */ 1874 void 1875 irdma_terminate_received(struct irdma_sc_qp *qp, 1876 struct irdma_aeqe_info *info) 1877 { 1878 u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET; 1879 BE32 *mpa; 1880 u8 ddp_ctl; 1881 u8 rdma_ctl; 1882 u16 aeq_id = 0; 1883 struct irdma_terminate_hdr *termhdr; 1884 1885 mpa = (BE32 *) irdma_locate_mpa(pkt); 1886 if (info->q2_data_written) { 1887 /* did not validate the frame - do it now */ 1888 ddp_ctl = (ntohl(mpa[0]) >> 8) & 0xff; 1889 rdma_ctl = ntohl(mpa[0]) & 0xff; 1890 if ((ddp_ctl & 0xc0) != 0x40) 1891 aeq_id = IRDMA_AE_LCE_QP_CATASTROPHIC; 1892 else if ((ddp_ctl & 0x03) != 1) 1893 aeq_id = IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION; 1894 else if (ntohl(mpa[2]) != 2) 1895 aeq_id = IRDMA_AE_DDP_UBE_INVALID_QN; 1896 else if (ntohl(mpa[3]) != 1) 1897 aeq_id = IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN; 1898 else if (ntohl(mpa[4]) != 0) 1899 aeq_id = IRDMA_AE_DDP_UBE_INVALID_MO; 1900 else if ((rdma_ctl & 0xc0) != 0x40) 1901 aeq_id = IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION; 1902 1903 info->ae_id = aeq_id; 1904 if (info->ae_id) { 1905 /* Bad terminate recvd - send back a terminate */ 1906 irdma_terminate_connection(qp, info); 1907 return; 1908 } 1909 } 1910 1911 qp->term_flags |= IRDMA_TERM_RCVD; 1912 qp->event_type = IRDMA_QP_EVENT_CATASTROPHIC; 1913 termhdr = (struct irdma_terminate_hdr *)&mpa[5]; 1914 if (termhdr->layer_etype == RDMAP_REMOTE_PROT || 1915 termhdr->layer_etype == RDMAP_REMOTE_OP) { 1916 irdma_terminate_done(qp, 0); 1917 } else { 1918 irdma_terminate_start_timer(qp); 1919 irdma_terminate_send_fin(qp); 1920 } 1921 } 1922 1923 static int 1924 irdma_null_ws_add(struct irdma_sc_vsi *vsi, u8 user_pri) 1925 { 1926 return 0; 1927 } 1928 1929 static void 1930 irdma_null_ws_remove(struct irdma_sc_vsi *vsi, u8 user_pri) 1931 { 1932 /* do nothing */ 1933 } 1934 1935 static void 1936 irdma_null_ws_reset(struct irdma_sc_vsi *vsi) 1937 { 1938 /* do nothing */ 1939 } 1940 1941 /** 1942 * irdma_sc_vsi_init - Init the vsi structure 1943 * @vsi: pointer to vsi structure to initialize 1944 * @info: the info used to initialize the vsi struct 1945 */ 1946 void 1947 irdma_sc_vsi_init(struct irdma_sc_vsi *vsi, 1948 struct irdma_vsi_init_info *info) 1949 { 1950 u8 i; 1951 1952 vsi->dev = info->dev; 1953 vsi->back_vsi = info->back_vsi; 1954 vsi->register_qset = info->register_qset; 1955 vsi->unregister_qset = info->unregister_qset; 1956 vsi->mtu = info->params->mtu; 1957 vsi->exception_lan_q = info->exception_lan_q; 1958 vsi->vsi_idx = info->pf_data_vsi_num; 1959 1960 irdma_set_qos_info(vsi, info->params); 1961 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) { 1962 mutex_init(&vsi->qos[i].qos_mutex); 1963 INIT_LIST_HEAD(&vsi->qos[i].qplist); 1964 } 1965 if (vsi->register_qset) { 1966 vsi->dev->ws_add = irdma_ws_add; 1967 vsi->dev->ws_remove = irdma_ws_remove; 1968 vsi->dev->ws_reset = irdma_ws_reset; 1969 } else { 1970 vsi->dev->ws_add = irdma_null_ws_add; 1971 vsi->dev->ws_remove = irdma_null_ws_remove; 1972 vsi->dev->ws_reset = irdma_null_ws_reset; 1973 } 1974 } 1975 1976 /** 1977 * irdma_get_stats_idx - Return stats index 1978 * @vsi: pointer to the vsi 1979 */ 1980 static u16 irdma_get_stats_idx(struct irdma_sc_vsi *vsi){ 1981 struct irdma_stats_inst_info stats_info = {0}; 1982 struct irdma_sc_dev *dev = vsi->dev; 1983 1984 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1985 if (!irdma_cqp_stats_inst_cmd(vsi, IRDMA_OP_STATS_ALLOCATE, 1986 &stats_info)) 1987 return stats_info.stats_idx; 1988 } 1989 1990 return IRDMA_INVALID_STATS_IDX; 1991 } 1992 1993 /** 1994 * irdma_vsi_stats_init - Initialize the vsi statistics 1995 * @vsi: pointer to the vsi structure 1996 * @info: The info structure used for initialization 1997 */ 1998 int 1999 irdma_vsi_stats_init(struct irdma_sc_vsi *vsi, 2000 struct irdma_vsi_stats_info *info) 2001 { 2002 struct irdma_dma_mem *stats_buff_mem; 2003 2004 vsi->pestat = info->pestat; 2005 vsi->pestat->hw = vsi->dev->hw; 2006 vsi->pestat->vsi = vsi; 2007 2008 stats_buff_mem = &vsi->pestat->gather_info.stats_buff_mem; 2009 stats_buff_mem->size = IRDMA_GATHER_STATS_BUF_SIZE * 2; 2010 stats_buff_mem->va = irdma_allocate_dma_mem(vsi->pestat->hw, 2011 stats_buff_mem, 2012 stats_buff_mem->size, 1); 2013 if (!stats_buff_mem->va) 2014 return -ENOMEM; 2015 2016 vsi->pestat->gather_info.gather_stats_va = stats_buff_mem->va; 2017 vsi->pestat->gather_info.last_gather_stats_va = 2018 (void *)((uintptr_t)stats_buff_mem->va + 2019 IRDMA_GATHER_STATS_BUF_SIZE); 2020 2021 irdma_hw_stats_start_timer(vsi); 2022 2023 /* when stat allocation is not required default to fcn_id. */ 2024 vsi->stats_idx = info->fcn_id; 2025 if (info->alloc_stats_inst) { 2026 u16 stats_idx = irdma_get_stats_idx(vsi); 2027 2028 if (stats_idx != IRDMA_INVALID_STATS_IDX) { 2029 vsi->stats_inst_alloc = true; 2030 vsi->stats_idx = stats_idx; 2031 vsi->pestat->gather_info.use_stats_inst = true; 2032 vsi->pestat->gather_info.stats_inst_index = stats_idx; 2033 } 2034 } 2035 2036 return 0; 2037 } 2038 2039 /** 2040 * irdma_vsi_stats_free - Free the vsi stats 2041 * @vsi: pointer to the vsi structure 2042 */ 2043 void 2044 irdma_vsi_stats_free(struct irdma_sc_vsi *vsi) 2045 { 2046 struct irdma_stats_inst_info stats_info = {0}; 2047 struct irdma_sc_dev *dev = vsi->dev; 2048 2049 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 2050 if (vsi->stats_inst_alloc) { 2051 stats_info.stats_idx = vsi->stats_idx; 2052 irdma_cqp_stats_inst_cmd(vsi, IRDMA_OP_STATS_FREE, 2053 &stats_info); 2054 } 2055 } 2056 2057 if (!vsi->pestat) 2058 return; 2059 2060 irdma_hw_stats_stop_timer(vsi); 2061 irdma_free_dma_mem(vsi->pestat->hw, 2062 &vsi->pestat->gather_info.stats_buff_mem); 2063 } 2064 2065 /** 2066 * irdma_get_encoded_wqe_size - given wq size, returns hardware encoded size 2067 * @wqsize: size of the wq (sq, rq) to encoded_size 2068 * @queue_type: queue type selected for the calculation algorithm 2069 */ 2070 u8 2071 irdma_get_encoded_wqe_size(u32 wqsize, enum irdma_queue_type queue_type) 2072 { 2073 u8 encoded_size = 0; 2074 2075 /* 2076 * cqp sq's hw coded value starts from 1 for size of 4 while it starts from 0 for qp' wq's. 2077 */ 2078 if (queue_type == IRDMA_QUEUE_TYPE_CQP) 2079 encoded_size = 1; 2080 wqsize >>= 2; 2081 while (wqsize >>= 1) 2082 encoded_size++; 2083 2084 return encoded_size; 2085 } 2086 2087 /** 2088 * irdma_sc_gather_stats - collect the statistics 2089 * @cqp: struct for cqp hw 2090 * @info: gather stats info structure 2091 * @scratch: u64 saved to be used during cqp completion 2092 */ 2093 static int 2094 irdma_sc_gather_stats(struct irdma_sc_cqp *cqp, 2095 struct irdma_stats_gather_info *info, 2096 u64 scratch) 2097 { 2098 __le64 *wqe; 2099 u64 temp; 2100 2101 if (info->stats_buff_mem.size < IRDMA_GATHER_STATS_BUF_SIZE) 2102 return -ENOSPC; 2103 2104 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 2105 if (!wqe) 2106 return -ENOSPC; 2107 2108 set_64bit_val(wqe, IRDMA_BYTE_40, 2109 FIELD_PREP(IRDMA_CQPSQ_STATS_HMC_FCN_INDEX, info->hmc_fcn_index)); 2110 set_64bit_val(wqe, IRDMA_BYTE_32, info->stats_buff_mem.pa); 2111 2112 temp = FIELD_PREP(IRDMA_CQPSQ_STATS_WQEVALID, cqp->polarity) | 2113 FIELD_PREP(IRDMA_CQPSQ_STATS_USE_INST, info->use_stats_inst) | 2114 FIELD_PREP(IRDMA_CQPSQ_STATS_INST_INDEX, 2115 info->stats_inst_index) | 2116 FIELD_PREP(IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX, 2117 info->use_hmc_fcn_index) | 2118 FIELD_PREP(IRDMA_CQPSQ_STATS_OP, IRDMA_CQP_OP_GATHER_STATS); 2119 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 2120 2121 set_64bit_val(wqe, IRDMA_BYTE_24, temp); 2122 2123 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_STATS, "GATHER_STATS WQE", wqe, 2124 IRDMA_CQP_WQE_SIZE * 8); 2125 2126 irdma_sc_cqp_post_sq(cqp); 2127 irdma_debug(cqp->dev, IRDMA_DEBUG_STATS, 2128 "CQP SQ head 0x%x tail 0x%x size 0x%x\n", cqp->sq_ring.head, 2129 cqp->sq_ring.tail, cqp->sq_ring.size); 2130 2131 return 0; 2132 } 2133 2134 /** 2135 * irdma_sc_manage_stats_inst - allocate or free stats instance 2136 * @cqp: struct for cqp hw 2137 * @info: stats info structure 2138 * @alloc: alloc vs. delete flag 2139 * @scratch: u64 saved to be used during cqp completion 2140 */ 2141 static int 2142 irdma_sc_manage_stats_inst(struct irdma_sc_cqp *cqp, 2143 struct irdma_stats_inst_info *info, 2144 bool alloc, u64 scratch) 2145 { 2146 __le64 *wqe; 2147 u64 temp; 2148 2149 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 2150 if (!wqe) 2151 return -ENOSPC; 2152 2153 set_64bit_val(wqe, IRDMA_BYTE_40, 2154 FIELD_PREP(IRDMA_CQPSQ_STATS_HMC_FCN_INDEX, info->hmc_fn_id)); 2155 temp = FIELD_PREP(IRDMA_CQPSQ_STATS_WQEVALID, cqp->polarity) | 2156 FIELD_PREP(IRDMA_CQPSQ_STATS_ALLOC_INST, alloc) | 2157 FIELD_PREP(IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX, 2158 info->use_hmc_fcn_index) | 2159 FIELD_PREP(IRDMA_CQPSQ_STATS_INST_INDEX, info->stats_idx) | 2160 FIELD_PREP(IRDMA_CQPSQ_STATS_OP, IRDMA_CQP_OP_MANAGE_STATS); 2161 2162 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 2163 2164 set_64bit_val(wqe, IRDMA_BYTE_24, temp); 2165 2166 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "MANAGE_STATS WQE", wqe, 2167 IRDMA_CQP_WQE_SIZE * 8); 2168 2169 irdma_sc_cqp_post_sq(cqp); 2170 return 0; 2171 } 2172 2173 /** 2174 * irdma_sc_set_up_map - set the up map table 2175 * @cqp: struct for cqp hw 2176 * @info: User priority map info 2177 * @scratch: u64 saved to be used during cqp completion 2178 */ 2179 static int 2180 irdma_sc_set_up_map(struct irdma_sc_cqp *cqp, 2181 struct irdma_up_info *info, u64 scratch) 2182 { 2183 __le64 *wqe; 2184 u64 temp = 0; 2185 int i; 2186 2187 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 2188 if (!wqe) 2189 return -ENOSPC; 2190 2191 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) 2192 temp |= (u64)info->map[i] << (i * 8); 2193 2194 set_64bit_val(wqe, IRDMA_BYTE_0, temp); 2195 set_64bit_val(wqe, IRDMA_BYTE_40, 2196 FIELD_PREP(IRDMA_CQPSQ_UP_CNPOVERRIDE, info->cnp_up_override) | 2197 FIELD_PREP(IRDMA_CQPSQ_UP_HMCFCNIDX, info->hmc_fcn_idx)); 2198 2199 temp = FIELD_PREP(IRDMA_CQPSQ_UP_WQEVALID, cqp->polarity) | 2200 FIELD_PREP(IRDMA_CQPSQ_UP_USEVLAN, info->use_vlan) | 2201 FIELD_PREP(IRDMA_CQPSQ_UP_USEOVERRIDE, 2202 info->use_cnp_up_override) | 2203 FIELD_PREP(IRDMA_CQPSQ_UP_OP, IRDMA_CQP_OP_UP_MAP); 2204 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 2205 2206 set_64bit_val(wqe, IRDMA_BYTE_24, temp); 2207 2208 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "UPMAP WQE", wqe, 2209 IRDMA_CQP_WQE_SIZE * 8); 2210 irdma_sc_cqp_post_sq(cqp); 2211 2212 return 0; 2213 } 2214 2215 /** 2216 * irdma_sc_manage_ws_node - create/modify/destroy WS node 2217 * @cqp: struct for cqp hw 2218 * @info: node info structure 2219 * @node_op: 0 for add 1 for modify, 2 for delete 2220 * @scratch: u64 saved to be used during cqp completion 2221 */ 2222 static int 2223 irdma_sc_manage_ws_node(struct irdma_sc_cqp *cqp, 2224 struct irdma_ws_node_info *info, 2225 enum irdma_ws_node_op node_op, u64 scratch) 2226 { 2227 __le64 *wqe; 2228 u64 temp = 0; 2229 2230 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 2231 if (!wqe) 2232 return -ENOSPC; 2233 2234 set_64bit_val(wqe, IRDMA_BYTE_32, 2235 FIELD_PREP(IRDMA_CQPSQ_WS_VSI, info->vsi) | 2236 FIELD_PREP(IRDMA_CQPSQ_WS_WEIGHT, info->weight)); 2237 2238 temp = FIELD_PREP(IRDMA_CQPSQ_WS_WQEVALID, cqp->polarity) | 2239 FIELD_PREP(IRDMA_CQPSQ_WS_NODEOP, node_op) | 2240 FIELD_PREP(IRDMA_CQPSQ_WS_ENABLENODE, info->enable) | 2241 FIELD_PREP(IRDMA_CQPSQ_WS_NODETYPE, info->type_leaf) | 2242 FIELD_PREP(IRDMA_CQPSQ_WS_PRIOTYPE, info->prio_type) | 2243 FIELD_PREP(IRDMA_CQPSQ_WS_TC, info->tc) | 2244 FIELD_PREP(IRDMA_CQPSQ_WS_OP, IRDMA_CQP_OP_WORK_SCHED_NODE) | 2245 FIELD_PREP(IRDMA_CQPSQ_WS_PARENTID, info->parent_id) | 2246 FIELD_PREP(IRDMA_CQPSQ_WS_NODEID, info->id); 2247 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 2248 2249 set_64bit_val(wqe, IRDMA_BYTE_24, temp); 2250 2251 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "MANAGE_WS WQE", wqe, 2252 IRDMA_CQP_WQE_SIZE * 8); 2253 irdma_sc_cqp_post_sq(cqp); 2254 2255 return 0; 2256 } 2257 2258 /** 2259 * irdma_sc_qp_flush_wqes - flush qp's wqe 2260 * @qp: sc qp 2261 * @info: dlush information 2262 * @scratch: u64 saved to be used during cqp completion 2263 * @post_sq: flag for cqp db to ring 2264 */ 2265 int 2266 irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp, 2267 struct irdma_qp_flush_info *info, u64 scratch, 2268 bool post_sq) 2269 { 2270 u64 temp = 0; 2271 __le64 *wqe; 2272 struct irdma_sc_cqp *cqp; 2273 u64 hdr; 2274 bool flush_sq = false, flush_rq = false; 2275 2276 if (info->rq && !qp->flush_rq) 2277 flush_rq = true; 2278 if (info->sq && !qp->flush_sq) 2279 flush_sq = true; 2280 qp->flush_sq |= flush_sq; 2281 qp->flush_rq |= flush_rq; 2282 2283 if (!flush_sq && !flush_rq) { 2284 irdma_debug(qp->dev, IRDMA_DEBUG_CQP, 2285 "Additional flush request ignored for qp %x\n", 2286 qp->qp_uk.qp_id); 2287 return -EALREADY; 2288 } 2289 2290 cqp = qp->pd->dev->cqp; 2291 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 2292 if (!wqe) 2293 return -ENOSPC; 2294 2295 if (info->userflushcode) { 2296 if (flush_rq) 2297 temp |= FIELD_PREP(IRDMA_CQPSQ_FWQE_RQMNERR, 2298 info->rq_minor_code) | 2299 FIELD_PREP(IRDMA_CQPSQ_FWQE_RQMJERR, 2300 info->rq_major_code); 2301 if (flush_sq) 2302 temp |= FIELD_PREP(IRDMA_CQPSQ_FWQE_SQMNERR, 2303 info->sq_minor_code) | 2304 FIELD_PREP(IRDMA_CQPSQ_FWQE_SQMJERR, 2305 info->sq_major_code); 2306 } 2307 set_64bit_val(wqe, IRDMA_BYTE_16, temp); 2308 2309 temp = (info->generate_ae) ? 2310 info->ae_code | FIELD_PREP(IRDMA_CQPSQ_FWQE_AESOURCE, 2311 info->ae_src) : 0; 2312 set_64bit_val(wqe, IRDMA_BYTE_8, temp); 2313 hdr = qp->qp_uk.qp_id | 2314 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_FLUSH_WQES) | 2315 FIELD_PREP(IRDMA_CQPSQ_FWQE_GENERATE_AE, info->generate_ae) | 2316 FIELD_PREP(IRDMA_CQPSQ_FWQE_USERFLCODE, info->userflushcode) | 2317 FIELD_PREP(IRDMA_CQPSQ_FWQE_FLUSHSQ, flush_sq) | 2318 FIELD_PREP(IRDMA_CQPSQ_FWQE_FLUSHRQ, flush_rq) | 2319 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 2320 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 2321 2322 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 2323 2324 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "QP_FLUSH WQE", wqe, 2325 IRDMA_CQP_WQE_SIZE * 8); 2326 if (post_sq) 2327 irdma_sc_cqp_post_sq(cqp); 2328 2329 return 0; 2330 } 2331 2332 /** 2333 * irdma_sc_gen_ae - generate AE, uses flush WQE CQP OP 2334 * @qp: sc qp 2335 * @info: gen ae information 2336 * @scratch: u64 saved to be used during cqp completion 2337 * @post_sq: flag for cqp db to ring 2338 */ 2339 static int 2340 irdma_sc_gen_ae(struct irdma_sc_qp *qp, 2341 struct irdma_gen_ae_info *info, u64 scratch, 2342 bool post_sq) 2343 { 2344 u64 temp; 2345 __le64 *wqe; 2346 struct irdma_sc_cqp *cqp; 2347 u64 hdr; 2348 2349 cqp = qp->pd->dev->cqp; 2350 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 2351 if (!wqe) 2352 return -ENOSPC; 2353 2354 temp = info->ae_code | FIELD_PREP(IRDMA_CQPSQ_FWQE_AESOURCE, 2355 info->ae_src); 2356 set_64bit_val(wqe, IRDMA_BYTE_8, temp); 2357 2358 hdr = qp->qp_uk.qp_id | FIELD_PREP(IRDMA_CQPSQ_OPCODE, 2359 IRDMA_CQP_OP_GEN_AE) | 2360 FIELD_PREP(IRDMA_CQPSQ_FWQE_GENERATE_AE, 1) | 2361 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 2362 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 2363 2364 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 2365 2366 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "GEN_AE WQE", wqe, 2367 IRDMA_CQP_WQE_SIZE * 8); 2368 if (post_sq) 2369 irdma_sc_cqp_post_sq(cqp); 2370 2371 return 0; 2372 } 2373 2374 /*** irdma_sc_qp_upload_context - upload qp's context 2375 * @dev: sc device struct 2376 * @info: upload context info ptr for return 2377 * @scratch: u64 saved to be used during cqp completion 2378 * @post_sq: flag for cqp db to ring 2379 */ 2380 static int 2381 irdma_sc_qp_upload_context(struct irdma_sc_dev *dev, 2382 struct irdma_upload_context_info *info, 2383 u64 scratch, bool post_sq) 2384 { 2385 __le64 *wqe; 2386 struct irdma_sc_cqp *cqp; 2387 u64 hdr; 2388 2389 cqp = dev->cqp; 2390 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 2391 if (!wqe) 2392 return -ENOSPC; 2393 2394 set_64bit_val(wqe, IRDMA_BYTE_16, info->buf_pa); 2395 2396 hdr = FIELD_PREP(IRDMA_CQPSQ_UCTX_QPID, info->qp_id) | 2397 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_UPLOAD_CONTEXT) | 2398 FIELD_PREP(IRDMA_CQPSQ_UCTX_QPTYPE, info->qp_type) | 2399 FIELD_PREP(IRDMA_CQPSQ_UCTX_RAWFORMAT, info->raw_format) | 2400 FIELD_PREP(IRDMA_CQPSQ_UCTX_FREEZEQP, info->freeze_qp) | 2401 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 2402 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 2403 2404 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 2405 2406 irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "QP_UPLOAD_CTX WQE", wqe, 2407 IRDMA_CQP_WQE_SIZE * 8); 2408 if (post_sq) 2409 irdma_sc_cqp_post_sq(cqp); 2410 2411 return 0; 2412 } 2413 2414 /** 2415 * irdma_sc_manage_push_page - Handle push page 2416 * @cqp: struct for cqp hw 2417 * @info: push page info 2418 * @scratch: u64 saved to be used during cqp completion 2419 * @post_sq: flag for cqp db to ring 2420 */ 2421 static int 2422 irdma_sc_manage_push_page(struct irdma_sc_cqp *cqp, 2423 struct irdma_cqp_manage_push_page_info *info, 2424 u64 scratch, bool post_sq) 2425 { 2426 __le64 *wqe; 2427 u64 hdr; 2428 2429 if (info->free_page && 2430 info->push_idx >= cqp->dev->hw_attrs.max_hw_device_pages) 2431 return -EINVAL; 2432 2433 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 2434 if (!wqe) 2435 return -ENOSPC; 2436 2437 set_64bit_val(wqe, IRDMA_BYTE_16, info->qs_handle); 2438 hdr = FIELD_PREP(IRDMA_CQPSQ_MPP_PPIDX, info->push_idx) | 2439 FIELD_PREP(IRDMA_CQPSQ_MPP_PPTYPE, info->push_page_type) | 2440 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_PUSH_PAGES) | 2441 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) | 2442 FIELD_PREP(IRDMA_CQPSQ_MPP_FREE_PAGE, info->free_page); 2443 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 2444 2445 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 2446 2447 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "MANAGE_PUSH_PAGES WQE", wqe, 2448 IRDMA_CQP_WQE_SIZE * 8); 2449 if (post_sq) 2450 irdma_sc_cqp_post_sq(cqp); 2451 2452 return 0; 2453 } 2454 2455 /** 2456 * irdma_sc_suspend_qp - suspend qp for param change 2457 * @cqp: struct for cqp hw 2458 * @qp: sc qp struct 2459 * @scratch: u64 saved to be used during cqp completion 2460 */ 2461 static int 2462 irdma_sc_suspend_qp(struct irdma_sc_cqp *cqp, struct irdma_sc_qp *qp, 2463 u64 scratch) 2464 { 2465 u64 hdr; 2466 __le64 *wqe; 2467 2468 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 2469 if (!wqe) 2470 return -ENOSPC; 2471 2472 hdr = FIELD_PREP(IRDMA_CQPSQ_SUSPENDQP_QPID, qp->qp_uk.qp_id) | 2473 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_SUSPEND_QP) | 2474 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 2475 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 2476 2477 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 2478 2479 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "SUSPEND_QP WQE", wqe, 2480 IRDMA_CQP_WQE_SIZE * 8); 2481 irdma_sc_cqp_post_sq(cqp); 2482 2483 return 0; 2484 } 2485 2486 /** 2487 * irdma_sc_resume_qp - resume qp after suspend 2488 * @cqp: struct for cqp hw 2489 * @qp: sc qp struct 2490 * @scratch: u64 saved to be used during cqp completion 2491 */ 2492 static int 2493 irdma_sc_resume_qp(struct irdma_sc_cqp *cqp, struct irdma_sc_qp *qp, 2494 u64 scratch) 2495 { 2496 u64 hdr; 2497 __le64 *wqe; 2498 2499 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 2500 if (!wqe) 2501 return -ENOSPC; 2502 2503 set_64bit_val(wqe, IRDMA_BYTE_16, 2504 FIELD_PREP(IRDMA_CQPSQ_RESUMEQP_QSHANDLE, qp->qs_handle)); 2505 2506 hdr = FIELD_PREP(IRDMA_CQPSQ_RESUMEQP_QPID, qp->qp_uk.qp_id) | 2507 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_RESUME_QP) | 2508 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 2509 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 2510 2511 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 2512 2513 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "RESUME_QP WQE", wqe, 2514 IRDMA_CQP_WQE_SIZE * 8); 2515 irdma_sc_cqp_post_sq(cqp); 2516 2517 return 0; 2518 } 2519 2520 /** 2521 * irdma_sc_cq_ack - acknowledge completion q 2522 * @cq: cq struct 2523 */ 2524 static inline void 2525 irdma_sc_cq_ack(struct irdma_sc_cq *cq) 2526 { 2527 db_wr32(cq->cq_uk.cq_id, cq->cq_uk.cq_ack_db); 2528 } 2529 2530 /** 2531 * irdma_sc_cq_init - initialize completion q 2532 * @cq: cq struct 2533 * @info: cq initialization info 2534 */ 2535 int 2536 irdma_sc_cq_init(struct irdma_sc_cq *cq, struct irdma_cq_init_info *info) 2537 { 2538 int ret_code; 2539 u32 pble_obj_cnt; 2540 2541 pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; 2542 if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt) 2543 return -EINVAL; 2544 2545 cq->cq_pa = info->cq_base_pa; 2546 cq->dev = info->dev; 2547 cq->ceq_id = info->ceq_id; 2548 info->cq_uk_init_info.cqe_alloc_db = cq->dev->cq_arm_db; 2549 info->cq_uk_init_info.cq_ack_db = cq->dev->cq_ack_db; 2550 ret_code = irdma_uk_cq_init(&cq->cq_uk, &info->cq_uk_init_info); 2551 if (ret_code) 2552 return ret_code; 2553 2554 cq->virtual_map = info->virtual_map; 2555 cq->pbl_chunk_size = info->pbl_chunk_size; 2556 cq->ceqe_mask = info->ceqe_mask; 2557 cq->cq_type = (info->type) ? info->type : IRDMA_CQ_TYPE_IWARP; 2558 cq->shadow_area_pa = info->shadow_area_pa; 2559 cq->shadow_read_threshold = info->shadow_read_threshold; 2560 cq->ceq_id_valid = info->ceq_id_valid; 2561 cq->tph_en = info->tph_en; 2562 cq->tph_val = info->tph_val; 2563 cq->first_pm_pbl_idx = info->first_pm_pbl_idx; 2564 cq->vsi = info->vsi; 2565 2566 return 0; 2567 } 2568 2569 /** 2570 * irdma_sc_cq_create - create completion q 2571 * @cq: cq struct 2572 * @scratch: u64 saved to be used during cqp completion 2573 * @check_overflow: flag for overflow check 2574 * @post_sq: flag for cqp db to ring 2575 */ 2576 static int 2577 irdma_sc_cq_create(struct irdma_sc_cq *cq, u64 scratch, 2578 bool check_overflow, bool post_sq) 2579 { 2580 __le64 *wqe; 2581 struct irdma_sc_cqp *cqp; 2582 u64 hdr; 2583 struct irdma_sc_ceq *ceq; 2584 int ret_code = 0; 2585 2586 cqp = cq->dev->cqp; 2587 if (cq->cq_uk.cq_id > (cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].max_cnt - 1)) 2588 return -EINVAL; 2589 2590 if (cq->ceq_id > (cq->dev->hmc_fpm_misc.max_ceqs - 1)) 2591 return -EINVAL; 2592 2593 ceq = cq->dev->ceq[cq->ceq_id]; 2594 if (ceq && ceq->reg_cq) { 2595 ret_code = irdma_sc_add_cq_ctx(ceq, cq); 2596 if (ret_code) 2597 return ret_code; 2598 } 2599 2600 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 2601 if (!wqe) { 2602 if (ceq && ceq->reg_cq) 2603 irdma_sc_remove_cq_ctx(ceq, cq); 2604 return -ENOSPC; 2605 } 2606 2607 set_64bit_val(wqe, IRDMA_BYTE_0, cq->cq_uk.cq_size); 2608 set_64bit_val(wqe, IRDMA_BYTE_8, RS_64_1(cq, 1)); 2609 set_64bit_val(wqe, IRDMA_BYTE_16, 2610 FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, cq->shadow_read_threshold)); 2611 set_64bit_val(wqe, IRDMA_BYTE_32, (cq->virtual_map ? 0 : cq->cq_pa)); 2612 set_64bit_val(wqe, IRDMA_BYTE_40, cq->shadow_area_pa); 2613 set_64bit_val(wqe, IRDMA_BYTE_48, 2614 FIELD_PREP(IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX, (cq->virtual_map ? cq->first_pm_pbl_idx : 0))); 2615 set_64bit_val(wqe, IRDMA_BYTE_56, 2616 FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq->tph_val) | 2617 FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq->vsi->vsi_idx)); 2618 2619 hdr = FLD_LS_64(cq->dev, cq->cq_uk.cq_id, IRDMA_CQPSQ_CQ_CQID) | 2620 FLD_LS_64(cq->dev, (cq->ceq_id_valid ? cq->ceq_id : 0), 2621 IRDMA_CQPSQ_CQ_CEQID) | 2622 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_CQ) | 2623 FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, cq->pbl_chunk_size) | 2624 FIELD_PREP(IRDMA_CQPSQ_CQ_CHKOVERFLOW, check_overflow) | 2625 FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, cq->virtual_map) | 2626 FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) | 2627 FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, cq->ceq_id_valid) | 2628 FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) | 2629 FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT, 2630 cq->cq_uk.avoid_mem_cflct) | 2631 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 2632 2633 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 2634 2635 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 2636 2637 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "CQ_CREATE WQE", wqe, 2638 IRDMA_CQP_WQE_SIZE * 8); 2639 if (post_sq) 2640 irdma_sc_cqp_post_sq(cqp); 2641 2642 return 0; 2643 } 2644 2645 /** 2646 * irdma_sc_cq_destroy - destroy completion q 2647 * @cq: cq struct 2648 * @scratch: u64 saved to be used during cqp completion 2649 * @post_sq: flag for cqp db to ring 2650 */ 2651 int 2652 irdma_sc_cq_destroy(struct irdma_sc_cq *cq, u64 scratch, bool post_sq) 2653 { 2654 struct irdma_sc_cqp *cqp; 2655 __le64 *wqe; 2656 u64 hdr; 2657 struct irdma_sc_ceq *ceq; 2658 2659 cqp = cq->dev->cqp; 2660 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 2661 if (!wqe) 2662 return -ENOSPC; 2663 2664 ceq = cq->dev->ceq[cq->ceq_id]; 2665 if (ceq && ceq->reg_cq) 2666 irdma_sc_remove_cq_ctx(ceq, cq); 2667 2668 set_64bit_val(wqe, IRDMA_BYTE_0, cq->cq_uk.cq_size); 2669 set_64bit_val(wqe, IRDMA_BYTE_8, RS_64_1(cq, 1)); 2670 set_64bit_val(wqe, IRDMA_BYTE_40, cq->shadow_area_pa); 2671 set_64bit_val(wqe, IRDMA_BYTE_48, 2672 (cq->virtual_map ? cq->first_pm_pbl_idx : 0)); 2673 2674 hdr = cq->cq_uk.cq_id | 2675 FLD_LS_64(cq->dev, (cq->ceq_id_valid ? cq->ceq_id : 0), 2676 IRDMA_CQPSQ_CQ_CEQID) | 2677 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_CQ) | 2678 FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, cq->pbl_chunk_size) | 2679 FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, cq->virtual_map) | 2680 FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) | 2681 FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, cq->ceq_id_valid) | 2682 FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) | 2683 FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT, cq->cq_uk.avoid_mem_cflct) | 2684 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 2685 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 2686 2687 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 2688 2689 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "CQ_DESTROY WQE", wqe, 2690 IRDMA_CQP_WQE_SIZE * 8); 2691 if (post_sq) 2692 irdma_sc_cqp_post_sq(cqp); 2693 2694 return 0; 2695 } 2696 2697 /** 2698 * irdma_sc_cq_resize - set resized cq buffer info 2699 * @cq: resized cq 2700 * @info: resized cq buffer info 2701 */ 2702 void 2703 irdma_sc_cq_resize(struct irdma_sc_cq *cq, struct irdma_modify_cq_info *info) 2704 { 2705 cq->virtual_map = info->virtual_map; 2706 cq->cq_pa = info->cq_pa; 2707 cq->first_pm_pbl_idx = info->first_pm_pbl_idx; 2708 cq->pbl_chunk_size = info->pbl_chunk_size; 2709 irdma_uk_cq_resize(&cq->cq_uk, info->cq_base, info->cq_size); 2710 } 2711 2712 /** 2713 * irdma_sc_cq_modify - modify a Completion Queue 2714 * @cq: cq struct 2715 * @info: modification info struct 2716 * @scratch: u64 saved to be used during cqp completion 2717 * @post_sq: flag to post to sq 2718 */ 2719 static int 2720 irdma_sc_cq_modify(struct irdma_sc_cq *cq, 2721 struct irdma_modify_cq_info *info, u64 scratch, 2722 bool post_sq) 2723 { 2724 struct irdma_sc_cqp *cqp; 2725 __le64 *wqe; 2726 u64 hdr; 2727 u32 pble_obj_cnt; 2728 2729 pble_obj_cnt = cq->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; 2730 if (info->cq_resize && info->virtual_map && 2731 info->first_pm_pbl_idx >= pble_obj_cnt) 2732 return -EINVAL; 2733 2734 cqp = cq->dev->cqp; 2735 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 2736 if (!wqe) 2737 return -ENOSPC; 2738 2739 set_64bit_val(wqe, IRDMA_BYTE_0, info->cq_size); 2740 set_64bit_val(wqe, IRDMA_BYTE_8, RS_64_1(cq, 1)); 2741 set_64bit_val(wqe, IRDMA_BYTE_16, 2742 FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, info->shadow_read_threshold)); 2743 set_64bit_val(wqe, IRDMA_BYTE_32, info->cq_pa); 2744 set_64bit_val(wqe, IRDMA_BYTE_40, cq->shadow_area_pa); 2745 set_64bit_val(wqe, IRDMA_BYTE_48, info->first_pm_pbl_idx); 2746 set_64bit_val(wqe, IRDMA_BYTE_56, 2747 FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq->tph_val) | 2748 FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq->vsi->vsi_idx)); 2749 2750 hdr = cq->cq_uk.cq_id | 2751 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MODIFY_CQ) | 2752 FIELD_PREP(IRDMA_CQPSQ_CQ_CQRESIZE, info->cq_resize) | 2753 FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, info->pbl_chunk_size) | 2754 FIELD_PREP(IRDMA_CQPSQ_CQ_CHKOVERFLOW, info->check_overflow) | 2755 FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, info->virtual_map) | 2756 FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) | 2757 FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) | 2758 FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT, 2759 cq->cq_uk.avoid_mem_cflct) | 2760 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 2761 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 2762 2763 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 2764 2765 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "CQ_MODIFY WQE", wqe, 2766 IRDMA_CQP_WQE_SIZE * 8); 2767 if (post_sq) 2768 irdma_sc_cqp_post_sq(cqp); 2769 2770 return 0; 2771 } 2772 2773 /** 2774 * irdma_check_cqp_progress - check cqp processing progress 2775 * @timeout: timeout info struct 2776 * @dev: sc device struct 2777 */ 2778 void 2779 irdma_check_cqp_progress(struct irdma_cqp_timeout *timeout, 2780 struct irdma_sc_dev *dev) 2781 { 2782 u64 completed_ops = atomic64_read(&dev->cqp->completed_ops); 2783 2784 if (timeout->compl_cqp_cmds != completed_ops) { 2785 timeout->compl_cqp_cmds = completed_ops; 2786 timeout->count = 0; 2787 } else if (timeout->compl_cqp_cmds != dev->cqp->requested_ops) { 2788 timeout->count++; 2789 } 2790 } 2791 2792 /** 2793 * irdma_get_cqp_reg_info - get head and tail for cqp using registers 2794 * @cqp: struct for cqp hw 2795 * @val: cqp tail register value 2796 * @tail: wqtail register value 2797 * @error: cqp processing err 2798 */ 2799 static inline void 2800 irdma_get_cqp_reg_info(struct irdma_sc_cqp *cqp, u32 *val, 2801 u32 *tail, u32 *error) 2802 { 2803 *val = readl(cqp->dev->hw_regs[IRDMA_CQPTAIL]); 2804 *tail = FIELD_GET(IRDMA_CQPTAIL_WQTAIL, *val); 2805 *error = FIELD_GET(IRDMA_CQPTAIL_CQP_OP_ERR, *val); 2806 } 2807 2808 /** 2809 * irdma_cqp_poll_registers - poll cqp registers 2810 * @cqp: struct for cqp hw 2811 * @tail: wqtail register value 2812 * @count: how many times to try for completion 2813 */ 2814 static int 2815 irdma_cqp_poll_registers(struct irdma_sc_cqp *cqp, u32 tail, 2816 u32 count) 2817 { 2818 u32 i = 0; 2819 u32 newtail, error, val; 2820 2821 while (i++ < count) { 2822 irdma_get_cqp_reg_info(cqp, &val, &newtail, &error); 2823 if (error) { 2824 error = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]); 2825 irdma_debug(cqp->dev, IRDMA_DEBUG_CQP, 2826 "CQPERRCODES error_code[x%08X]\n", error); 2827 return -EIO; 2828 } 2829 if (newtail != tail) { 2830 /* SUCCESS */ 2831 IRDMA_RING_MOVE_TAIL(cqp->sq_ring); 2832 atomic64_inc(&cqp->completed_ops); 2833 return 0; 2834 } 2835 irdma_usec_delay(cqp->dev->hw_attrs.max_sleep_count); 2836 } 2837 2838 return -ETIMEDOUT; 2839 } 2840 2841 /** 2842 * irdma_sc_decode_fpm_commit - decode a 64 bit value into count and base 2843 * @dev: sc device struct 2844 * @buf: pointer to commit buffer 2845 * @buf_idx: buffer index 2846 * @obj_info: object info pointer 2847 * @rsrc_idx: indexs of memory resource 2848 */ 2849 static u64 irdma_sc_decode_fpm_commit(struct irdma_sc_dev *dev, __le64 * buf, 2850 u32 buf_idx, struct irdma_hmc_obj_info *obj_info, 2851 u32 rsrc_idx){ 2852 u64 temp; 2853 2854 get_64bit_val(buf, buf_idx, &temp); 2855 2856 switch (rsrc_idx) { 2857 case IRDMA_HMC_IW_QP: 2858 obj_info[rsrc_idx].cnt = (u32)FIELD_GET(IRDMA_COMMIT_FPM_QPCNT, temp); 2859 break; 2860 case IRDMA_HMC_IW_CQ: 2861 obj_info[rsrc_idx].cnt = (u32)FLD_RS_64(dev, temp, IRDMA_COMMIT_FPM_CQCNT); 2862 break; 2863 case IRDMA_HMC_IW_APBVT_ENTRY: 2864 if (dev->hw_attrs.uk_attrs.hw_rev <= IRDMA_GEN_2) 2865 obj_info[rsrc_idx].cnt = 1; 2866 else 2867 obj_info[rsrc_idx].cnt = 0; 2868 break; 2869 default: 2870 obj_info[rsrc_idx].cnt = (u32)temp; 2871 break; 2872 } 2873 2874 obj_info[rsrc_idx].base = (u64)RS_64_1(temp, IRDMA_COMMIT_FPM_BASE_S) * 512; 2875 2876 return temp; 2877 } 2878 2879 /** 2880 * irdma_sc_parse_fpm_commit_buf - parse fpm commit buffer 2881 * @dev: pointer to dev struct 2882 * @buf: ptr to fpm commit buffer 2883 * @info: ptr to irdma_hmc_obj_info struct 2884 * @sd: number of SDs for HMC objects 2885 * 2886 * parses fpm commit info and copy base value 2887 * of hmc objects in hmc_info 2888 */ 2889 static void 2890 irdma_sc_parse_fpm_commit_buf(struct irdma_sc_dev *dev, __le64 * buf, 2891 struct irdma_hmc_obj_info *info, 2892 u32 *sd) 2893 { 2894 u64 size; 2895 u32 i; 2896 u64 max_base = 0; 2897 u32 last_hmc_obj = 0; 2898 2899 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_0, info, 2900 IRDMA_HMC_IW_QP); 2901 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_8, info, 2902 IRDMA_HMC_IW_CQ); 2903 /* skiping RSRVD */ 2904 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_24, info, 2905 IRDMA_HMC_IW_HTE); 2906 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_32, info, 2907 IRDMA_HMC_IW_ARP); 2908 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_40, info, 2909 IRDMA_HMC_IW_APBVT_ENTRY); 2910 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_48, info, 2911 IRDMA_HMC_IW_MR); 2912 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_56, info, 2913 IRDMA_HMC_IW_XF); 2914 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_64, info, 2915 IRDMA_HMC_IW_XFFL); 2916 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_72, info, 2917 IRDMA_HMC_IW_Q1); 2918 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_80, info, 2919 IRDMA_HMC_IW_Q1FL); 2920 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_88, info, 2921 IRDMA_HMC_IW_TIMER); 2922 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_112, info, 2923 IRDMA_HMC_IW_PBLE); 2924 /* skipping RSVD. */ 2925 if (dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1) { 2926 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_96, info, 2927 IRDMA_HMC_IW_FSIMC); 2928 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_104, info, 2929 IRDMA_HMC_IW_FSIAV); 2930 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_128, info, 2931 IRDMA_HMC_IW_RRF); 2932 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_136, info, 2933 IRDMA_HMC_IW_RRFFL); 2934 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_144, info, 2935 IRDMA_HMC_IW_HDR); 2936 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_152, info, 2937 IRDMA_HMC_IW_MD); 2938 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_160, info, 2939 IRDMA_HMC_IW_OOISC); 2940 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_168, info, 2941 IRDMA_HMC_IW_OOISCFFL); 2942 } 2943 2944 /* searching for the last object in HMC to find the size of the HMC area. */ 2945 for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++) { 2946 if (info[i].base > max_base) { 2947 max_base = info[i].base; 2948 last_hmc_obj = i; 2949 } 2950 } 2951 2952 size = info[last_hmc_obj].cnt * info[last_hmc_obj].size + 2953 info[last_hmc_obj].base; 2954 2955 if (size & 0x1FFFFF) 2956 *sd = (u32)((size >> 21) + 1); /* add 1 for remainder */ 2957 else 2958 *sd = (u32)(size >> 21); 2959 2960 } 2961 2962 /** 2963 * irdma_sc_decode_fpm_query() - Decode a 64 bit value into max count and size 2964 * @buf: ptr to fpm query buffer 2965 * @buf_idx: index into buf 2966 * @obj_info: ptr to irdma_hmc_obj_info struct 2967 * @rsrc_idx: resource index into info 2968 * 2969 * Decode a 64 bit value from fpm query buffer into max count and size 2970 */ 2971 static u64 irdma_sc_decode_fpm_query(__le64 * buf, u32 buf_idx, 2972 struct irdma_hmc_obj_info *obj_info, 2973 u32 rsrc_idx){ 2974 u64 temp; 2975 u32 size; 2976 2977 get_64bit_val(buf, buf_idx, &temp); 2978 obj_info[rsrc_idx].max_cnt = (u32)temp; 2979 size = (u32)RS_64_1(temp, 32); 2980 obj_info[rsrc_idx].size = LS_64_1(1, size); 2981 2982 return temp; 2983 } 2984 2985 /** 2986 * irdma_sc_parse_fpm_query_buf() - parses fpm query buffer 2987 * @dev: ptr to shared code device 2988 * @buf: ptr to fpm query buffer 2989 * @hmc_info: ptr to irdma_hmc_obj_info struct 2990 * @hmc_fpm_misc: ptr to fpm data 2991 * 2992 * parses fpm query buffer and copy max_cnt and 2993 * size value of hmc objects in hmc_info 2994 */ 2995 static int 2996 irdma_sc_parse_fpm_query_buf(struct irdma_sc_dev *dev, __le64 * buf, 2997 struct irdma_hmc_info *hmc_info, 2998 struct irdma_hmc_fpm_misc *hmc_fpm_misc) 2999 { 3000 struct irdma_hmc_obj_info *obj_info; 3001 u64 temp; 3002 u32 size; 3003 u16 max_pe_sds; 3004 3005 obj_info = hmc_info->hmc_obj; 3006 3007 get_64bit_val(buf, IRDMA_BYTE_0, &temp); 3008 hmc_info->first_sd_index = (u16)FIELD_GET(IRDMA_QUERY_FPM_FIRST_PE_SD_INDEX, temp); 3009 max_pe_sds = (u16)FIELD_GET(IRDMA_QUERY_FPM_MAX_PE_SDS, temp); 3010 3011 hmc_fpm_misc->max_sds = max_pe_sds; 3012 hmc_info->sd_table.sd_cnt = max_pe_sds + hmc_info->first_sd_index; 3013 get_64bit_val(buf, 8, &temp); 3014 obj_info[IRDMA_HMC_IW_QP].max_cnt = (u32)FIELD_GET(IRDMA_QUERY_FPM_MAX_QPS, temp); 3015 size = (u32)RS_64_1(temp, 32); 3016 obj_info[IRDMA_HMC_IW_QP].size = LS_64_1(1, size); 3017 3018 get_64bit_val(buf, 16, &temp); 3019 obj_info[IRDMA_HMC_IW_CQ].max_cnt = (u32)FIELD_GET(IRDMA_QUERY_FPM_MAX_CQS, temp); 3020 size = (u32)RS_64_1(temp, 32); 3021 obj_info[IRDMA_HMC_IW_CQ].size = LS_64_1(1, size); 3022 3023 irdma_sc_decode_fpm_query(buf, 32, obj_info, IRDMA_HMC_IW_HTE); 3024 irdma_sc_decode_fpm_query(buf, 40, obj_info, IRDMA_HMC_IW_ARP); 3025 3026 obj_info[IRDMA_HMC_IW_APBVT_ENTRY].size = 8192; 3027 obj_info[IRDMA_HMC_IW_APBVT_ENTRY].max_cnt = 1; 3028 3029 irdma_sc_decode_fpm_query(buf, 48, obj_info, IRDMA_HMC_IW_MR); 3030 irdma_sc_decode_fpm_query(buf, 56, obj_info, IRDMA_HMC_IW_XF); 3031 3032 get_64bit_val(buf, 64, &temp); 3033 obj_info[IRDMA_HMC_IW_XFFL].max_cnt = (u32)temp; 3034 obj_info[IRDMA_HMC_IW_XFFL].size = 4; 3035 hmc_fpm_misc->xf_block_size = FIELD_GET(IRDMA_QUERY_FPM_XFBLOCKSIZE, temp); 3036 if (!hmc_fpm_misc->xf_block_size) 3037 return -EINVAL; 3038 3039 irdma_sc_decode_fpm_query(buf, 72, obj_info, IRDMA_HMC_IW_Q1); 3040 get_64bit_val(buf, 80, &temp); 3041 obj_info[IRDMA_HMC_IW_Q1FL].max_cnt = (u32)temp; 3042 obj_info[IRDMA_HMC_IW_Q1FL].size = 4; 3043 3044 hmc_fpm_misc->q1_block_size = FIELD_GET(IRDMA_QUERY_FPM_Q1BLOCKSIZE, temp); 3045 if (!hmc_fpm_misc->q1_block_size) 3046 return -EINVAL; 3047 3048 irdma_sc_decode_fpm_query(buf, 88, obj_info, IRDMA_HMC_IW_TIMER); 3049 3050 get_64bit_val(buf, 112, &temp); 3051 obj_info[IRDMA_HMC_IW_PBLE].max_cnt = (u32)temp; 3052 obj_info[IRDMA_HMC_IW_PBLE].size = 8; 3053 3054 get_64bit_val(buf, 120, &temp); 3055 hmc_fpm_misc->max_ceqs = FIELD_GET(IRDMA_QUERY_FPM_MAX_CEQS, temp); 3056 hmc_fpm_misc->ht_multiplier = FIELD_GET(IRDMA_QUERY_FPM_HTMULTIPLIER, temp); 3057 hmc_fpm_misc->timer_bucket = FIELD_GET(IRDMA_QUERY_FPM_TIMERBUCKET, temp); 3058 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 3059 return 0; 3060 irdma_sc_decode_fpm_query(buf, 96, obj_info, IRDMA_HMC_IW_FSIMC); 3061 irdma_sc_decode_fpm_query(buf, 104, obj_info, IRDMA_HMC_IW_FSIAV); 3062 irdma_sc_decode_fpm_query(buf, 128, obj_info, IRDMA_HMC_IW_RRF); 3063 3064 get_64bit_val(buf, IRDMA_BYTE_136, &temp); 3065 obj_info[IRDMA_HMC_IW_RRFFL].max_cnt = (u32)temp; 3066 obj_info[IRDMA_HMC_IW_RRFFL].size = 4; 3067 hmc_fpm_misc->rrf_block_size = FIELD_GET(IRDMA_QUERY_FPM_RRFBLOCKSIZE, temp); 3068 if (!hmc_fpm_misc->rrf_block_size && 3069 obj_info[IRDMA_HMC_IW_RRFFL].max_cnt) 3070 return -EINVAL; 3071 3072 irdma_sc_decode_fpm_query(buf, 144, obj_info, IRDMA_HMC_IW_HDR); 3073 irdma_sc_decode_fpm_query(buf, 152, obj_info, IRDMA_HMC_IW_MD); 3074 irdma_sc_decode_fpm_query(buf, 160, obj_info, IRDMA_HMC_IW_OOISC); 3075 3076 get_64bit_val(buf, IRDMA_BYTE_168, &temp); 3077 obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt = (u32)temp; 3078 obj_info[IRDMA_HMC_IW_OOISCFFL].size = 4; 3079 hmc_fpm_misc->ooiscf_block_size = FIELD_GET(IRDMA_QUERY_FPM_OOISCFBLOCKSIZE, temp); 3080 if (!hmc_fpm_misc->ooiscf_block_size && 3081 obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt) 3082 return -EINVAL; 3083 3084 return 0; 3085 } 3086 3087 /** 3088 * irdma_sc_find_reg_cq - find cq ctx index 3089 * @ceq: ceq sc structure 3090 * @cq: cq sc structure 3091 */ 3092 static u32 irdma_sc_find_reg_cq(struct irdma_sc_ceq *ceq, 3093 struct irdma_sc_cq *cq){ 3094 u32 i; 3095 3096 for (i = 0; i < ceq->reg_cq_size; i++) { 3097 if (cq == ceq->reg_cq[i]) 3098 return i; 3099 } 3100 3101 return IRDMA_INVALID_CQ_IDX; 3102 } 3103 3104 /** 3105 * irdma_sc_add_cq_ctx - add cq ctx tracking for ceq 3106 * @ceq: ceq sc structure 3107 * @cq: cq sc structure 3108 */ 3109 int 3110 irdma_sc_add_cq_ctx(struct irdma_sc_ceq *ceq, struct irdma_sc_cq *cq) 3111 { 3112 unsigned long flags; 3113 3114 spin_lock_irqsave(&ceq->req_cq_lock, flags); 3115 3116 if (ceq->reg_cq_size == ceq->elem_cnt) { 3117 spin_unlock_irqrestore(&ceq->req_cq_lock, flags); 3118 return -ENOSPC; 3119 } 3120 3121 ceq->reg_cq[ceq->reg_cq_size++] = cq; 3122 3123 spin_unlock_irqrestore(&ceq->req_cq_lock, flags); 3124 3125 return 0; 3126 } 3127 3128 /** 3129 * irdma_sc_remove_cq_ctx - remove cq ctx tracking for ceq 3130 * @ceq: ceq sc structure 3131 * @cq: cq sc structure 3132 */ 3133 void 3134 irdma_sc_remove_cq_ctx(struct irdma_sc_ceq *ceq, struct irdma_sc_cq *cq) 3135 { 3136 unsigned long flags; 3137 u32 cq_ctx_idx; 3138 3139 spin_lock_irqsave(&ceq->req_cq_lock, flags); 3140 cq_ctx_idx = irdma_sc_find_reg_cq(ceq, cq); 3141 if (cq_ctx_idx == IRDMA_INVALID_CQ_IDX) 3142 goto exit; 3143 3144 ceq->reg_cq_size--; 3145 if (cq_ctx_idx != ceq->reg_cq_size) 3146 ceq->reg_cq[cq_ctx_idx] = ceq->reg_cq[ceq->reg_cq_size]; 3147 ceq->reg_cq[ceq->reg_cq_size] = NULL; 3148 3149 exit: 3150 spin_unlock_irqrestore(&ceq->req_cq_lock, flags); 3151 } 3152 3153 /** 3154 * irdma_sc_cqp_init - Initialize buffers for a control Queue Pair 3155 * @cqp: IWARP control queue pair pointer 3156 * @info: IWARP control queue pair init info pointer 3157 * 3158 * Initializes the object and context buffers for a control Queue Pair. 3159 */ 3160 int 3161 irdma_sc_cqp_init(struct irdma_sc_cqp *cqp, 3162 struct irdma_cqp_init_info *info) 3163 { 3164 u8 hw_sq_size; 3165 3166 if (info->sq_size > IRDMA_CQP_SW_SQSIZE_2048 || 3167 info->sq_size < IRDMA_CQP_SW_SQSIZE_4 || 3168 ((info->sq_size & (info->sq_size - 1)))) 3169 return -EINVAL; 3170 3171 hw_sq_size = irdma_get_encoded_wqe_size(info->sq_size, 3172 IRDMA_QUEUE_TYPE_CQP); 3173 cqp->size = sizeof(*cqp); 3174 cqp->sq_size = info->sq_size; 3175 cqp->hw_sq_size = hw_sq_size; 3176 cqp->sq_base = info->sq; 3177 cqp->host_ctx = info->host_ctx; 3178 cqp->sq_pa = info->sq_pa; 3179 cqp->host_ctx_pa = info->host_ctx_pa; 3180 cqp->dev = info->dev; 3181 cqp->struct_ver = info->struct_ver; 3182 cqp->hw_maj_ver = info->hw_maj_ver; 3183 cqp->hw_min_ver = info->hw_min_ver; 3184 cqp->scratch_array = info->scratch_array; 3185 cqp->polarity = 0; 3186 cqp->en_datacenter_tcp = info->en_datacenter_tcp; 3187 cqp->ena_vf_count = info->ena_vf_count; 3188 cqp->hmc_profile = info->hmc_profile; 3189 cqp->ceqs_per_vf = info->ceqs_per_vf; 3190 cqp->disable_packed = info->disable_packed; 3191 cqp->rocev2_rto_policy = info->rocev2_rto_policy; 3192 cqp->protocol_used = info->protocol_used; 3193 irdma_memcpy(&cqp->dcqcn_params, &info->dcqcn_params, sizeof(cqp->dcqcn_params)); 3194 cqp->en_rem_endpoint_trk = info->en_rem_endpoint_trk; 3195 info->dev->cqp = cqp; 3196 3197 IRDMA_RING_INIT(cqp->sq_ring, cqp->sq_size); 3198 cqp->requested_ops = 0; 3199 atomic64_set(&cqp->completed_ops, 0); 3200 /* for the cqp commands backlog. */ 3201 INIT_LIST_HEAD(&cqp->dev->cqp_cmd_head); 3202 3203 writel(0, cqp->dev->hw_regs[IRDMA_CQPTAIL]); 3204 writel(0, cqp->dev->hw_regs[IRDMA_CQPDB]); 3205 writel(0, cqp->dev->hw_regs[IRDMA_CCQPSTATUS]); 3206 3207 irdma_debug(cqp->dev, IRDMA_DEBUG_WQE, 3208 "sq_size[%04d] hw_sq_size[%04d] sq_base[%p] sq_pa[%llxh] cqp[%p] polarity[x%04x]\n", 3209 cqp->sq_size, cqp->hw_sq_size, cqp->sq_base, 3210 (unsigned long long)cqp->sq_pa, cqp, cqp->polarity); 3211 return 0; 3212 } 3213 3214 /** 3215 * irdma_sc_cqp_create - create cqp during bringup 3216 * @cqp: struct for cqp hw 3217 * @maj_err: If error, major err number 3218 * @min_err: If error, minor err number 3219 */ 3220 int 3221 irdma_sc_cqp_create(struct irdma_sc_cqp *cqp, u16 *maj_err, u16 *min_err) 3222 { 3223 u64 temp; 3224 u8 hw_rev; 3225 u32 cnt = 0, p1, p2, val = 0, err_code; 3226 int ret_code; 3227 3228 hw_rev = cqp->dev->hw_attrs.uk_attrs.hw_rev; 3229 cqp->sdbuf.size = IRDMA_UPDATE_SD_BUFF_SIZE * cqp->sq_size; 3230 cqp->sdbuf.va = irdma_allocate_dma_mem(cqp->dev->hw, &cqp->sdbuf, 3231 cqp->sdbuf.size, 3232 IRDMA_SD_BUF_ALIGNMENT); 3233 if (!cqp->sdbuf.va) 3234 return -ENOMEM; 3235 3236 spin_lock_init(&cqp->dev->cqp_lock); 3237 3238 temp = FIELD_PREP(IRDMA_CQPHC_SQSIZE, cqp->hw_sq_size) | 3239 FIELD_PREP(IRDMA_CQPHC_SVER, cqp->struct_ver) | 3240 FIELD_PREP(IRDMA_CQPHC_DISABLE_PFPDUS, cqp->disable_packed) | 3241 FIELD_PREP(IRDMA_CQPHC_CEQPERVF, cqp->ceqs_per_vf); 3242 if (hw_rev >= IRDMA_GEN_2) { 3243 temp |= FIELD_PREP(IRDMA_CQPHC_ROCEV2_RTO_POLICY, 3244 cqp->rocev2_rto_policy) | 3245 FIELD_PREP(IRDMA_CQPHC_PROTOCOL_USED, 3246 cqp->protocol_used); 3247 } 3248 3249 set_64bit_val(cqp->host_ctx, IRDMA_BYTE_0, temp); 3250 set_64bit_val(cqp->host_ctx, IRDMA_BYTE_8, cqp->sq_pa); 3251 3252 temp = FIELD_PREP(IRDMA_CQPHC_ENABLED_VFS, cqp->ena_vf_count) | 3253 FIELD_PREP(IRDMA_CQPHC_HMC_PROFILE, cqp->hmc_profile); 3254 if (hw_rev >= IRDMA_GEN_2) 3255 temp |= FIELD_PREP(IRDMA_CQPHC_EN_REM_ENDPOINT_TRK, 3256 cqp->en_rem_endpoint_trk); 3257 set_64bit_val(cqp->host_ctx, IRDMA_BYTE_16, temp); 3258 set_64bit_val(cqp->host_ctx, IRDMA_BYTE_24, (uintptr_t)cqp); 3259 temp = FIELD_PREP(IRDMA_CQPHC_HW_MAJVER, cqp->hw_maj_ver) | 3260 FIELD_PREP(IRDMA_CQPHC_HW_MINVER, cqp->hw_min_ver); 3261 if (hw_rev >= IRDMA_GEN_2) { 3262 temp |= FIELD_PREP(IRDMA_CQPHC_MIN_RATE, cqp->dcqcn_params.min_rate) | 3263 FIELD_PREP(IRDMA_CQPHC_MIN_DEC_FACTOR, cqp->dcqcn_params.min_dec_factor); 3264 } 3265 set_64bit_val(cqp->host_ctx, IRDMA_BYTE_32, temp); 3266 set_64bit_val(cqp->host_ctx, IRDMA_BYTE_40, 0); 3267 temp = 0; 3268 if (hw_rev >= IRDMA_GEN_2) { 3269 temp |= FIELD_PREP(IRDMA_CQPHC_DCQCN_T, cqp->dcqcn_params.dcqcn_t) | 3270 FIELD_PREP(IRDMA_CQPHC_RAI_FACTOR, cqp->dcqcn_params.rai_factor) | 3271 FIELD_PREP(IRDMA_CQPHC_HAI_FACTOR, cqp->dcqcn_params.hai_factor); 3272 } 3273 set_64bit_val(cqp->host_ctx, IRDMA_BYTE_48, temp); 3274 temp = 0; 3275 if (hw_rev >= IRDMA_GEN_2) { 3276 temp |= FIELD_PREP(IRDMA_CQPHC_DCQCN_B, cqp->dcqcn_params.dcqcn_b) | 3277 FIELD_PREP(IRDMA_CQPHC_DCQCN_F, cqp->dcqcn_params.dcqcn_f) | 3278 FIELD_PREP(IRDMA_CQPHC_CC_CFG_VALID, cqp->dcqcn_params.cc_cfg_valid) | 3279 FIELD_PREP(IRDMA_CQPHC_RREDUCE_MPERIOD, cqp->dcqcn_params.rreduce_mperiod); 3280 } 3281 set_64bit_val(cqp->host_ctx, IRDMA_BYTE_56, temp); 3282 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "CQP_HOST_CTX WQE", 3283 cqp->host_ctx, IRDMA_CQP_CTX_SIZE * 8); 3284 p1 = RS_32_1(cqp->host_ctx_pa, 32); 3285 p2 = (u32)cqp->host_ctx_pa; 3286 3287 writel(p1, cqp->dev->hw_regs[IRDMA_CCQPHIGH]); 3288 writel(p2, cqp->dev->hw_regs[IRDMA_CCQPLOW]); 3289 3290 do { 3291 if (cnt++ > cqp->dev->hw_attrs.max_done_count) { 3292 ret_code = -ETIMEDOUT; 3293 goto err; 3294 } 3295 irdma_usec_delay(cqp->dev->hw_attrs.max_sleep_count); 3296 val = readl(cqp->dev->hw_regs[IRDMA_CCQPSTATUS]); 3297 } while (!val); 3298 3299 if (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_ERR)) { 3300 ret_code = -EOPNOTSUPP; 3301 goto err; 3302 } 3303 3304 cqp->process_cqp_sds = irdma_update_sds_noccq; 3305 return 0; 3306 3307 err: 3308 spin_lock_destroy(&cqp->dev->cqp_lock); 3309 irdma_free_dma_mem(cqp->dev->hw, &cqp->sdbuf); 3310 err_code = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]); 3311 *min_err = FIELD_GET(IRDMA_CQPERRCODES_CQP_MINOR_CODE, err_code); 3312 *maj_err = FIELD_GET(IRDMA_CQPERRCODES_CQP_MAJOR_CODE, err_code); 3313 return ret_code; 3314 } 3315 3316 /** 3317 * irdma_sc_cqp_post_sq - post of cqp's sq 3318 * @cqp: struct for cqp hw 3319 */ 3320 void 3321 irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp) 3322 { 3323 db_wr32(IRDMA_RING_CURRENT_HEAD(cqp->sq_ring), cqp->dev->cqp_db); 3324 3325 irdma_debug(cqp->dev, IRDMA_DEBUG_WQE, 3326 "CQP SQ head 0x%x tail 0x%x size 0x%x\n", cqp->sq_ring.head, 3327 cqp->sq_ring.tail, cqp->sq_ring.size); 3328 } 3329 3330 /** 3331 * irdma_sc_cqp_get_next_send_wqe_idx - get next wqe on cqp sq 3332 * and pass back index 3333 * @cqp: CQP HW structure 3334 * @scratch: private data for CQP WQE 3335 * @wqe_idx: WQE index of CQP SQ 3336 */ 3337 __le64 * 3338 irdma_sc_cqp_get_next_send_wqe_idx(struct irdma_sc_cqp *cqp, u64 scratch, 3339 u32 *wqe_idx) 3340 { 3341 __le64 *wqe = NULL; 3342 int ret_code; 3343 3344 if (IRDMA_RING_FULL_ERR(cqp->sq_ring)) { 3345 irdma_debug(cqp->dev, IRDMA_DEBUG_WQE, 3346 "CQP SQ is full, head 0x%x tail 0x%x size 0x%x\n", 3347 cqp->sq_ring.head, cqp->sq_ring.tail, 3348 cqp->sq_ring.size); 3349 return NULL; 3350 } 3351 IRDMA_ATOMIC_RING_MOVE_HEAD(cqp->sq_ring, *wqe_idx, ret_code); 3352 if (ret_code) 3353 return NULL; 3354 3355 cqp->requested_ops++; 3356 if (!*wqe_idx) 3357 cqp->polarity = !cqp->polarity; 3358 wqe = cqp->sq_base[*wqe_idx].elem; 3359 cqp->scratch_array[*wqe_idx] = scratch; 3360 3361 memset(&wqe[0], 0, 24); 3362 memset(&wqe[4], 0, 32); 3363 3364 return wqe; 3365 } 3366 3367 /** 3368 * irdma_sc_cqp_destroy - destroy cqp during close 3369 * @cqp: struct for cqp hw 3370 * @free_hwcqp: true for regular cqp destroy; false for reset path 3371 */ 3372 int 3373 irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp, bool free_hwcqp) 3374 { 3375 u32 cnt = 0, val; 3376 int ret_code = 0; 3377 3378 if (free_hwcqp) { 3379 writel(0, cqp->dev->hw_regs[IRDMA_CCQPHIGH]); 3380 writel(0, cqp->dev->hw_regs[IRDMA_CCQPLOW]); 3381 do { 3382 if (cnt++ > cqp->dev->hw_attrs.max_done_count) { 3383 ret_code = -ETIMEDOUT; 3384 break; 3385 } 3386 irdma_usec_delay(cqp->dev->hw_attrs.max_sleep_count); 3387 val = readl(cqp->dev->hw_regs[IRDMA_CCQPSTATUS]); 3388 } while (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_DONE)); 3389 } 3390 irdma_free_dma_mem(cqp->dev->hw, &cqp->sdbuf); 3391 spin_lock_destroy(&cqp->dev->cqp_lock); 3392 return ret_code; 3393 } 3394 3395 /** 3396 * irdma_sc_ccq_arm - enable intr for control cq 3397 * @ccq: ccq sc struct 3398 */ 3399 void 3400 irdma_sc_ccq_arm(struct irdma_sc_cq *ccq) 3401 { 3402 u64 temp_val; 3403 u16 sw_cq_sel; 3404 u8 arm_next_se; 3405 u8 arm_seq_num; 3406 3407 get_64bit_val(ccq->cq_uk.shadow_area, IRDMA_BYTE_32, &temp_val); 3408 sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val); 3409 arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val); 3410 arm_seq_num = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val); 3411 arm_seq_num++; 3412 temp_val = FIELD_PREP(IRDMA_CQ_DBSA_ARM_SEQ_NUM, arm_seq_num) | 3413 FIELD_PREP(IRDMA_CQ_DBSA_SW_CQ_SELECT, sw_cq_sel) | 3414 FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) | 3415 FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, 1); 3416 set_64bit_val(ccq->cq_uk.shadow_area, IRDMA_BYTE_32, temp_val); 3417 3418 irdma_wmb(); /* make sure shadow area is updated before arming */ 3419 3420 db_wr32(ccq->cq_uk.cq_id, ccq->dev->cq_arm_db); 3421 } 3422 3423 /** 3424 * irdma_sc_ccq_get_cqe_info - get ccq's cq entry 3425 * @ccq: ccq sc struct 3426 * @info: completion q entry to return 3427 */ 3428 int 3429 irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq, 3430 struct irdma_ccq_cqe_info *info) 3431 { 3432 u64 qp_ctx, temp, temp1; 3433 __le64 *cqe; 3434 struct irdma_sc_cqp *cqp; 3435 u32 wqe_idx; 3436 u32 error; 3437 u8 polarity; 3438 int ret_code = 0; 3439 3440 if (ccq->cq_uk.avoid_mem_cflct) 3441 cqe = IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(&ccq->cq_uk); 3442 else 3443 cqe = IRDMA_GET_CURRENT_CQ_ELEM(&ccq->cq_uk); 3444 3445 get_64bit_val(cqe, IRDMA_BYTE_24, &temp); 3446 polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, temp); 3447 if (polarity != ccq->cq_uk.polarity) 3448 return -ENOENT; 3449 3450 /* Ensure CEQE contents are read after valid bit is checked */ 3451 rmb(); 3452 3453 get_64bit_val(cqe, IRDMA_BYTE_8, &qp_ctx); 3454 cqp = (struct irdma_sc_cqp *)(irdma_uintptr) qp_ctx; 3455 info->error = (bool)FIELD_GET(IRDMA_CQ_ERROR, temp); 3456 info->maj_err_code = IRDMA_CQPSQ_MAJ_NO_ERROR; 3457 info->min_err_code = (u16)FIELD_GET(IRDMA_CQ_MINERR, temp); 3458 if (info->error) { 3459 info->maj_err_code = (u16)FIELD_GET(IRDMA_CQ_MAJERR, temp); 3460 error = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]); 3461 irdma_debug(cqp->dev, IRDMA_DEBUG_CQP, 3462 "CQPERRCODES error_code[x%08X]\n", error); 3463 } 3464 3465 wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, temp); 3466 info->scratch = cqp->scratch_array[wqe_idx]; 3467 3468 get_64bit_val(cqe, IRDMA_BYTE_16, &temp1); 3469 info->op_ret_val = (u32)FIELD_GET(IRDMA_CCQ_OPRETVAL, temp1); 3470 3471 get_64bit_val(cqp->sq_base[wqe_idx].elem, IRDMA_BYTE_24, &temp1); 3472 info->op_code = (u8)FIELD_GET(IRDMA_CQPSQ_OPCODE, temp1); 3473 info->cqp = cqp; 3474 3475 /* move the head for cq */ 3476 IRDMA_RING_MOVE_HEAD(ccq->cq_uk.cq_ring, ret_code); 3477 if (!IRDMA_RING_CURRENT_HEAD(ccq->cq_uk.cq_ring)) 3478 ccq->cq_uk.polarity ^= 1; 3479 3480 /* update cq tail in cq shadow memory also */ 3481 IRDMA_RING_MOVE_TAIL(ccq->cq_uk.cq_ring); 3482 set_64bit_val(ccq->cq_uk.shadow_area, IRDMA_BYTE_0, 3483 IRDMA_RING_CURRENT_HEAD(ccq->cq_uk.cq_ring)); 3484 3485 irdma_wmb(); /* make sure shadow area is updated before moving tail */ 3486 3487 IRDMA_RING_MOVE_TAIL(cqp->sq_ring); 3488 atomic64_inc(&cqp->completed_ops); 3489 3490 return ret_code; 3491 } 3492 3493 /** 3494 * irdma_sc_poll_for_cqp_op_done - Waits for last write to complete in CQP SQ 3495 * @cqp: struct for cqp hw 3496 * @op_code: cqp opcode for completion 3497 * @compl_info: completion q entry to return 3498 */ 3499 int 3500 irdma_sc_poll_for_cqp_op_done(struct irdma_sc_cqp *cqp, u8 op_code, 3501 struct irdma_ccq_cqe_info *compl_info) 3502 { 3503 struct irdma_ccq_cqe_info info = {0}; 3504 struct irdma_sc_cq *ccq; 3505 int ret_code = 0; 3506 u32 cnt = 0; 3507 3508 ccq = cqp->dev->ccq; 3509 while (1) { 3510 if (cnt++ > 100 * cqp->dev->hw_attrs.max_done_count) 3511 return -ETIMEDOUT; 3512 3513 if (irdma_sc_ccq_get_cqe_info(ccq, &info)) { 3514 irdma_usec_delay(cqp->dev->hw_attrs.max_sleep_count); 3515 continue; 3516 } 3517 if (info.error && info.op_code != IRDMA_CQP_OP_QUERY_STAG) { 3518 ret_code = -EIO; 3519 break; 3520 } 3521 /* make sure op code matches */ 3522 if (op_code == info.op_code) 3523 break; 3524 irdma_debug(cqp->dev, IRDMA_DEBUG_WQE, 3525 "opcode mismatch for my op code 0x%x, returned opcode %x\n", 3526 op_code, info.op_code); 3527 } 3528 3529 if (compl_info) 3530 irdma_memcpy(compl_info, &info, sizeof(*compl_info)); 3531 3532 return ret_code; 3533 } 3534 3535 /** 3536 * irdma_sc_manage_hmc_pm_func_table - manage of function table 3537 * @cqp: struct for cqp hw 3538 * @scratch: u64 saved to be used during cqp completion 3539 * @info: info for the manage function table operation 3540 * @post_sq: flag for cqp db to ring 3541 */ 3542 static int 3543 irdma_sc_manage_hmc_pm_func_table(struct irdma_sc_cqp *cqp, 3544 struct irdma_hmc_fcn_info *info, 3545 u64 scratch, bool post_sq) 3546 { 3547 __le64 *wqe; 3548 u64 hdr; 3549 3550 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 3551 if (!wqe) 3552 return -ENOSPC; 3553 3554 hdr = FIELD_PREP(IRDMA_CQPSQ_MHMC_VFIDX, info->vf_id) | 3555 FIELD_PREP(IRDMA_CQPSQ_OPCODE, 3556 IRDMA_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE) | 3557 FIELD_PREP(IRDMA_CQPSQ_MHMC_FREEPMFN, info->free_fcn) | 3558 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 3559 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 3560 3561 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 3562 3563 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, 3564 "MANAGE_HMC_PM_FUNC_TABLE WQE", wqe, 3565 IRDMA_CQP_WQE_SIZE * 8); 3566 if (post_sq) 3567 irdma_sc_cqp_post_sq(cqp); 3568 3569 return 0; 3570 } 3571 3572 /** 3573 * irdma_sc_commit_fpm_val_done - wait for cqp eqe completion 3574 * for fpm commit 3575 * @cqp: struct for cqp hw 3576 */ 3577 static int 3578 irdma_sc_commit_fpm_val_done(struct irdma_sc_cqp *cqp) 3579 { 3580 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_COMMIT_FPM_VAL, 3581 NULL); 3582 } 3583 3584 /** 3585 * irdma_sc_commit_fpm_val - cqp wqe for commit fpm values 3586 * @cqp: struct for cqp hw 3587 * @scratch: u64 saved to be used during cqp completion 3588 * @hmc_fn_id: hmc function id 3589 * @commit_fpm_mem: Memory for fpm values 3590 * @post_sq: flag for cqp db to ring 3591 * @wait_type: poll ccq or cqp registers for cqp completion 3592 */ 3593 static int 3594 irdma_sc_commit_fpm_val(struct irdma_sc_cqp *cqp, u64 scratch, 3595 u16 hmc_fn_id, 3596 struct irdma_dma_mem *commit_fpm_mem, 3597 bool post_sq, u8 wait_type) 3598 { 3599 __le64 *wqe; 3600 u64 hdr; 3601 u32 tail, val, error; 3602 int ret_code = 0; 3603 3604 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 3605 if (!wqe) 3606 return -ENOSPC; 3607 3608 set_64bit_val(wqe, IRDMA_BYTE_16, hmc_fn_id); 3609 set_64bit_val(wqe, IRDMA_BYTE_32, commit_fpm_mem->pa); 3610 3611 hdr = FIELD_PREP(IRDMA_CQPSQ_BUFSIZE, IRDMA_COMMIT_FPM_BUF_SIZE) | 3612 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_COMMIT_FPM_VAL) | 3613 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 3614 3615 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 3616 3617 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 3618 3619 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "COMMIT_FPM_VAL WQE", wqe, 3620 IRDMA_CQP_WQE_SIZE * 8); 3621 irdma_get_cqp_reg_info(cqp, &val, &tail, &error); 3622 3623 if (post_sq) { 3624 irdma_sc_cqp_post_sq(cqp); 3625 if (wait_type == IRDMA_CQP_WAIT_POLL_REGS) 3626 ret_code = irdma_cqp_poll_registers(cqp, tail, 3627 cqp->dev->hw_attrs.max_done_count); 3628 else if (wait_type == IRDMA_CQP_WAIT_POLL_CQ) 3629 ret_code = irdma_sc_commit_fpm_val_done(cqp); 3630 } 3631 3632 return ret_code; 3633 } 3634 3635 /** 3636 * irdma_sc_query_fpm_val_done - poll for cqp wqe completion for 3637 * query fpm 3638 * @cqp: struct for cqp hw 3639 */ 3640 static int 3641 irdma_sc_query_fpm_val_done(struct irdma_sc_cqp *cqp) 3642 { 3643 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_QUERY_FPM_VAL, 3644 NULL); 3645 } 3646 3647 /** 3648 * irdma_sc_query_fpm_val - cqp wqe query fpm values 3649 * @cqp: struct for cqp hw 3650 * @scratch: u64 saved to be used during cqp completion 3651 * @hmc_fn_id: hmc function id 3652 * @query_fpm_mem: memory for return fpm values 3653 * @post_sq: flag for cqp db to ring 3654 * @wait_type: poll ccq or cqp registers for cqp completion 3655 */ 3656 static int 3657 irdma_sc_query_fpm_val(struct irdma_sc_cqp *cqp, u64 scratch, 3658 u16 hmc_fn_id, 3659 struct irdma_dma_mem *query_fpm_mem, 3660 bool post_sq, u8 wait_type) 3661 { 3662 __le64 *wqe; 3663 u64 hdr; 3664 u32 tail, val, error; 3665 int ret_code = 0; 3666 3667 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 3668 if (!wqe) 3669 return -ENOSPC; 3670 3671 set_64bit_val(wqe, IRDMA_BYTE_16, hmc_fn_id); 3672 set_64bit_val(wqe, IRDMA_BYTE_32, query_fpm_mem->pa); 3673 3674 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_QUERY_FPM_VAL) | 3675 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 3676 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 3677 3678 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 3679 3680 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "QUERY_FPM WQE", wqe, 3681 IRDMA_CQP_WQE_SIZE * 8); 3682 irdma_get_cqp_reg_info(cqp, &val, &tail, &error); 3683 3684 if (post_sq) { 3685 irdma_sc_cqp_post_sq(cqp); 3686 if (wait_type == IRDMA_CQP_WAIT_POLL_REGS) 3687 ret_code = irdma_cqp_poll_registers(cqp, tail, 3688 cqp->dev->hw_attrs.max_done_count); 3689 else if (wait_type == IRDMA_CQP_WAIT_POLL_CQ) 3690 ret_code = irdma_sc_query_fpm_val_done(cqp); 3691 } 3692 3693 return ret_code; 3694 } 3695 3696 /** 3697 * irdma_sc_ceq_init - initialize ceq 3698 * @ceq: ceq sc structure 3699 * @info: ceq initialization info 3700 */ 3701 int 3702 irdma_sc_ceq_init(struct irdma_sc_ceq *ceq, 3703 struct irdma_ceq_init_info *info) 3704 { 3705 u32 pble_obj_cnt; 3706 3707 if (info->elem_cnt < info->dev->hw_attrs.min_hw_ceq_size || 3708 info->elem_cnt > info->dev->hw_attrs.max_hw_ceq_size) 3709 return -EINVAL; 3710 3711 if (info->ceq_id > (info->dev->hmc_fpm_misc.max_ceqs - 1)) 3712 return -EINVAL; 3713 pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; 3714 3715 if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt) 3716 return -EINVAL; 3717 3718 ceq->size = sizeof(*ceq); 3719 ceq->ceqe_base = (struct irdma_ceqe *)info->ceqe_base; 3720 ceq->ceq_id = info->ceq_id; 3721 ceq->dev = info->dev; 3722 ceq->elem_cnt = info->elem_cnt; 3723 ceq->ceq_elem_pa = info->ceqe_pa; 3724 ceq->virtual_map = info->virtual_map; 3725 ceq->itr_no_expire = info->itr_no_expire; 3726 ceq->reg_cq = info->reg_cq; 3727 ceq->reg_cq_size = 0; 3728 spin_lock_init(&ceq->req_cq_lock); 3729 ceq->pbl_chunk_size = (ceq->virtual_map ? info->pbl_chunk_size : 0); 3730 ceq->first_pm_pbl_idx = (ceq->virtual_map ? info->first_pm_pbl_idx : 0); 3731 ceq->pbl_list = (ceq->virtual_map ? info->pbl_list : NULL); 3732 ceq->tph_en = info->tph_en; 3733 ceq->tph_val = info->tph_val; 3734 ceq->vsi = info->vsi; 3735 ceq->polarity = 1; 3736 IRDMA_RING_INIT(ceq->ceq_ring, ceq->elem_cnt); 3737 ceq->dev->ceq[info->ceq_id] = ceq; 3738 3739 return 0; 3740 } 3741 3742 /** 3743 * irdma_sc_ceq_create - create ceq wqe 3744 * @ceq: ceq sc structure 3745 * @scratch: u64 saved to be used during cqp completion 3746 * @post_sq: flag for cqp db to ring 3747 */ 3748 static int 3749 irdma_sc_ceq_create(struct irdma_sc_ceq *ceq, u64 scratch, 3750 bool post_sq) 3751 { 3752 struct irdma_sc_cqp *cqp; 3753 __le64 *wqe; 3754 u64 hdr; 3755 3756 cqp = ceq->dev->cqp; 3757 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 3758 if (!wqe) 3759 return -ENOSPC; 3760 set_64bit_val(wqe, IRDMA_BYTE_16, ceq->elem_cnt); 3761 set_64bit_val(wqe, IRDMA_BYTE_32, 3762 (ceq->virtual_map ? 0 : ceq->ceq_elem_pa)); 3763 set_64bit_val(wqe, IRDMA_BYTE_48, 3764 (ceq->virtual_map ? ceq->first_pm_pbl_idx : 0)); 3765 set_64bit_val(wqe, IRDMA_BYTE_56, 3766 FIELD_PREP(IRDMA_CQPSQ_TPHVAL, ceq->tph_val) | 3767 FIELD_PREP(IRDMA_CQPSQ_VSIIDX, ceq->vsi->vsi_idx)); 3768 hdr = FIELD_PREP(IRDMA_CQPSQ_CEQ_CEQID, ceq->ceq_id) | 3769 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_CEQ) | 3770 FIELD_PREP(IRDMA_CQPSQ_CEQ_LPBLSIZE, ceq->pbl_chunk_size) | 3771 FIELD_PREP(IRDMA_CQPSQ_CEQ_VMAP, ceq->virtual_map) | 3772 FIELD_PREP(IRDMA_CQPSQ_CEQ_ITRNOEXPIRE, ceq->itr_no_expire) | 3773 FIELD_PREP(IRDMA_CQPSQ_TPHEN, ceq->tph_en) | 3774 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 3775 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 3776 3777 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 3778 3779 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "CEQ_CREATE WQE", wqe, 3780 IRDMA_CQP_WQE_SIZE * 8); 3781 if (post_sq) 3782 irdma_sc_cqp_post_sq(cqp); 3783 3784 return 0; 3785 } 3786 3787 /** 3788 * irdma_sc_cceq_create_done - poll for control ceq wqe to complete 3789 * @ceq: ceq sc structure 3790 */ 3791 static int 3792 irdma_sc_cceq_create_done(struct irdma_sc_ceq *ceq) 3793 { 3794 struct irdma_sc_cqp *cqp; 3795 3796 cqp = ceq->dev->cqp; 3797 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_CREATE_CEQ, 3798 NULL); 3799 } 3800 3801 /** 3802 * irdma_sc_cceq_destroy_done - poll for destroy cceq to complete 3803 * @ceq: ceq sc structure 3804 */ 3805 int 3806 irdma_sc_cceq_destroy_done(struct irdma_sc_ceq *ceq) 3807 { 3808 struct irdma_sc_cqp *cqp; 3809 3810 if (ceq->reg_cq) 3811 irdma_sc_remove_cq_ctx(ceq, ceq->dev->ccq); 3812 3813 cqp = ceq->dev->cqp; 3814 cqp->process_cqp_sds = irdma_update_sds_noccq; 3815 3816 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_DESTROY_CEQ, 3817 NULL); 3818 } 3819 3820 /** 3821 * irdma_sc_cceq_create - create cceq 3822 * @ceq: ceq sc structure 3823 * @scratch: u64 saved to be used during cqp completion 3824 */ 3825 int 3826 irdma_sc_cceq_create(struct irdma_sc_ceq *ceq, u64 scratch) 3827 { 3828 int ret_code; 3829 struct irdma_sc_dev *dev = ceq->dev; 3830 3831 dev->ccq->vsi = ceq->vsi; 3832 if (ceq->reg_cq) { 3833 ret_code = irdma_sc_add_cq_ctx(ceq, ceq->dev->ccq); 3834 if (ret_code) 3835 return ret_code; 3836 } 3837 3838 ret_code = irdma_sc_ceq_create(ceq, scratch, true); 3839 if (!ret_code) 3840 return irdma_sc_cceq_create_done(ceq); 3841 3842 return ret_code; 3843 } 3844 3845 /** 3846 * irdma_sc_ceq_destroy - destroy ceq 3847 * @ceq: ceq sc structure 3848 * @scratch: u64 saved to be used during cqp completion 3849 * @post_sq: flag for cqp db to ring 3850 */ 3851 int 3852 irdma_sc_ceq_destroy(struct irdma_sc_ceq *ceq, u64 scratch, bool post_sq) 3853 { 3854 struct irdma_sc_cqp *cqp; 3855 __le64 *wqe; 3856 u64 hdr; 3857 3858 cqp = ceq->dev->cqp; 3859 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 3860 if (!wqe) 3861 return -ENOSPC; 3862 3863 set_64bit_val(wqe, IRDMA_BYTE_16, ceq->elem_cnt); 3864 set_64bit_val(wqe, IRDMA_BYTE_48, ceq->first_pm_pbl_idx); 3865 hdr = ceq->ceq_id | 3866 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_CEQ) | 3867 FIELD_PREP(IRDMA_CQPSQ_CEQ_LPBLSIZE, ceq->pbl_chunk_size) | 3868 FIELD_PREP(IRDMA_CQPSQ_CEQ_VMAP, ceq->virtual_map) | 3869 FIELD_PREP(IRDMA_CQPSQ_TPHEN, ceq->tph_en) | 3870 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 3871 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 3872 3873 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 3874 3875 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "CEQ_DESTROY WQE", wqe, 3876 IRDMA_CQP_WQE_SIZE * 8); 3877 ceq->dev->ceq[ceq->ceq_id] = NULL; 3878 if (post_sq) 3879 irdma_sc_cqp_post_sq(cqp); 3880 3881 return 0; 3882 } 3883 3884 /** 3885 * irdma_sc_process_ceq - process ceq 3886 * @dev: sc device struct 3887 * @ceq: ceq sc structure 3888 * 3889 * It is expected caller serializes this function with cleanup_ceqes() 3890 * because these functions manipulate the same ceq 3891 */ 3892 void * 3893 irdma_sc_process_ceq(struct irdma_sc_dev *dev, struct irdma_sc_ceq *ceq) 3894 { 3895 u64 temp; 3896 __le64 *ceqe; 3897 struct irdma_sc_cq *cq = NULL; 3898 struct irdma_sc_cq *temp_cq; 3899 u8 polarity; 3900 u32 cq_idx; 3901 unsigned long flags; 3902 3903 do { 3904 cq_idx = 0; 3905 ceqe = IRDMA_GET_CURRENT_CEQ_ELEM(ceq); 3906 get_64bit_val(ceqe, IRDMA_BYTE_0, &temp); 3907 polarity = (u8)FIELD_GET(IRDMA_CEQE_VALID, temp); 3908 if (polarity != ceq->polarity) 3909 return NULL; 3910 3911 temp_cq = (struct irdma_sc_cq *)(irdma_uintptr) LS_64_1(temp, 1); 3912 if (!temp_cq) { 3913 cq_idx = IRDMA_INVALID_CQ_IDX; 3914 IRDMA_RING_MOVE_TAIL(ceq->ceq_ring); 3915 3916 if (!IRDMA_RING_CURRENT_TAIL(ceq->ceq_ring)) 3917 ceq->polarity ^= 1; 3918 continue; 3919 } 3920 3921 cq = temp_cq; 3922 if (ceq->reg_cq) { 3923 spin_lock_irqsave(&ceq->req_cq_lock, flags); 3924 cq_idx = irdma_sc_find_reg_cq(ceq, cq); 3925 spin_unlock_irqrestore(&ceq->req_cq_lock, flags); 3926 } 3927 3928 IRDMA_RING_MOVE_TAIL(ceq->ceq_ring); 3929 if (!IRDMA_RING_CURRENT_TAIL(ceq->ceq_ring)) 3930 ceq->polarity ^= 1; 3931 } while (cq_idx == IRDMA_INVALID_CQ_IDX); 3932 3933 if (cq) 3934 irdma_sc_cq_ack(cq); 3935 return cq; 3936 } 3937 3938 /** 3939 * irdma_sc_cleanup_ceqes - clear the valid ceqes ctx matching the cq 3940 * @cq: cq for which the ceqes need to be cleaned up 3941 * @ceq: ceq ptr 3942 * 3943 * The function is called after the cq is destroyed to cleanup 3944 * its pending ceqe entries. It is expected caller serializes this 3945 * function with process_ceq() in interrupt context. 3946 */ 3947 void 3948 irdma_sc_cleanup_ceqes(struct irdma_sc_cq *cq, struct irdma_sc_ceq *ceq) 3949 { 3950 struct irdma_sc_cq *next_cq; 3951 u8 ceq_polarity = ceq->polarity; 3952 __le64 *ceqe; 3953 u8 polarity; 3954 u64 temp; 3955 int next; 3956 u32 i; 3957 3958 next = IRDMA_RING_GET_NEXT_TAIL(ceq->ceq_ring, 0); 3959 3960 for (i = 1; i <= IRDMA_RING_SIZE(*ceq); i++) { 3961 ceqe = IRDMA_GET_CEQ_ELEM_AT_POS(ceq, next); 3962 3963 get_64bit_val(ceqe, IRDMA_BYTE_0, &temp); 3964 polarity = (u8)FIELD_GET(IRDMA_CEQE_VALID, temp); 3965 if (polarity != ceq_polarity) 3966 return; 3967 3968 next_cq = (struct irdma_sc_cq *)(irdma_uintptr) LS_64_1(temp, 1); 3969 if (cq == next_cq) 3970 set_64bit_val(ceqe, IRDMA_BYTE_0, temp & IRDMA_CEQE_VALID); 3971 3972 next = IRDMA_RING_GET_NEXT_TAIL(ceq->ceq_ring, i); 3973 if (!next) 3974 ceq_polarity ^= 1; 3975 } 3976 } 3977 3978 /** 3979 * irdma_sc_aeq_init - initialize aeq 3980 * @aeq: aeq structure ptr 3981 * @info: aeq initialization info 3982 */ 3983 int 3984 irdma_sc_aeq_init(struct irdma_sc_aeq *aeq, 3985 struct irdma_aeq_init_info *info) 3986 { 3987 u32 pble_obj_cnt; 3988 3989 if (info->elem_cnt < info->dev->hw_attrs.min_hw_aeq_size || 3990 info->elem_cnt > info->dev->hw_attrs.max_hw_aeq_size) 3991 return -EINVAL; 3992 3993 pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; 3994 3995 if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt) 3996 return -EINVAL; 3997 3998 aeq->size = sizeof(*aeq); 3999 aeq->polarity = 1; 4000 aeq->aeqe_base = (struct irdma_sc_aeqe *)info->aeqe_base; 4001 aeq->dev = info->dev; 4002 aeq->elem_cnt = info->elem_cnt; 4003 aeq->aeq_elem_pa = info->aeq_elem_pa; 4004 IRDMA_RING_INIT(aeq->aeq_ring, aeq->elem_cnt); 4005 aeq->virtual_map = info->virtual_map; 4006 aeq->pbl_list = (aeq->virtual_map ? info->pbl_list : NULL); 4007 aeq->pbl_chunk_size = (aeq->virtual_map ? info->pbl_chunk_size : 0); 4008 aeq->first_pm_pbl_idx = (aeq->virtual_map ? info->first_pm_pbl_idx : 0); 4009 aeq->msix_idx = info->msix_idx; 4010 info->dev->aeq = aeq; 4011 4012 return 0; 4013 } 4014 4015 /** 4016 * irdma_sc_aeq_create - create aeq 4017 * @aeq: aeq structure ptr 4018 * @scratch: u64 saved to be used during cqp completion 4019 * @post_sq: flag for cqp db to ring 4020 */ 4021 static int 4022 irdma_sc_aeq_create(struct irdma_sc_aeq *aeq, u64 scratch, 4023 bool post_sq) 4024 { 4025 __le64 *wqe; 4026 struct irdma_sc_cqp *cqp; 4027 u64 hdr; 4028 4029 cqp = aeq->dev->cqp; 4030 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 4031 if (!wqe) 4032 return -ENOSPC; 4033 set_64bit_val(wqe, IRDMA_BYTE_16, aeq->elem_cnt); 4034 set_64bit_val(wqe, IRDMA_BYTE_32, 4035 (aeq->virtual_map ? 0 : aeq->aeq_elem_pa)); 4036 set_64bit_val(wqe, IRDMA_BYTE_48, 4037 (aeq->virtual_map ? aeq->first_pm_pbl_idx : 0)); 4038 4039 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_AEQ) | 4040 FIELD_PREP(IRDMA_CQPSQ_AEQ_LPBLSIZE, aeq->pbl_chunk_size) | 4041 FIELD_PREP(IRDMA_CQPSQ_AEQ_VMAP, aeq->virtual_map) | 4042 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 4043 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 4044 4045 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 4046 4047 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "AEQ_CREATE WQE", wqe, 4048 IRDMA_CQP_WQE_SIZE * 8); 4049 if (post_sq) 4050 irdma_sc_cqp_post_sq(cqp); 4051 4052 return 0; 4053 } 4054 4055 /** 4056 * irdma_sc_aeq_destroy - destroy aeq during close 4057 * @aeq: aeq structure ptr 4058 * @scratch: u64 saved to be used during cqp completion 4059 * @post_sq: flag for cqp db to ring 4060 */ 4061 int 4062 irdma_sc_aeq_destroy(struct irdma_sc_aeq *aeq, u64 scratch, bool post_sq) 4063 { 4064 __le64 *wqe; 4065 struct irdma_sc_cqp *cqp; 4066 struct irdma_sc_dev *dev; 4067 u64 hdr; 4068 4069 dev = aeq->dev; 4070 writel(0, dev->hw_regs[IRDMA_PFINT_AEQCTL]); 4071 4072 cqp = dev->cqp; 4073 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 4074 if (!wqe) 4075 return -ENOSPC; 4076 set_64bit_val(wqe, IRDMA_BYTE_16, aeq->elem_cnt); 4077 set_64bit_val(wqe, IRDMA_BYTE_48, aeq->first_pm_pbl_idx); 4078 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_AEQ) | 4079 FIELD_PREP(IRDMA_CQPSQ_AEQ_LPBLSIZE, aeq->pbl_chunk_size) | 4080 FIELD_PREP(IRDMA_CQPSQ_AEQ_VMAP, aeq->virtual_map) | 4081 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 4082 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 4083 4084 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 4085 4086 irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "AEQ_DESTROY WQE", wqe, 4087 IRDMA_CQP_WQE_SIZE * 8); 4088 if (post_sq) 4089 irdma_sc_cqp_post_sq(cqp); 4090 return 0; 4091 } 4092 4093 /** 4094 * irdma_sc_get_next_aeqe - get next aeq entry 4095 * @aeq: aeq structure ptr 4096 * @info: aeqe info to be returned 4097 */ 4098 int 4099 irdma_sc_get_next_aeqe(struct irdma_sc_aeq *aeq, 4100 struct irdma_aeqe_info *info) 4101 { 4102 u64 temp, compl_ctx; 4103 __le64 *aeqe; 4104 u8 ae_src; 4105 u8 polarity; 4106 4107 aeqe = IRDMA_GET_CURRENT_AEQ_ELEM(aeq); 4108 get_64bit_val(aeqe, IRDMA_BYTE_8, &temp); 4109 polarity = (u8)FIELD_GET(IRDMA_AEQE_VALID, temp); 4110 4111 if (aeq->polarity != polarity) 4112 return -ENOENT; 4113 4114 /* Ensure AEQE contents are read after valid bit is checked */ 4115 rmb(); 4116 4117 get_64bit_val(aeqe, IRDMA_BYTE_0, &compl_ctx); 4118 4119 irdma_debug_buf(aeq->dev, IRDMA_DEBUG_WQE, "AEQ_ENTRY WQE", aeqe, 16); 4120 4121 ae_src = (u8)FIELD_GET(IRDMA_AEQE_AESRC, temp); 4122 info->wqe_idx = (u16)FIELD_GET(IRDMA_AEQE_WQDESCIDX, temp); 4123 info->qp_cq_id = (u32)FIELD_GET(IRDMA_AEQE_QPCQID_LOW, temp) | 4124 ((u32)FIELD_GET(IRDMA_AEQE_QPCQID_HI, temp) << 18); 4125 info->ae_id = (u16)FIELD_GET(IRDMA_AEQE_AECODE, temp); 4126 info->tcp_state = (u8)FIELD_GET(IRDMA_AEQE_TCPSTATE, temp); 4127 info->iwarp_state = (u8)FIELD_GET(IRDMA_AEQE_IWSTATE, temp); 4128 info->q2_data_written = (u8)FIELD_GET(IRDMA_AEQE_Q2DATA, temp); 4129 info->aeqe_overflow = (bool)FIELD_GET(IRDMA_AEQE_OVERFLOW, temp); 4130 4131 info->ae_src = ae_src; 4132 switch (info->ae_id) { 4133 case IRDMA_AE_PRIV_OPERATION_DENIED: 4134 case IRDMA_AE_AMP_INVALIDATE_TYPE1_MW: 4135 case IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW: 4136 case IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG: 4137 case IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH: 4138 case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG: 4139 case IRDMA_AE_UDA_XMIT_BAD_PD: 4140 case IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT: 4141 case IRDMA_AE_BAD_CLOSE: 4142 case IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO: 4143 case IRDMA_AE_STAG_ZERO_INVALID: 4144 case IRDMA_AE_IB_RREQ_AND_Q1_FULL: 4145 case IRDMA_AE_IB_INVALID_REQUEST: 4146 case IRDMA_AE_WQE_UNEXPECTED_OPCODE: 4147 case IRDMA_AE_IB_REMOTE_ACCESS_ERROR: 4148 case IRDMA_AE_IB_REMOTE_OP_ERROR: 4149 case IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION: 4150 case IRDMA_AE_DDP_UBE_INVALID_MO: 4151 case IRDMA_AE_DDP_UBE_INVALID_QN: 4152 case IRDMA_AE_DDP_NO_L_BIT: 4153 case IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION: 4154 case IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE: 4155 case IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST: 4156 case IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP: 4157 case IRDMA_AE_ROCE_RSP_LENGTH_ERROR: 4158 case IRDMA_AE_INVALID_ARP_ENTRY: 4159 case IRDMA_AE_INVALID_TCP_OPTION_RCVD: 4160 case IRDMA_AE_STALE_ARP_ENTRY: 4161 case IRDMA_AE_INVALID_AH_ENTRY: 4162 case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR: 4163 case IRDMA_AE_LLP_SEGMENT_TOO_SMALL: 4164 case IRDMA_AE_LLP_TOO_MANY_RETRIES: 4165 case IRDMA_AE_LCE_QP_CATASTROPHIC: 4166 case IRDMA_AE_LLP_DOUBT_REACHABILITY: 4167 case IRDMA_AE_LLP_CONNECTION_ESTABLISHED: 4168 case IRDMA_AE_RESET_SENT: 4169 case IRDMA_AE_TERMINATE_SENT: 4170 case IRDMA_AE_RESET_NOT_SENT: 4171 case IRDMA_AE_QP_SUSPEND_COMPLETE: 4172 case IRDMA_AE_UDA_L4LEN_INVALID: 4173 info->qp = true; 4174 info->compl_ctx = compl_ctx; 4175 break; 4176 case IRDMA_AE_LCE_CQ_CATASTROPHIC: 4177 info->cq = true; 4178 info->compl_ctx = LS_64_1(compl_ctx, 1); 4179 ae_src = IRDMA_AE_SOURCE_RSVD; 4180 break; 4181 case IRDMA_AE_ROCE_EMPTY_MCG: 4182 case IRDMA_AE_ROCE_BAD_MC_IP_ADDR: 4183 case IRDMA_AE_ROCE_BAD_MC_QPID: 4184 case IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH: 4185 /* fallthrough */ 4186 case IRDMA_AE_LLP_CONNECTION_RESET: 4187 case IRDMA_AE_LLP_SYN_RECEIVED: 4188 case IRDMA_AE_LLP_FIN_RECEIVED: 4189 case IRDMA_AE_LLP_CLOSE_COMPLETE: 4190 case IRDMA_AE_LLP_TERMINATE_RECEIVED: 4191 case IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE: 4192 ae_src = IRDMA_AE_SOURCE_RSVD; 4193 info->qp = true; 4194 info->compl_ctx = compl_ctx; 4195 break; 4196 case IRDMA_AE_RESOURCE_EXHAUSTION: 4197 /* 4198 * ae_src contains the exhausted resource with a unique decoding. Set RSVD here to prevent matching 4199 * with a CQ or QP. 4200 */ 4201 ae_src = IRDMA_AE_SOURCE_RSVD; 4202 break; 4203 default: 4204 break; 4205 } 4206 4207 switch (ae_src) { 4208 case IRDMA_AE_SOURCE_RQ: 4209 case IRDMA_AE_SOURCE_RQ_0011: 4210 info->qp = true; 4211 info->rq = true; 4212 info->compl_ctx = compl_ctx; 4213 info->err_rq_idx_valid = true; 4214 break; 4215 case IRDMA_AE_SOURCE_CQ: 4216 case IRDMA_AE_SOURCE_CQ_0110: 4217 case IRDMA_AE_SOURCE_CQ_1010: 4218 case IRDMA_AE_SOURCE_CQ_1110: 4219 info->cq = true; 4220 info->compl_ctx = LS_64_1(compl_ctx, 1); 4221 break; 4222 case IRDMA_AE_SOURCE_SQ: 4223 case IRDMA_AE_SOURCE_SQ_0111: 4224 info->qp = true; 4225 info->sq = true; 4226 info->compl_ctx = compl_ctx; 4227 break; 4228 case IRDMA_AE_SOURCE_IN_WR: 4229 info->qp = true; 4230 info->compl_ctx = compl_ctx; 4231 info->in_rdrsp_wr = true; 4232 break; 4233 case IRDMA_AE_SOURCE_IN_RR: 4234 info->qp = true; 4235 info->compl_ctx = compl_ctx; 4236 info->in_rdrsp_wr = true; 4237 break; 4238 case IRDMA_AE_SOURCE_OUT_RR: 4239 case IRDMA_AE_SOURCE_OUT_RR_1111: 4240 info->qp = true; 4241 info->compl_ctx = compl_ctx; 4242 info->out_rdrsp = true; 4243 break; 4244 case IRDMA_AE_SOURCE_RSVD: 4245 default: 4246 break; 4247 } 4248 4249 IRDMA_RING_MOVE_TAIL(aeq->aeq_ring); 4250 if (!IRDMA_RING_CURRENT_TAIL(aeq->aeq_ring)) 4251 aeq->polarity ^= 1; 4252 4253 return 0; 4254 } 4255 4256 /** 4257 * irdma_sc_repost_aeq_entries - repost completed aeq entries 4258 * @dev: sc device struct 4259 * @count: allocate count 4260 */ 4261 void 4262 irdma_sc_repost_aeq_entries(struct irdma_sc_dev *dev, u32 count) 4263 { 4264 db_wr32(count, dev->aeq_alloc_db); 4265 4266 } 4267 4268 /** 4269 * irdma_sc_ccq_init - initialize control cq 4270 * @cq: sc's cq ctruct 4271 * @info: info for control cq initialization 4272 */ 4273 int 4274 irdma_sc_ccq_init(struct irdma_sc_cq *cq, struct irdma_ccq_init_info *info) 4275 { 4276 u32 pble_obj_cnt; 4277 4278 if (info->num_elem < info->dev->hw_attrs.uk_attrs.min_hw_cq_size || 4279 info->num_elem > info->dev->hw_attrs.uk_attrs.max_hw_cq_size) 4280 return -EINVAL; 4281 4282 if (info->ceq_id > (info->dev->hmc_fpm_misc.max_ceqs - 1)) 4283 return -EINVAL; 4284 4285 pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; 4286 4287 if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt) 4288 return -EINVAL; 4289 4290 cq->cq_pa = info->cq_pa; 4291 cq->cq_uk.cq_base = info->cq_base; 4292 cq->shadow_area_pa = info->shadow_area_pa; 4293 cq->cq_uk.shadow_area = info->shadow_area; 4294 cq->shadow_read_threshold = info->shadow_read_threshold; 4295 cq->dev = info->dev; 4296 cq->ceq_id = info->ceq_id; 4297 cq->cq_uk.cq_size = info->num_elem; 4298 cq->cq_type = IRDMA_CQ_TYPE_CQP; 4299 cq->ceqe_mask = info->ceqe_mask; 4300 IRDMA_RING_INIT(cq->cq_uk.cq_ring, info->num_elem); 4301 cq->cq_uk.cq_id = 0; /* control cq is id 0 always */ 4302 cq->ceq_id_valid = info->ceq_id_valid; 4303 cq->tph_en = info->tph_en; 4304 cq->tph_val = info->tph_val; 4305 cq->cq_uk.avoid_mem_cflct = info->avoid_mem_cflct; 4306 cq->pbl_list = info->pbl_list; 4307 cq->virtual_map = info->virtual_map; 4308 cq->pbl_chunk_size = info->pbl_chunk_size; 4309 cq->first_pm_pbl_idx = info->first_pm_pbl_idx; 4310 cq->cq_uk.polarity = true; 4311 cq->vsi = info->vsi; 4312 cq->cq_uk.cq_ack_db = cq->dev->cq_ack_db; 4313 4314 /* Only applicable to CQs other than CCQ so initialize to zero */ 4315 cq->cq_uk.cqe_alloc_db = NULL; 4316 4317 info->dev->ccq = cq; 4318 return 0; 4319 } 4320 4321 /** 4322 * irdma_sc_ccq_create_done - poll cqp for ccq create 4323 * @ccq: ccq sc struct 4324 */ 4325 static inline int 4326 irdma_sc_ccq_create_done(struct irdma_sc_cq *ccq) 4327 { 4328 struct irdma_sc_cqp *cqp; 4329 4330 cqp = ccq->dev->cqp; 4331 4332 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_CREATE_CQ, NULL); 4333 } 4334 4335 /** 4336 * irdma_sc_ccq_create - create control cq 4337 * @ccq: ccq sc struct 4338 * @scratch: u64 saved to be used during cqp completion 4339 * @check_overflow: overlow flag for ccq 4340 * @post_sq: flag for cqp db to ring 4341 */ 4342 int 4343 irdma_sc_ccq_create(struct irdma_sc_cq *ccq, u64 scratch, 4344 bool check_overflow, bool post_sq) 4345 { 4346 int ret_code; 4347 4348 ret_code = irdma_sc_cq_create(ccq, scratch, check_overflow, post_sq); 4349 if (ret_code) 4350 return ret_code; 4351 4352 if (post_sq) { 4353 ret_code = irdma_sc_ccq_create_done(ccq); 4354 if (ret_code) 4355 return ret_code; 4356 } 4357 ccq->dev->cqp->process_cqp_sds = irdma_cqp_sds_cmd; 4358 4359 return 0; 4360 } 4361 4362 /** 4363 * irdma_sc_ccq_destroy - destroy ccq during close 4364 * @ccq: ccq sc struct 4365 * @scratch: u64 saved to be used during cqp completion 4366 * @post_sq: flag for cqp db to ring 4367 */ 4368 int 4369 irdma_sc_ccq_destroy(struct irdma_sc_cq *ccq, u64 scratch, bool post_sq) 4370 { 4371 struct irdma_sc_cqp *cqp; 4372 __le64 *wqe; 4373 u64 hdr; 4374 int ret_code = 0; 4375 u32 tail, val, error; 4376 4377 cqp = ccq->dev->cqp; 4378 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 4379 if (!wqe) 4380 return -ENOSPC; 4381 4382 set_64bit_val(wqe, IRDMA_BYTE_0, ccq->cq_uk.cq_size); 4383 set_64bit_val(wqe, IRDMA_BYTE_8, RS_64_1(ccq, 1)); 4384 set_64bit_val(wqe, IRDMA_BYTE_40, ccq->shadow_area_pa); 4385 4386 hdr = ccq->cq_uk.cq_id | 4387 FLD_LS_64(ccq->dev, (ccq->ceq_id_valid ? ccq->ceq_id : 0), 4388 IRDMA_CQPSQ_CQ_CEQID) | 4389 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_CQ) | 4390 FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, ccq->ceqe_mask) | 4391 FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, ccq->ceq_id_valid) | 4392 FIELD_PREP(IRDMA_CQPSQ_TPHEN, ccq->tph_en) | 4393 FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT, ccq->cq_uk.avoid_mem_cflct) | 4394 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 4395 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 4396 4397 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 4398 4399 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "CCQ_DESTROY WQE", wqe, 4400 IRDMA_CQP_WQE_SIZE * 8); 4401 irdma_get_cqp_reg_info(cqp, &val, &tail, &error); 4402 4403 if (post_sq) { 4404 irdma_sc_cqp_post_sq(cqp); 4405 ret_code = irdma_cqp_poll_registers(cqp, tail, 4406 cqp->dev->hw_attrs.max_done_count); 4407 } 4408 4409 cqp->process_cqp_sds = irdma_update_sds_noccq; 4410 4411 return ret_code; 4412 } 4413 4414 /** 4415 * irdma_sc_init_iw_hmc() - queries fpm values using cqp and populates hmc_info 4416 * @dev : ptr to irdma_dev struct 4417 * @hmc_fn_id: hmc function id 4418 */ 4419 int 4420 irdma_sc_init_iw_hmc(struct irdma_sc_dev *dev, u16 hmc_fn_id) 4421 { 4422 struct irdma_hmc_info *hmc_info; 4423 struct irdma_hmc_fpm_misc *hmc_fpm_misc; 4424 struct irdma_dma_mem query_fpm_mem; 4425 int ret_code = 0; 4426 u8 wait_type; 4427 4428 hmc_info = dev->hmc_info; 4429 hmc_fpm_misc = &dev->hmc_fpm_misc; 4430 query_fpm_mem.pa = dev->fpm_query_buf_pa; 4431 query_fpm_mem.va = dev->fpm_query_buf; 4432 hmc_info->hmc_fn_id = hmc_fn_id; 4433 wait_type = (u8)IRDMA_CQP_WAIT_POLL_REGS; 4434 4435 ret_code = irdma_sc_query_fpm_val(dev->cqp, 0, hmc_info->hmc_fn_id, 4436 &query_fpm_mem, true, wait_type); 4437 if (ret_code) 4438 return ret_code; 4439 4440 /* parse the fpm_query_buf and fill hmc obj info */ 4441 ret_code = irdma_sc_parse_fpm_query_buf(dev, query_fpm_mem.va, hmc_info, 4442 hmc_fpm_misc); 4443 4444 irdma_debug_buf(dev, IRDMA_DEBUG_HMC, "QUERY FPM BUFFER", 4445 query_fpm_mem.va, IRDMA_QUERY_FPM_BUF_SIZE); 4446 return ret_code; 4447 } 4448 4449 /** 4450 * irdma_sc_cfg_iw_fpm() - commits hmc obj cnt values using cqp 4451 * command and populates fpm base address in hmc_info 4452 * @dev : ptr to irdma_dev struct 4453 * @hmc_fn_id: hmc function id 4454 */ 4455 static int 4456 irdma_sc_cfg_iw_fpm(struct irdma_sc_dev *dev, u16 hmc_fn_id) 4457 { 4458 struct irdma_hmc_obj_info *obj_info; 4459 __le64 *buf; 4460 struct irdma_hmc_info *hmc_info; 4461 struct irdma_dma_mem commit_fpm_mem; 4462 int ret_code = 0; 4463 u8 wait_type; 4464 4465 hmc_info = dev->hmc_info; 4466 obj_info = hmc_info->hmc_obj; 4467 buf = dev->fpm_commit_buf; 4468 4469 set_64bit_val(buf, IRDMA_BYTE_0, (u64)obj_info[IRDMA_HMC_IW_QP].cnt); 4470 set_64bit_val(buf, IRDMA_BYTE_8, (u64)obj_info[IRDMA_HMC_IW_CQ].cnt); 4471 set_64bit_val(buf, IRDMA_BYTE_16, (u64)0); /* RSRVD */ 4472 set_64bit_val(buf, IRDMA_BYTE_24, (u64)obj_info[IRDMA_HMC_IW_HTE].cnt); 4473 set_64bit_val(buf, IRDMA_BYTE_32, (u64)obj_info[IRDMA_HMC_IW_ARP].cnt); 4474 set_64bit_val(buf, IRDMA_BYTE_40, (u64)0); /* RSVD */ 4475 set_64bit_val(buf, IRDMA_BYTE_48, (u64)obj_info[IRDMA_HMC_IW_MR].cnt); 4476 set_64bit_val(buf, IRDMA_BYTE_56, (u64)obj_info[IRDMA_HMC_IW_XF].cnt); 4477 set_64bit_val(buf, IRDMA_BYTE_64, (u64)obj_info[IRDMA_HMC_IW_XFFL].cnt); 4478 set_64bit_val(buf, IRDMA_BYTE_72, (u64)obj_info[IRDMA_HMC_IW_Q1].cnt); 4479 set_64bit_val(buf, IRDMA_BYTE_80, (u64)obj_info[IRDMA_HMC_IW_Q1FL].cnt); 4480 set_64bit_val(buf, IRDMA_BYTE_88, 4481 (u64)obj_info[IRDMA_HMC_IW_TIMER].cnt); 4482 set_64bit_val(buf, IRDMA_BYTE_96, 4483 (u64)obj_info[IRDMA_HMC_IW_FSIMC].cnt); 4484 set_64bit_val(buf, IRDMA_BYTE_104, 4485 (u64)obj_info[IRDMA_HMC_IW_FSIAV].cnt); 4486 set_64bit_val(buf, IRDMA_BYTE_112, 4487 (u64)obj_info[IRDMA_HMC_IW_PBLE].cnt); 4488 set_64bit_val(buf, IRDMA_BYTE_120, (u64)0); /* RSVD */ 4489 set_64bit_val(buf, IRDMA_BYTE_128, (u64)obj_info[IRDMA_HMC_IW_RRF].cnt); 4490 set_64bit_val(buf, IRDMA_BYTE_136, 4491 (u64)obj_info[IRDMA_HMC_IW_RRFFL].cnt); 4492 set_64bit_val(buf, IRDMA_BYTE_144, (u64)obj_info[IRDMA_HMC_IW_HDR].cnt); 4493 set_64bit_val(buf, IRDMA_BYTE_152, (u64)obj_info[IRDMA_HMC_IW_MD].cnt); 4494 set_64bit_val(buf, IRDMA_BYTE_160, 4495 (u64)obj_info[IRDMA_HMC_IW_OOISC].cnt); 4496 set_64bit_val(buf, IRDMA_BYTE_168, 4497 (u64)obj_info[IRDMA_HMC_IW_OOISCFFL].cnt); 4498 commit_fpm_mem.pa = dev->fpm_commit_buf_pa; 4499 commit_fpm_mem.va = dev->fpm_commit_buf; 4500 4501 wait_type = (u8)IRDMA_CQP_WAIT_POLL_REGS; 4502 irdma_debug_buf(dev, IRDMA_DEBUG_HMC, "COMMIT FPM BUFFER", 4503 commit_fpm_mem.va, IRDMA_COMMIT_FPM_BUF_SIZE); 4504 ret_code = irdma_sc_commit_fpm_val(dev->cqp, 0, hmc_info->hmc_fn_id, 4505 &commit_fpm_mem, true, wait_type); 4506 if (!ret_code) 4507 irdma_sc_parse_fpm_commit_buf(dev, dev->fpm_commit_buf, 4508 hmc_info->hmc_obj, 4509 &hmc_info->sd_table.sd_cnt); 4510 irdma_debug_buf(dev, IRDMA_DEBUG_HMC, "COMMIT FPM BUFFER", 4511 commit_fpm_mem.va, IRDMA_COMMIT_FPM_BUF_SIZE); 4512 4513 return ret_code; 4514 } 4515 4516 /** 4517 * cqp_sds_wqe_fill - fill cqp wqe doe sd 4518 * @cqp: struct for cqp hw 4519 * @info: sd info for wqe 4520 * @scratch: u64 saved to be used during cqp completion 4521 */ 4522 static int 4523 cqp_sds_wqe_fill(struct irdma_sc_cqp *cqp, 4524 struct irdma_update_sds_info *info, u64 scratch) 4525 { 4526 u64 data; 4527 u64 hdr; 4528 __le64 *wqe; 4529 int mem_entries, wqe_entries; 4530 struct irdma_dma_mem *sdbuf = &cqp->sdbuf; 4531 u64 offset = 0; 4532 u32 wqe_idx; 4533 4534 wqe = irdma_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx); 4535 if (!wqe) 4536 return -ENOSPC; 4537 4538 wqe_entries = (info->cnt > 3) ? 3 : info->cnt; 4539 mem_entries = info->cnt - wqe_entries; 4540 4541 if (mem_entries) { 4542 offset = wqe_idx * IRDMA_UPDATE_SD_BUFF_SIZE; 4543 irdma_memcpy(((char *)sdbuf->va + offset), &info->entry[3], mem_entries << 4); 4544 4545 data = (u64)sdbuf->pa + offset; 4546 } else { 4547 data = 0; 4548 } 4549 data |= FLD_LS_64(cqp->dev, info->hmc_fn_id, IRDMA_CQPSQ_UPESD_HMCFNID); 4550 set_64bit_val(wqe, IRDMA_BYTE_16, data); 4551 4552 switch (wqe_entries) { 4553 case 3: 4554 set_64bit_val(wqe, IRDMA_BYTE_48, 4555 (FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info->entry[2].cmd) | 4556 FIELD_PREP(IRDMA_CQPSQ_UPESD_ENTRY_VALID, 1))); 4557 4558 set_64bit_val(wqe, IRDMA_BYTE_56, info->entry[2].data); 4559 /* fallthrough */ 4560 case 2: 4561 set_64bit_val(wqe, IRDMA_BYTE_32, 4562 (FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info->entry[1].cmd) | 4563 FIELD_PREP(IRDMA_CQPSQ_UPESD_ENTRY_VALID, 1))); 4564 4565 set_64bit_val(wqe, IRDMA_BYTE_40, info->entry[1].data); 4566 /* fallthrough */ 4567 case 1: 4568 set_64bit_val(wqe, IRDMA_BYTE_0, 4569 FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info->entry[0].cmd)); 4570 4571 set_64bit_val(wqe, IRDMA_BYTE_8, info->entry[0].data); 4572 break; 4573 default: 4574 break; 4575 } 4576 4577 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_UPDATE_PE_SDS) | 4578 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) | 4579 FIELD_PREP(IRDMA_CQPSQ_UPESD_ENTRY_COUNT, mem_entries); 4580 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 4581 4582 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 4583 4584 if (mem_entries) 4585 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "UPDATE_PE_SDS WQE Buffer", 4586 (char *)sdbuf->va + offset, mem_entries << 4); 4587 4588 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "UPDATE_PE_SDS WQE", wqe, 4589 IRDMA_CQP_WQE_SIZE * 8); 4590 4591 return 0; 4592 } 4593 4594 /** 4595 * irdma_update_pe_sds - cqp wqe for sd 4596 * @dev: ptr to irdma_dev struct 4597 * @info: sd info for sd's 4598 * @scratch: u64 saved to be used during cqp completion 4599 */ 4600 static int 4601 irdma_update_pe_sds(struct irdma_sc_dev *dev, 4602 struct irdma_update_sds_info *info, u64 scratch) 4603 { 4604 struct irdma_sc_cqp *cqp = dev->cqp; 4605 int ret_code; 4606 4607 ret_code = cqp_sds_wqe_fill(cqp, info, scratch); 4608 if (!ret_code) 4609 irdma_sc_cqp_post_sq(cqp); 4610 4611 return ret_code; 4612 } 4613 4614 /** 4615 * irdma_update_sds_noccq - update sd before ccq created 4616 * @dev: sc device struct 4617 * @info: sd info for sd's 4618 */ 4619 int 4620 irdma_update_sds_noccq(struct irdma_sc_dev *dev, 4621 struct irdma_update_sds_info *info) 4622 { 4623 u32 error, val, tail; 4624 struct irdma_sc_cqp *cqp = dev->cqp; 4625 int ret_code; 4626 4627 ret_code = cqp_sds_wqe_fill(cqp, info, 0); 4628 if (ret_code) 4629 return ret_code; 4630 4631 irdma_get_cqp_reg_info(cqp, &val, &tail, &error); 4632 4633 irdma_sc_cqp_post_sq(cqp); 4634 return irdma_cqp_poll_registers(cqp, tail, 4635 cqp->dev->hw_attrs.max_done_count); 4636 } 4637 4638 /** 4639 * irdma_sc_static_hmc_pages_allocated - cqp wqe to allocate hmc pages 4640 * @cqp: struct for cqp hw 4641 * @scratch: u64 saved to be used during cqp completion 4642 * @hmc_fn_id: hmc function id 4643 * @post_sq: flag for cqp db to ring 4644 * @poll_registers: flag to poll register for cqp completion 4645 */ 4646 int 4647 irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch, 4648 u16 hmc_fn_id, bool post_sq, 4649 bool poll_registers) 4650 { 4651 u64 hdr; 4652 __le64 *wqe; 4653 u32 tail, val, error; 4654 4655 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 4656 if (!wqe) 4657 return -ENOSPC; 4658 4659 set_64bit_val(wqe, IRDMA_BYTE_16, 4660 FIELD_PREP(IRDMA_SHMC_PAGE_ALLOCATED_HMC_FN_ID, hmc_fn_id)); 4661 4662 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, 4663 IRDMA_CQP_OP_SHMC_PAGES_ALLOCATED) | 4664 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 4665 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 4666 4667 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 4668 4669 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "SHMC_PAGES_ALLOCATED WQE", 4670 wqe, IRDMA_CQP_WQE_SIZE * 8); 4671 irdma_get_cqp_reg_info(cqp, &val, &tail, &error); 4672 4673 if (post_sq) { 4674 irdma_sc_cqp_post_sq(cqp); 4675 if (poll_registers) 4676 /* check for cqp sq tail update */ 4677 return irdma_cqp_poll_registers(cqp, tail, 4678 cqp->dev->hw_attrs.max_done_count); 4679 else 4680 return irdma_sc_poll_for_cqp_op_done(cqp, 4681 IRDMA_CQP_OP_SHMC_PAGES_ALLOCATED, 4682 NULL); 4683 } 4684 4685 return 0; 4686 } 4687 4688 /** 4689 * irdma_cqp_ring_full - check if cqp ring is full 4690 * @cqp: struct for cqp hw 4691 */ 4692 static bool 4693 irdma_cqp_ring_full(struct irdma_sc_cqp *cqp) 4694 { 4695 return IRDMA_RING_FULL_ERR(cqp->sq_ring); 4696 } 4697 4698 /** 4699 * irdma_est_sd - returns approximate number of SDs for HMC 4700 * @dev: sc device struct 4701 * @hmc_info: hmc structure, size and count for HMC objects 4702 */ 4703 static u32 irdma_est_sd(struct irdma_sc_dev *dev, 4704 struct irdma_hmc_info *hmc_info){ 4705 struct irdma_hmc_obj_info *pble_info; 4706 u64 size = 0; 4707 u64 sd; 4708 int i; 4709 4710 for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++) 4711 if (i != IRDMA_HMC_IW_PBLE) 4712 size += round_up(hmc_info->hmc_obj[i].cnt * 4713 hmc_info->hmc_obj[i].size, 512); 4714 4715 pble_info = &hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE]; 4716 size += round_up(pble_info->cnt * pble_info->size, 512); 4717 if (size & 0x1FFFFF) 4718 sd = (size >> 21) + 1; /* add 1 for remainder */ 4719 else 4720 sd = size >> 21; 4721 if (sd > 0xFFFFFFFF) { 4722 irdma_debug(dev, IRDMA_DEBUG_HMC, "sd overflow[%ld]\n", sd); 4723 sd = 0xFFFFFFFF - 1; 4724 } 4725 4726 return (u32)sd; 4727 } 4728 4729 /** 4730 * irdma_sc_query_rdma_features - query RDMA features and FW ver 4731 * @cqp: struct for cqp hw 4732 * @buf: buffer to hold query info 4733 * @scratch: u64 saved to be used during cqp completion 4734 */ 4735 static int 4736 irdma_sc_query_rdma_features(struct irdma_sc_cqp *cqp, 4737 struct irdma_dma_mem *buf, u64 scratch) 4738 { 4739 __le64 *wqe; 4740 u64 temp; 4741 u32 tail, val, error; 4742 int status; 4743 4744 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 4745 if (!wqe) 4746 return -ENOSPC; 4747 4748 temp = buf->pa; 4749 set_64bit_val(wqe, IRDMA_BYTE_32, temp); 4750 4751 temp = FIELD_PREP(IRDMA_CQPSQ_QUERY_RDMA_FEATURES_WQEVALID, 4752 cqp->polarity) | 4753 FIELD_PREP(IRDMA_CQPSQ_QUERY_RDMA_FEATURES_BUF_LEN, buf->size) | 4754 FIELD_PREP(IRDMA_CQPSQ_UP_OP, IRDMA_CQP_OP_QUERY_RDMA_FEATURES); 4755 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 4756 4757 set_64bit_val(wqe, IRDMA_BYTE_24, temp); 4758 4759 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "QUERY RDMA FEATURES", wqe, 4760 IRDMA_CQP_WQE_SIZE * 8); 4761 irdma_get_cqp_reg_info(cqp, &val, &tail, &error); 4762 4763 irdma_sc_cqp_post_sq(cqp); 4764 status = irdma_cqp_poll_registers(cqp, tail, 4765 cqp->dev->hw_attrs.max_done_count); 4766 if (error || status) 4767 status = -EIO; 4768 4769 return status; 4770 } 4771 4772 /** 4773 * irdma_get_rdma_features - get RDMA features 4774 * @dev: sc device struct 4775 */ 4776 int 4777 irdma_get_rdma_features(struct irdma_sc_dev *dev) 4778 { 4779 int ret_code; 4780 struct irdma_dma_mem feat_buf; 4781 u64 temp; 4782 u16 byte_idx, feat_type, feat_cnt, feat_idx; 4783 4784 feat_buf.size = IRDMA_FEATURE_BUF_SIZE; 4785 feat_buf.va = irdma_allocate_dma_mem(dev->hw, &feat_buf, feat_buf.size, 4786 IRDMA_FEATURE_BUF_ALIGNMENT); 4787 if (!feat_buf.va) 4788 return -ENOMEM; 4789 4790 ret_code = irdma_sc_query_rdma_features(dev->cqp, &feat_buf, 0); 4791 if (ret_code) 4792 goto exit; 4793 4794 get_64bit_val(feat_buf.va, IRDMA_BYTE_0, &temp); 4795 feat_cnt = (u16)FIELD_GET(IRDMA_FEATURE_CNT, temp); 4796 if (feat_cnt < IRDMA_MIN_FEATURES) { 4797 ret_code = -EINVAL; 4798 goto exit; 4799 } else if (feat_cnt > IRDMA_MAX_FEATURES) { 4800 irdma_debug(dev, IRDMA_DEBUG_DEV, 4801 "feature buf size insufficient, retrying with larger buffer\n"); 4802 irdma_free_dma_mem(dev->hw, &feat_buf); 4803 feat_buf.size = 8 * feat_cnt; 4804 feat_buf.va = irdma_allocate_dma_mem(dev->hw, &feat_buf, 4805 feat_buf.size, 4806 IRDMA_FEATURE_BUF_ALIGNMENT); 4807 if (!feat_buf.va) 4808 return -ENOMEM; 4809 4810 ret_code = irdma_sc_query_rdma_features(dev->cqp, &feat_buf, 0); 4811 if (ret_code) 4812 goto exit; 4813 4814 get_64bit_val(feat_buf.va, IRDMA_BYTE_0, &temp); 4815 feat_cnt = (u16)FIELD_GET(IRDMA_FEATURE_CNT, temp); 4816 if (feat_cnt < IRDMA_MIN_FEATURES) { 4817 ret_code = -EINVAL; 4818 goto exit; 4819 } 4820 } 4821 4822 irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "QUERY RDMA FEATURES", feat_buf.va, 4823 feat_cnt * 8); 4824 4825 for (byte_idx = 0, feat_idx = 0; feat_idx < min(feat_cnt, (u16)IRDMA_MAX_FEATURES); 4826 feat_idx++, byte_idx += 8) { 4827 get_64bit_val(feat_buf.va, byte_idx, &temp); 4828 feat_type = FIELD_GET(IRDMA_FEATURE_TYPE, temp); 4829 dev->feature_info[feat_type] = temp; 4830 } 4831 exit: 4832 irdma_free_dma_mem(dev->hw, &feat_buf); 4833 return ret_code; 4834 } 4835 4836 static u32 irdma_q1_cnt(struct irdma_sc_dev *dev, 4837 struct irdma_hmc_info *hmc_info, u32 qpwanted){ 4838 u32 q1_cnt; 4839 4840 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) { 4841 q1_cnt = roundup_pow_of_two(dev->hw_attrs.max_hw_ird * 2 * qpwanted); 4842 } else { 4843 if (dev->cqp->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY) 4844 q1_cnt = roundup_pow_of_two(dev->hw_attrs.max_hw_ird * 2 * qpwanted + 512); 4845 else 4846 q1_cnt = dev->hw_attrs.max_hw_ird * 2 * qpwanted; 4847 } 4848 4849 return q1_cnt; 4850 } 4851 4852 static void 4853 cfg_fpm_value_gen_1(struct irdma_sc_dev *dev, 4854 struct irdma_hmc_info *hmc_info, u32 qpwanted) 4855 { 4856 hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt = roundup_pow_of_two(qpwanted * dev->hw_attrs.max_hw_wqes); 4857 } 4858 4859 static void 4860 cfg_fpm_value_gen_2(struct irdma_sc_dev *dev, 4861 struct irdma_hmc_info *hmc_info, u32 qpwanted) 4862 { 4863 struct irdma_hmc_fpm_misc *hmc_fpm_misc = &dev->hmc_fpm_misc; 4864 4865 hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt = 4866 4 * hmc_fpm_misc->xf_block_size * qpwanted; 4867 4868 hmc_info->hmc_obj[IRDMA_HMC_IW_HDR].cnt = qpwanted; 4869 4870 if (hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].max_cnt) 4871 hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].cnt = 32 * qpwanted; 4872 if (hmc_info->hmc_obj[IRDMA_HMC_IW_RRFFL].max_cnt) 4873 hmc_info->hmc_obj[IRDMA_HMC_IW_RRFFL].cnt = 4874 hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].cnt / 4875 hmc_fpm_misc->rrf_block_size; 4876 if (dev->cqp->protocol_used == IRDMA_IWARP_PROTOCOL_ONLY) { 4877 if (hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].max_cnt) 4878 hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].cnt = 32 * qpwanted; 4879 if (hmc_info->hmc_obj[IRDMA_HMC_IW_OOISCFFL].max_cnt) 4880 hmc_info->hmc_obj[IRDMA_HMC_IW_OOISCFFL].cnt = 4881 hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].cnt / 4882 hmc_fpm_misc->ooiscf_block_size; 4883 } 4884 } 4885 4886 /** 4887 * irdma_cfg_fpm_val - configure HMC objects 4888 * @dev: sc device struct 4889 * @qp_count: desired qp count 4890 */ 4891 int 4892 irdma_cfg_fpm_val(struct irdma_sc_dev *dev, u32 qp_count) 4893 { 4894 struct irdma_virt_mem virt_mem; 4895 u32 i, mem_size; 4896 u32 qpwanted, mrwanted, pblewanted; 4897 u32 powerof2, hte; 4898 u32 sd_needed; 4899 u32 sd_diff; 4900 u32 loop_count = 0; 4901 struct irdma_hmc_info *hmc_info; 4902 struct irdma_hmc_fpm_misc *hmc_fpm_misc; 4903 int ret_code = 0; 4904 u32 max_sds; 4905 4906 hmc_info = dev->hmc_info; 4907 hmc_fpm_misc = &dev->hmc_fpm_misc; 4908 ret_code = irdma_sc_init_iw_hmc(dev, dev->hmc_fn_id); 4909 if (ret_code) { 4910 irdma_debug(dev, IRDMA_DEBUG_HMC, 4911 "irdma_sc_init_iw_hmc returned error_code = %d\n", 4912 ret_code); 4913 return ret_code; 4914 } 4915 4916 max_sds = hmc_fpm_misc->max_sds; 4917 4918 for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++) 4919 hmc_info->hmc_obj[i].cnt = hmc_info->hmc_obj[i].max_cnt; 4920 4921 sd_needed = irdma_est_sd(dev, hmc_info); 4922 irdma_debug(dev, IRDMA_DEBUG_HMC, "sd count %d where max sd is %d\n", 4923 hmc_info->sd_table.sd_cnt, max_sds); 4924 4925 qpwanted = min(qp_count, hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt); 4926 4927 powerof2 = 1; 4928 while (powerof2 <= qpwanted) 4929 powerof2 *= 2; 4930 powerof2 /= 2; 4931 qpwanted = powerof2; 4932 4933 mrwanted = hmc_info->hmc_obj[IRDMA_HMC_IW_MR].max_cnt; 4934 pblewanted = hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].max_cnt; 4935 4936 irdma_debug(dev, IRDMA_DEBUG_HMC, 4937 "req_qp=%d max_sd=%d, max_qp = %d, max_cq=%d, max_mr=%d, max_pble=%d, mc=%d, av=%d\n", 4938 qp_count, max_sds, 4939 hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt, 4940 hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].max_cnt, 4941 hmc_info->hmc_obj[IRDMA_HMC_IW_MR].max_cnt, 4942 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].max_cnt, 4943 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].max_cnt, 4944 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].max_cnt); 4945 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt = 4946 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].max_cnt; 4947 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt = 4948 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].max_cnt; 4949 hmc_info->hmc_obj[IRDMA_HMC_IW_ARP].cnt = 4950 hmc_info->hmc_obj[IRDMA_HMC_IW_ARP].max_cnt; 4951 if (dev->hw_attrs.uk_attrs.hw_rev <= IRDMA_GEN_2) 4952 hmc_info->hmc_obj[IRDMA_HMC_IW_APBVT_ENTRY].cnt = 1; 4953 4954 while (irdma_q1_cnt(dev, hmc_info, qpwanted) > hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].max_cnt) 4955 qpwanted /= 2; 4956 4957 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) { 4958 cfg_fpm_value_gen_1(dev, hmc_info, qpwanted); 4959 while (hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt > hmc_info->hmc_obj[IRDMA_HMC_IW_XF].max_cnt) { 4960 qpwanted /= 2; 4961 cfg_fpm_value_gen_1(dev, hmc_info, qpwanted); 4962 } 4963 } 4964 4965 do { 4966 ++loop_count; 4967 hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt = qpwanted; 4968 hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt = 4969 min(2 * qpwanted, hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt); 4970 hmc_info->hmc_obj[IRDMA_HMC_IW_RESERVED].cnt = 0; /* Reserved */ 4971 hmc_info->hmc_obj[IRDMA_HMC_IW_MR].cnt = mrwanted; 4972 4973 hte = round_up(qpwanted + hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt, 512); 4974 powerof2 = 1; 4975 while (powerof2 < hte) 4976 powerof2 *= 2; 4977 hmc_info->hmc_obj[IRDMA_HMC_IW_HTE].cnt = 4978 powerof2 * hmc_fpm_misc->ht_multiplier; 4979 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 4980 cfg_fpm_value_gen_1(dev, hmc_info, qpwanted); 4981 else 4982 cfg_fpm_value_gen_2(dev, hmc_info, qpwanted); 4983 4984 hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].cnt = irdma_q1_cnt(dev, hmc_info, qpwanted); 4985 hmc_info->hmc_obj[IRDMA_HMC_IW_XFFL].cnt = 4986 hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt / hmc_fpm_misc->xf_block_size; 4987 hmc_info->hmc_obj[IRDMA_HMC_IW_Q1FL].cnt = 4988 hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].cnt / hmc_fpm_misc->q1_block_size; 4989 hmc_info->hmc_obj[IRDMA_HMC_IW_TIMER].cnt = 4990 (round_up(qpwanted, 512) / 512 + 1) * hmc_fpm_misc->timer_bucket; 4991 4992 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt = pblewanted; 4993 sd_needed = irdma_est_sd(dev, hmc_info); 4994 irdma_debug(dev, IRDMA_DEBUG_HMC, 4995 "sd_needed = %d, max_sds=%d, mrwanted=%d, pblewanted=%d qpwanted=%d\n", 4996 sd_needed, max_sds, mrwanted, pblewanted, qpwanted); 4997 4998 /* Do not reduce resources further. All objects fit with max SDs */ 4999 if (sd_needed <= max_sds) 5000 break; 5001 5002 sd_diff = sd_needed - max_sds; 5003 if (sd_diff > 128) { 5004 if (!(loop_count % 2) && qpwanted > 128) { 5005 qpwanted /= 2; 5006 } else { 5007 mrwanted /= 2; 5008 pblewanted /= 2; 5009 } 5010 continue; 5011 } 5012 if (dev->cqp->hmc_profile != IRDMA_HMC_PROFILE_FAVOR_VF && 5013 pblewanted > (512 * FPM_MULTIPLIER * sd_diff)) { 5014 pblewanted -= 256 * FPM_MULTIPLIER * sd_diff; 5015 continue; 5016 } else if (pblewanted > 100 * FPM_MULTIPLIER) { 5017 pblewanted -= 10 * FPM_MULTIPLIER; 5018 } else if (pblewanted > 16 * FPM_MULTIPLIER) { 5019 pblewanted -= FPM_MULTIPLIER; 5020 } else if (qpwanted <= 128) { 5021 if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt > 256) 5022 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt /= 2; 5023 if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt > 256) 5024 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt /= 2; 5025 } 5026 if (mrwanted > FPM_MULTIPLIER) 5027 mrwanted -= FPM_MULTIPLIER; 5028 if (!(loop_count % 10) && qpwanted > 128) { 5029 qpwanted /= 2; 5030 if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt > 256) 5031 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt /= 2; 5032 } 5033 } while (loop_count < 2000); 5034 5035 if (sd_needed > max_sds) { 5036 irdma_debug(dev, IRDMA_DEBUG_HMC, 5037 "cfg_fpm failed loop_cnt=%d, sd_needed=%d, max sd count %d\n", 5038 loop_count, sd_needed, hmc_info->sd_table.sd_cnt); 5039 return -EINVAL; 5040 } 5041 5042 if (loop_count > 1 && sd_needed < max_sds) { 5043 pblewanted += (max_sds - sd_needed) * 256 * FPM_MULTIPLIER; 5044 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt = pblewanted; 5045 sd_needed = irdma_est_sd(dev, hmc_info); 5046 } 5047 5048 irdma_debug(dev, IRDMA_DEBUG_HMC, 5049 "loop_cnt=%d, sd_needed=%d, qpcnt = %d, cqcnt=%d, mrcnt=%d, pblecnt=%d, mc=%d, ah=%d, max sd count %d, first sd index %d\n", 5050 loop_count, sd_needed, 5051 hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt, 5052 hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt, 5053 hmc_info->hmc_obj[IRDMA_HMC_IW_MR].cnt, 5054 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt, 5055 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt, 5056 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt, 5057 hmc_info->sd_table.sd_cnt, hmc_info->first_sd_index); 5058 5059 ret_code = irdma_sc_cfg_iw_fpm(dev, dev->hmc_fn_id); 5060 if (ret_code) { 5061 irdma_debug(dev, IRDMA_DEBUG_HMC, 5062 "cfg_iw_fpm returned error_code[x%08X]\n", 5063 readl(dev->hw_regs[IRDMA_CQPERRCODES])); 5064 return ret_code; 5065 } 5066 5067 mem_size = sizeof(struct irdma_hmc_sd_entry) * 5068 (hmc_info->sd_table.sd_cnt + hmc_info->first_sd_index + 1); 5069 virt_mem.size = mem_size; 5070 virt_mem.va = kzalloc(virt_mem.size, GFP_KERNEL); 5071 if (!virt_mem.va) { 5072 irdma_debug(dev, IRDMA_DEBUG_HMC, 5073 "failed to allocate memory for sd_entry buffer\n"); 5074 return -ENOMEM; 5075 } 5076 hmc_info->sd_table.sd_entry = virt_mem.va; 5077 5078 return ret_code; 5079 } 5080 5081 /** 5082 * irdma_exec_cqp_cmd - execute cqp cmd when wqe are available 5083 * @dev: rdma device 5084 * @pcmdinfo: cqp command info 5085 */ 5086 static int 5087 irdma_exec_cqp_cmd(struct irdma_sc_dev *dev, 5088 struct cqp_cmds_info *pcmdinfo) 5089 { 5090 int status; 5091 struct irdma_dma_mem val_mem; 5092 bool alloc = false; 5093 5094 dev->cqp_cmd_stats[pcmdinfo->cqp_cmd]++; 5095 switch (pcmdinfo->cqp_cmd) { 5096 case IRDMA_OP_CEQ_DESTROY: 5097 status = irdma_sc_ceq_destroy(pcmdinfo->in.u.ceq_destroy.ceq, 5098 pcmdinfo->in.u.ceq_destroy.scratch, 5099 pcmdinfo->post_sq); 5100 break; 5101 case IRDMA_OP_AEQ_DESTROY: 5102 status = irdma_sc_aeq_destroy(pcmdinfo->in.u.aeq_destroy.aeq, 5103 pcmdinfo->in.u.aeq_destroy.scratch, 5104 pcmdinfo->post_sq); 5105 break; 5106 case IRDMA_OP_CEQ_CREATE: 5107 status = irdma_sc_ceq_create(pcmdinfo->in.u.ceq_create.ceq, 5108 pcmdinfo->in.u.ceq_create.scratch, 5109 pcmdinfo->post_sq); 5110 break; 5111 case IRDMA_OP_AEQ_CREATE: 5112 status = irdma_sc_aeq_create(pcmdinfo->in.u.aeq_create.aeq, 5113 pcmdinfo->in.u.aeq_create.scratch, 5114 pcmdinfo->post_sq); 5115 break; 5116 case IRDMA_OP_QP_UPLOAD_CONTEXT: 5117 status = irdma_sc_qp_upload_context(pcmdinfo->in.u.qp_upload_context.dev, 5118 &pcmdinfo->in.u.qp_upload_context.info, 5119 pcmdinfo->in.u.qp_upload_context.scratch, 5120 pcmdinfo->post_sq); 5121 break; 5122 case IRDMA_OP_CQ_CREATE: 5123 status = irdma_sc_cq_create(pcmdinfo->in.u.cq_create.cq, 5124 pcmdinfo->in.u.cq_create.scratch, 5125 pcmdinfo->in.u.cq_create.check_overflow, 5126 pcmdinfo->post_sq); 5127 break; 5128 case IRDMA_OP_CQ_MODIFY: 5129 status = irdma_sc_cq_modify(pcmdinfo->in.u.cq_modify.cq, 5130 &pcmdinfo->in.u.cq_modify.info, 5131 pcmdinfo->in.u.cq_modify.scratch, 5132 pcmdinfo->post_sq); 5133 break; 5134 case IRDMA_OP_CQ_DESTROY: 5135 status = irdma_sc_cq_destroy(pcmdinfo->in.u.cq_destroy.cq, 5136 pcmdinfo->in.u.cq_destroy.scratch, 5137 pcmdinfo->post_sq); 5138 break; 5139 case IRDMA_OP_QP_FLUSH_WQES: 5140 status = irdma_sc_qp_flush_wqes(pcmdinfo->in.u.qp_flush_wqes.qp, 5141 &pcmdinfo->in.u.qp_flush_wqes.info, 5142 pcmdinfo->in.u.qp_flush_wqes.scratch, 5143 pcmdinfo->post_sq); 5144 break; 5145 case IRDMA_OP_GEN_AE: 5146 status = irdma_sc_gen_ae(pcmdinfo->in.u.gen_ae.qp, 5147 &pcmdinfo->in.u.gen_ae.info, 5148 pcmdinfo->in.u.gen_ae.scratch, 5149 pcmdinfo->post_sq); 5150 break; 5151 case IRDMA_OP_MANAGE_PUSH_PAGE: 5152 status = irdma_sc_manage_push_page(pcmdinfo->in.u.manage_push_page.cqp, 5153 &pcmdinfo->in.u.manage_push_page.info, 5154 pcmdinfo->in.u.manage_push_page.scratch, 5155 pcmdinfo->post_sq); 5156 break; 5157 case IRDMA_OP_UPDATE_PE_SDS: 5158 status = irdma_update_pe_sds(pcmdinfo->in.u.update_pe_sds.dev, 5159 &pcmdinfo->in.u.update_pe_sds.info, 5160 pcmdinfo->in.u.update_pe_sds.scratch); 5161 break; 5162 case IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE: 5163 /* switch to calling through the call table */ 5164 status = 5165 irdma_sc_manage_hmc_pm_func_table(pcmdinfo->in.u.manage_hmc_pm.dev->cqp, 5166 &pcmdinfo->in.u.manage_hmc_pm.info, 5167 pcmdinfo->in.u.manage_hmc_pm.scratch, 5168 true); 5169 break; 5170 case IRDMA_OP_SUSPEND: 5171 status = irdma_sc_suspend_qp(pcmdinfo->in.u.suspend_resume.cqp, 5172 pcmdinfo->in.u.suspend_resume.qp, 5173 pcmdinfo->in.u.suspend_resume.scratch); 5174 break; 5175 case IRDMA_OP_RESUME: 5176 status = irdma_sc_resume_qp(pcmdinfo->in.u.suspend_resume.cqp, 5177 pcmdinfo->in.u.suspend_resume.qp, 5178 pcmdinfo->in.u.suspend_resume.scratch); 5179 break; 5180 case IRDMA_OP_QUERY_FPM_VAL: 5181 val_mem.pa = pcmdinfo->in.u.query_fpm_val.fpm_val_pa; 5182 val_mem.va = pcmdinfo->in.u.query_fpm_val.fpm_val_va; 5183 status = irdma_sc_query_fpm_val(pcmdinfo->in.u.query_fpm_val.cqp, 5184 pcmdinfo->in.u.query_fpm_val.scratch, 5185 pcmdinfo->in.u.query_fpm_val.hmc_fn_id, 5186 &val_mem, true, IRDMA_CQP_WAIT_EVENT); 5187 break; 5188 case IRDMA_OP_COMMIT_FPM_VAL: 5189 val_mem.pa = pcmdinfo->in.u.commit_fpm_val.fpm_val_pa; 5190 val_mem.va = pcmdinfo->in.u.commit_fpm_val.fpm_val_va; 5191 status = irdma_sc_commit_fpm_val(pcmdinfo->in.u.commit_fpm_val.cqp, 5192 pcmdinfo->in.u.commit_fpm_val.scratch, 5193 pcmdinfo->in.u.commit_fpm_val.hmc_fn_id, 5194 &val_mem, 5195 true, 5196 IRDMA_CQP_WAIT_EVENT); 5197 break; 5198 case IRDMA_OP_STATS_ALLOCATE: 5199 alloc = true; 5200 /* fallthrough */ 5201 case IRDMA_OP_STATS_FREE: 5202 status = irdma_sc_manage_stats_inst(pcmdinfo->in.u.stats_manage.cqp, 5203 &pcmdinfo->in.u.stats_manage.info, 5204 alloc, 5205 pcmdinfo->in.u.stats_manage.scratch); 5206 break; 5207 case IRDMA_OP_STATS_GATHER: 5208 status = irdma_sc_gather_stats(pcmdinfo->in.u.stats_gather.cqp, 5209 &pcmdinfo->in.u.stats_gather.info, 5210 pcmdinfo->in.u.stats_gather.scratch); 5211 break; 5212 case IRDMA_OP_WS_MODIFY_NODE: 5213 status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp, 5214 &pcmdinfo->in.u.ws_node.info, 5215 IRDMA_MODIFY_NODE, 5216 pcmdinfo->in.u.ws_node.scratch); 5217 break; 5218 case IRDMA_OP_WS_DELETE_NODE: 5219 status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp, 5220 &pcmdinfo->in.u.ws_node.info, 5221 IRDMA_DEL_NODE, 5222 pcmdinfo->in.u.ws_node.scratch); 5223 break; 5224 case IRDMA_OP_WS_ADD_NODE: 5225 status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp, 5226 &pcmdinfo->in.u.ws_node.info, 5227 IRDMA_ADD_NODE, 5228 pcmdinfo->in.u.ws_node.scratch); 5229 break; 5230 case IRDMA_OP_SET_UP_MAP: 5231 status = irdma_sc_set_up_map(pcmdinfo->in.u.up_map.cqp, 5232 &pcmdinfo->in.u.up_map.info, 5233 pcmdinfo->in.u.up_map.scratch); 5234 break; 5235 case IRDMA_OP_QUERY_RDMA_FEATURES: 5236 status = irdma_sc_query_rdma_features(pcmdinfo->in.u.query_rdma.cqp, 5237 &pcmdinfo->in.u.query_rdma.query_buff_mem, 5238 pcmdinfo->in.u.query_rdma.scratch); 5239 break; 5240 case IRDMA_OP_DELETE_ARP_CACHE_ENTRY: 5241 status = irdma_sc_del_arp_cache_entry(pcmdinfo->in.u.del_arp_cache_entry.cqp, 5242 pcmdinfo->in.u.del_arp_cache_entry.scratch, 5243 pcmdinfo->in.u.del_arp_cache_entry.arp_index, 5244 pcmdinfo->post_sq); 5245 break; 5246 case IRDMA_OP_MANAGE_APBVT_ENTRY: 5247 status = irdma_sc_manage_apbvt_entry(pcmdinfo->in.u.manage_apbvt_entry.cqp, 5248 &pcmdinfo->in.u.manage_apbvt_entry.info, 5249 pcmdinfo->in.u.manage_apbvt_entry.scratch, 5250 pcmdinfo->post_sq); 5251 break; 5252 case IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY: 5253 status = irdma_sc_manage_qhash_table_entry(pcmdinfo->in.u.manage_qhash_table_entry.cqp, 5254 &pcmdinfo->in.u.manage_qhash_table_entry.info, 5255 pcmdinfo->in.u.manage_qhash_table_entry.scratch, 5256 pcmdinfo->post_sq); 5257 break; 5258 case IRDMA_OP_QP_MODIFY: 5259 status = irdma_sc_qp_modify(pcmdinfo->in.u.qp_modify.qp, 5260 &pcmdinfo->in.u.qp_modify.info, 5261 pcmdinfo->in.u.qp_modify.scratch, 5262 pcmdinfo->post_sq); 5263 break; 5264 case IRDMA_OP_QP_CREATE: 5265 status = irdma_sc_qp_create(pcmdinfo->in.u.qp_create.qp, 5266 &pcmdinfo->in.u.qp_create.info, 5267 pcmdinfo->in.u.qp_create.scratch, 5268 pcmdinfo->post_sq); 5269 break; 5270 case IRDMA_OP_QP_DESTROY: 5271 status = irdma_sc_qp_destroy(pcmdinfo->in.u.qp_destroy.qp, 5272 pcmdinfo->in.u.qp_destroy.scratch, 5273 pcmdinfo->in.u.qp_destroy.remove_hash_idx, 5274 pcmdinfo->in.u.qp_destroy.ignore_mw_bnd, 5275 pcmdinfo->post_sq); 5276 break; 5277 case IRDMA_OP_ALLOC_STAG: 5278 status = irdma_sc_alloc_stag(pcmdinfo->in.u.alloc_stag.dev, 5279 &pcmdinfo->in.u.alloc_stag.info, 5280 pcmdinfo->in.u.alloc_stag.scratch, 5281 pcmdinfo->post_sq); 5282 break; 5283 case IRDMA_OP_MR_REG_NON_SHARED: 5284 status = irdma_sc_mr_reg_non_shared(pcmdinfo->in.u.mr_reg_non_shared.dev, 5285 &pcmdinfo->in.u.mr_reg_non_shared.info, 5286 pcmdinfo->in.u.mr_reg_non_shared.scratch, 5287 pcmdinfo->post_sq); 5288 break; 5289 case IRDMA_OP_DEALLOC_STAG: 5290 status = irdma_sc_dealloc_stag(pcmdinfo->in.u.dealloc_stag.dev, 5291 &pcmdinfo->in.u.dealloc_stag.info, 5292 pcmdinfo->in.u.dealloc_stag.scratch, 5293 pcmdinfo->post_sq); 5294 break; 5295 case IRDMA_OP_MW_ALLOC: 5296 status = irdma_sc_mw_alloc(pcmdinfo->in.u.mw_alloc.dev, 5297 &pcmdinfo->in.u.mw_alloc.info, 5298 pcmdinfo->in.u.mw_alloc.scratch, 5299 pcmdinfo->post_sq); 5300 break; 5301 case IRDMA_OP_ADD_ARP_CACHE_ENTRY: 5302 status = irdma_sc_add_arp_cache_entry(pcmdinfo->in.u.add_arp_cache_entry.cqp, 5303 &pcmdinfo->in.u.add_arp_cache_entry.info, 5304 pcmdinfo->in.u.add_arp_cache_entry.scratch, 5305 pcmdinfo->post_sq); 5306 break; 5307 case IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY: 5308 status = irdma_sc_alloc_local_mac_entry(pcmdinfo->in.u.alloc_local_mac_entry.cqp, 5309 pcmdinfo->in.u.alloc_local_mac_entry.scratch, 5310 pcmdinfo->post_sq); 5311 break; 5312 case IRDMA_OP_ADD_LOCAL_MAC_ENTRY: 5313 status = irdma_sc_add_local_mac_entry(pcmdinfo->in.u.add_local_mac_entry.cqp, 5314 &pcmdinfo->in.u.add_local_mac_entry.info, 5315 pcmdinfo->in.u.add_local_mac_entry.scratch, 5316 pcmdinfo->post_sq); 5317 break; 5318 case IRDMA_OP_DELETE_LOCAL_MAC_ENTRY: 5319 status = irdma_sc_del_local_mac_entry(pcmdinfo->in.u.del_local_mac_entry.cqp, 5320 pcmdinfo->in.u.del_local_mac_entry.scratch, 5321 pcmdinfo->in.u.del_local_mac_entry.entry_idx, 5322 pcmdinfo->in.u.del_local_mac_entry.ignore_ref_count, 5323 pcmdinfo->post_sq); 5324 break; 5325 case IRDMA_OP_AH_CREATE: 5326 status = irdma_sc_create_ah(pcmdinfo->in.u.ah_create.cqp, 5327 &pcmdinfo->in.u.ah_create.info, 5328 pcmdinfo->in.u.ah_create.scratch); 5329 break; 5330 case IRDMA_OP_AH_DESTROY: 5331 status = irdma_sc_destroy_ah(pcmdinfo->in.u.ah_destroy.cqp, 5332 &pcmdinfo->in.u.ah_destroy.info, 5333 pcmdinfo->in.u.ah_destroy.scratch); 5334 break; 5335 case IRDMA_OP_MC_CREATE: 5336 status = irdma_sc_create_mcast_grp(pcmdinfo->in.u.mc_create.cqp, 5337 &pcmdinfo->in.u.mc_create.info, 5338 pcmdinfo->in.u.mc_create.scratch); 5339 break; 5340 case IRDMA_OP_MC_DESTROY: 5341 status = irdma_sc_destroy_mcast_grp(pcmdinfo->in.u.mc_destroy.cqp, 5342 &pcmdinfo->in.u.mc_destroy.info, 5343 pcmdinfo->in.u.mc_destroy.scratch); 5344 break; 5345 case IRDMA_OP_MC_MODIFY: 5346 status = irdma_sc_modify_mcast_grp(pcmdinfo->in.u.mc_modify.cqp, 5347 &pcmdinfo->in.u.mc_modify.info, 5348 pcmdinfo->in.u.mc_modify.scratch); 5349 break; 5350 default: 5351 status = -EOPNOTSUPP; 5352 break; 5353 } 5354 5355 return status; 5356 } 5357 5358 /** 5359 * irdma_process_cqp_cmd - process all cqp commands 5360 * @dev: sc device struct 5361 * @pcmdinfo: cqp command info 5362 */ 5363 int 5364 irdma_process_cqp_cmd(struct irdma_sc_dev *dev, 5365 struct cqp_cmds_info *pcmdinfo) 5366 { 5367 int status = 0; 5368 unsigned long flags; 5369 5370 spin_lock_irqsave(&dev->cqp_lock, flags); 5371 if (list_empty(&dev->cqp_cmd_head) && !irdma_cqp_ring_full(dev->cqp)) 5372 status = irdma_exec_cqp_cmd(dev, pcmdinfo); 5373 else 5374 list_add_tail(&pcmdinfo->cqp_cmd_entry, &dev->cqp_cmd_head); 5375 spin_unlock_irqrestore(&dev->cqp_lock, flags); 5376 return status; 5377 } 5378 5379 /** 5380 * irdma_process_bh - called from tasklet for cqp list 5381 * @dev: sc device struct 5382 */ 5383 int 5384 irdma_process_bh(struct irdma_sc_dev *dev) 5385 { 5386 int status = 0; 5387 struct cqp_cmds_info *pcmdinfo; 5388 unsigned long flags; 5389 5390 spin_lock_irqsave(&dev->cqp_lock, flags); 5391 while (!list_empty(&dev->cqp_cmd_head) && 5392 !irdma_cqp_ring_full(dev->cqp)) { 5393 pcmdinfo = (struct cqp_cmds_info *)irdma_remove_cqp_head(dev); 5394 status = irdma_exec_cqp_cmd(dev, pcmdinfo); 5395 if (status) 5396 break; 5397 } 5398 spin_unlock_irqrestore(&dev->cqp_lock, flags); 5399 return status; 5400 } 5401 5402 /** 5403 * irdma_cfg_aeq- Configure AEQ interrupt 5404 * @dev: pointer to the device structure 5405 * @idx: vector index 5406 * @enable: True to enable, False disables 5407 */ 5408 void 5409 irdma_cfg_aeq(struct irdma_sc_dev *dev, u32 idx, bool enable) 5410 { 5411 u32 reg_val; 5412 reg_val = FIELD_PREP(IRDMA_PFINT_AEQCTL_CAUSE_ENA, enable) | 5413 FIELD_PREP(IRDMA_PFINT_AEQCTL_MSIX_INDX, idx) | 5414 FIELD_PREP(IRDMA_PFINT_AEQCTL_ITR_INDX, IRDMA_IDX_NOITR); 5415 5416 writel(reg_val, dev->hw_regs[IRDMA_PFINT_AEQCTL]); 5417 } 5418 5419 /** 5420 * sc_vsi_update_stats - Update statistics 5421 * @vsi: sc_vsi instance to update 5422 */ 5423 void 5424 sc_vsi_update_stats(struct irdma_sc_vsi *vsi) 5425 { 5426 struct irdma_gather_stats *gather_stats; 5427 struct irdma_gather_stats *last_gather_stats; 5428 5429 gather_stats = vsi->pestat->gather_info.gather_stats_va; 5430 last_gather_stats = vsi->pestat->gather_info.last_gather_stats_va; 5431 irdma_update_stats(&vsi->pestat->hw_stats, gather_stats, 5432 last_gather_stats, vsi->dev->hw_stats_map, 5433 vsi->dev->hw_attrs.max_stat_idx); 5434 } 5435 5436 /** 5437 * irdma_wait_pe_ready - Check if firmware is ready 5438 * @dev: provides access to registers 5439 */ 5440 static int 5441 irdma_wait_pe_ready(struct irdma_sc_dev *dev) 5442 { 5443 u32 statuscpu0; 5444 u32 statuscpu1; 5445 u32 statuscpu2; 5446 u32 retrycount = 0; 5447 5448 do { 5449 statuscpu0 = readl(dev->hw_regs[IRDMA_GLPE_CPUSTATUS0]); 5450 statuscpu1 = readl(dev->hw_regs[IRDMA_GLPE_CPUSTATUS1]); 5451 statuscpu2 = readl(dev->hw_regs[IRDMA_GLPE_CPUSTATUS2]); 5452 if (statuscpu0 == 0x80 && statuscpu1 == 0x80 && 5453 statuscpu2 == 0x80) 5454 return 0; 5455 mdelay(1000); 5456 } while (retrycount++ < dev->hw_attrs.max_pe_ready_count); 5457 return -1; 5458 } 5459 5460 static inline void 5461 irdma_sc_init_hw(struct irdma_sc_dev *dev) 5462 { 5463 switch (dev->hw_attrs.uk_attrs.hw_rev) { 5464 case IRDMA_GEN_2: 5465 icrdma_init_hw(dev); 5466 break; 5467 } 5468 } 5469 5470 /** 5471 * irdma_sc_dev_init - Initialize control part of device 5472 * @dev: Device pointer 5473 * @info: Device init info 5474 */ 5475 int 5476 irdma_sc_dev_init(struct irdma_sc_dev *dev, struct irdma_device_init_info *info) 5477 { 5478 u32 val; 5479 int ret_code = 0; 5480 u8 db_size; 5481 5482 INIT_LIST_HEAD(&dev->cqp_cmd_head); /* for CQP command backlog */ 5483 mutex_init(&dev->ws_mutex); 5484 dev->debug_mask = info->debug_mask; 5485 dev->hmc_fn_id = info->hmc_fn_id; 5486 dev->fpm_query_buf_pa = info->fpm_query_buf_pa; 5487 dev->fpm_query_buf = info->fpm_query_buf; 5488 dev->fpm_commit_buf_pa = info->fpm_commit_buf_pa; 5489 dev->fpm_commit_buf = info->fpm_commit_buf; 5490 dev->hw = info->hw; 5491 dev->hw->hw_addr = info->bar0; 5492 /* Setup the hardware limits, hmc may limit further */ 5493 dev->hw_attrs.min_hw_qp_id = IRDMA_MIN_IW_QP_ID; 5494 dev->hw_attrs.min_hw_aeq_size = IRDMA_MIN_AEQ_ENTRIES; 5495 dev->hw_attrs.max_hw_aeq_size = IRDMA_MAX_AEQ_ENTRIES; 5496 dev->hw_attrs.min_hw_ceq_size = IRDMA_MIN_CEQ_ENTRIES; 5497 dev->hw_attrs.max_hw_ceq_size = IRDMA_MAX_CEQ_ENTRIES; 5498 dev->hw_attrs.uk_attrs.min_hw_cq_size = IRDMA_MIN_CQ_SIZE; 5499 dev->hw_attrs.uk_attrs.max_hw_cq_size = IRDMA_MAX_CQ_SIZE; 5500 dev->hw_attrs.max_hw_outbound_msg_size = IRDMA_MAX_OUTBOUND_MSG_SIZE; 5501 dev->hw_attrs.max_mr_size = IRDMA_MAX_MR_SIZE; 5502 dev->hw_attrs.max_hw_inbound_msg_size = IRDMA_MAX_INBOUND_MSG_SIZE; 5503 dev->hw_attrs.max_hw_device_pages = IRDMA_MAX_PUSH_PAGE_COUNT; 5504 dev->hw_attrs.uk_attrs.max_hw_inline = IRDMA_MAX_INLINE_DATA_SIZE; 5505 dev->hw_attrs.max_hw_wqes = IRDMA_MAX_WQ_ENTRIES; 5506 dev->hw_attrs.max_qp_wr = IRDMA_MAX_QP_WRS(IRDMA_MAX_QUANTA_PER_WR); 5507 5508 dev->hw_attrs.uk_attrs.max_hw_rq_quanta = IRDMA_QP_SW_MAX_RQ_QUANTA; 5509 dev->hw_attrs.uk_attrs.max_hw_wq_quanta = IRDMA_QP_SW_MAX_WQ_QUANTA; 5510 dev->hw_attrs.max_hw_pds = IRDMA_MAX_PDS; 5511 dev->hw_attrs.max_hw_ena_vf_count = IRDMA_MAX_PE_ENA_VF_COUNT; 5512 5513 dev->hw_attrs.max_pe_ready_count = 14; 5514 dev->hw_attrs.max_done_count = IRDMA_DONE_COUNT; 5515 dev->hw_attrs.max_sleep_count = IRDMA_SLEEP_COUNT; 5516 dev->hw_attrs.max_cqp_compl_wait_time_ms = CQP_COMPL_WAIT_TIME_MS; 5517 5518 irdma_sc_init_hw(dev); 5519 5520 if (irdma_wait_pe_ready(dev)) 5521 return -ETIMEDOUT; 5522 5523 val = readl(dev->hw_regs[IRDMA_GLPCI_LBARCTRL]); 5524 db_size = (u8)FIELD_GET(IRDMA_GLPCI_LBARCTRL_PE_DB_SIZE, val); 5525 if (db_size != IRDMA_PE_DB_SIZE_4M && db_size != IRDMA_PE_DB_SIZE_8M) { 5526 irdma_debug(dev, IRDMA_DEBUG_DEV, 5527 "RDMA PE doorbell is not enabled in CSR val 0x%x db_size=%d\n", 5528 val, db_size); 5529 return -ENODEV; 5530 } 5531 dev->db_addr = dev->hw->hw_addr + (uintptr_t)dev->hw_regs[IRDMA_DB_ADDR_OFFSET]; 5532 5533 return ret_code; 5534 } 5535 5536 /** 5537 * irdma_stat_val - Extract HW counter value from statistics buffer 5538 * @stats_val: pointer to statistics buffer 5539 * @byteoff: byte offset of counter value in the buffer (8B-aligned) 5540 * @bitoff: bit offset of counter value within 8B entry 5541 * @bitmask: maximum counter value (e.g. 0xffffff for 24-bit counter) 5542 */ 5543 static inline u64 irdma_stat_val(const u64 *stats_val, u16 byteoff, 5544 u8 bitoff, u64 bitmask){ 5545 u16 idx = byteoff / sizeof(*stats_val); 5546 5547 return (stats_val[idx] >> bitoff) & bitmask; 5548 } 5549 5550 /** 5551 * irdma_stat_delta - Calculate counter delta 5552 * @new_val: updated counter value 5553 * @old_val: last counter value 5554 * @max_val: maximum counter value (e.g. 0xffffff for 24-bit counter) 5555 */ 5556 static inline u64 irdma_stat_delta(u64 new_val, u64 old_val, u64 max_val) { 5557 if (new_val >= old_val) 5558 return new_val - old_val; 5559 else 5560 /* roll-over case */ 5561 return max_val - old_val + new_val + 1; 5562 } 5563 5564 /** 5565 * irdma_update_stats - Update statistics 5566 * @hw_stats: hw_stats instance to update 5567 * @gather_stats: updated stat counters 5568 * @last_gather_stats: last stat counters 5569 * @map: HW stat map (hw_stats => gather_stats) 5570 * @max_stat_idx: number of HW stats 5571 */ 5572 void 5573 irdma_update_stats(struct irdma_dev_hw_stats *hw_stats, 5574 struct irdma_gather_stats *gather_stats, 5575 struct irdma_gather_stats *last_gather_stats, 5576 const struct irdma_hw_stat_map *map, 5577 u16 max_stat_idx) 5578 { 5579 u64 *stats_val = hw_stats->stats_val; 5580 u16 i; 5581 5582 for (i = 0; i < max_stat_idx; i++) { 5583 u64 new_val = irdma_stat_val(gather_stats->val, 5584 map[i].byteoff, map[i].bitoff, 5585 map[i].bitmask); 5586 u64 last_val = irdma_stat_val(last_gather_stats->val, 5587 map[i].byteoff, map[i].bitoff, 5588 map[i].bitmask); 5589 5590 stats_val[i] += irdma_stat_delta(new_val, last_val, 5591 map[i].bitmask); 5592 } 5593 5594 irdma_memcpy(last_gather_stats, gather_stats, 5595 sizeof(*last_gather_stats)); 5596 } 5597