1 /*- 2 * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 3 * 4 * Copyright (c) 2015 - 2023 Intel Corporation 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenFabrics.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #include "osdep.h" 36 #include "irdma_hmc.h" 37 #include "irdma_defs.h" 38 #include "irdma_type.h" 39 #include "irdma_ws.h" 40 #include "irdma_protos.h" 41 42 /** 43 * irdma_qp_from_entry - Given entry, get to the qp structure 44 * @entry: Points to list of qp structure 45 */ 46 static struct irdma_sc_qp * 47 irdma_qp_from_entry(struct list_head *entry) 48 { 49 if (!entry) 50 return NULL; 51 52 return (struct irdma_sc_qp *)((char *)entry - 53 offsetof(struct irdma_sc_qp, list)); 54 } 55 56 /** 57 * irdma_get_qp_from_list - get next qp from a list 58 * @head: Listhead of qp's 59 * @qp: current qp 60 */ 61 struct irdma_sc_qp * 62 irdma_get_qp_from_list(struct list_head *head, 63 struct irdma_sc_qp *qp) 64 { 65 struct list_head *lastentry; 66 struct list_head *entry = NULL; 67 68 if (list_empty(head)) 69 return NULL; 70 71 if (!qp) { 72 entry = (head)->next; 73 } else { 74 lastentry = &qp->list; 75 entry = (lastentry)->next; 76 if (entry == head) 77 return NULL; 78 } 79 80 return irdma_qp_from_entry(entry); 81 } 82 83 /** 84 * irdma_sc_suspend_resume_qps - suspend/resume all qp's on VSI 85 * @vsi: the VSI struct pointer 86 * @op: Set to IRDMA_OP_RESUME or IRDMA_OP_SUSPEND 87 */ 88 void 89 irdma_sc_suspend_resume_qps(struct irdma_sc_vsi *vsi, u8 op) 90 { 91 struct irdma_sc_qp *qp = NULL; 92 u8 i; 93 94 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) { 95 mutex_lock(&vsi->qos[i].qos_mutex); 96 qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp); 97 while (qp) { 98 if (op == IRDMA_OP_RESUME) { 99 if (!qp->dev->ws_add(vsi, i)) { 100 qp->qs_handle = 101 vsi->qos[qp->user_pri].qs_handle; 102 irdma_cqp_qp_suspend_resume(qp, op); 103 } else { 104 irdma_cqp_qp_suspend_resume(qp, op); 105 irdma_modify_qp_to_err(qp); 106 } 107 } else if (op == IRDMA_OP_SUSPEND) { 108 /* issue cqp suspend command */ 109 if (!irdma_cqp_qp_suspend_resume(qp, op)) 110 atomic_inc(&vsi->qp_suspend_reqs); 111 } 112 qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp); 113 } 114 mutex_unlock(&vsi->qos[i].qos_mutex); 115 } 116 } 117 118 static void 119 irdma_set_qos_info(struct irdma_sc_vsi *vsi, struct irdma_l2params *l2p) 120 { 121 u8 i; 122 123 vsi->qos_rel_bw = l2p->vsi_rel_bw; 124 vsi->qos_prio_type = l2p->vsi_prio_type; 125 vsi->dscp_mode = l2p->dscp_mode; 126 if (l2p->dscp_mode) { 127 irdma_memcpy(vsi->dscp_map, l2p->dscp_map, sizeof(vsi->dscp_map)); 128 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) 129 l2p->up2tc[i] = i; 130 } 131 for (i = 0; i < IRDMA_MAX_TRAFFIC_CLASS; i++) 132 vsi->tc_print_warning[i] = true; 133 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) { 134 if (vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 135 vsi->qos[i].qs_handle = l2p->qs_handle_list[i]; 136 if (vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_2) 137 irdma_init_config_check(&vsi->cfg_check[i], 138 l2p->up2tc[i], 139 l2p->qs_handle_list[i]); 140 vsi->qos[i].traffic_class = l2p->up2tc[i]; 141 vsi->qos[i].rel_bw = 142 l2p->tc_info[vsi->qos[i].traffic_class].rel_bw; 143 vsi->qos[i].prio_type = 144 l2p->tc_info[vsi->qos[i].traffic_class].prio_type; 145 vsi->qos[i].valid = false; 146 } 147 } 148 149 /** 150 * irdma_change_l2params - given the new l2 parameters, change all qp 151 * @vsi: RDMA VSI pointer 152 * @l2params: New parameters from l2 153 */ 154 void 155 irdma_change_l2params(struct irdma_sc_vsi *vsi, 156 struct irdma_l2params *l2params) 157 { 158 if (l2params->tc_changed) { 159 vsi->tc_change_pending = false; 160 irdma_set_qos_info(vsi, l2params); 161 irdma_sc_suspend_resume_qps(vsi, IRDMA_OP_RESUME); 162 } 163 if (l2params->mtu_changed) { 164 vsi->mtu = l2params->mtu; 165 if (vsi->ieq) 166 irdma_reinitialize_ieq(vsi); 167 } 168 } 169 170 /** 171 * irdma_qp_rem_qos - remove qp from qos lists during destroy qp 172 * @qp: qp to be removed from qos 173 */ 174 void 175 irdma_qp_rem_qos(struct irdma_sc_qp *qp) 176 { 177 struct irdma_sc_vsi *vsi = qp->vsi; 178 179 irdma_debug(qp->dev, IRDMA_DEBUG_DCB, 180 "DCB: Remove qp[%d] UP[%d] qset[%d] on_qoslist[%d]\n", 181 qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle, 182 qp->on_qoslist); 183 mutex_lock(&vsi->qos[qp->user_pri].qos_mutex); 184 if (qp->on_qoslist) { 185 qp->on_qoslist = false; 186 list_del(&qp->list); 187 } 188 mutex_unlock(&vsi->qos[qp->user_pri].qos_mutex); 189 } 190 191 /** 192 * irdma_qp_add_qos - called during setctx for qp to be added to qos 193 * @qp: qp to be added to qos 194 */ 195 void 196 irdma_qp_add_qos(struct irdma_sc_qp *qp) 197 { 198 struct irdma_sc_vsi *vsi = qp->vsi; 199 200 irdma_debug(qp->dev, IRDMA_DEBUG_DCB, 201 "DCB: Add qp[%d] UP[%d] qset[%d] on_qoslist[%d]\n", 202 qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle, 203 qp->on_qoslist); 204 mutex_lock(&vsi->qos[qp->user_pri].qos_mutex); 205 if (!qp->on_qoslist) { 206 list_add(&qp->list, &vsi->qos[qp->user_pri].qplist); 207 qp->on_qoslist = true; 208 qp->qs_handle = vsi->qos[qp->user_pri].qs_handle; 209 } 210 mutex_unlock(&vsi->qos[qp->user_pri].qos_mutex); 211 } 212 213 /** 214 * irdma_sc_pd_init - initialize sc pd struct 215 * @dev: sc device struct 216 * @pd: sc pd ptr 217 * @pd_id: pd_id for allocated pd 218 * @abi_ver: User/Kernel ABI version 219 */ 220 void 221 irdma_sc_pd_init(struct irdma_sc_dev *dev, struct irdma_sc_pd *pd, u32 pd_id, 222 int abi_ver) 223 { 224 pd->pd_id = pd_id; 225 pd->abi_ver = abi_ver; 226 pd->dev = dev; 227 } 228 229 /** 230 * irdma_sc_add_arp_cache_entry - cqp wqe add arp cache entry 231 * @cqp: struct for cqp hw 232 * @info: arp entry information 233 * @scratch: u64 saved to be used during cqp completion 234 * @post_sq: flag for cqp db to ring 235 */ 236 static int 237 irdma_sc_add_arp_cache_entry(struct irdma_sc_cqp *cqp, 238 struct irdma_add_arp_cache_entry_info *info, 239 u64 scratch, bool post_sq) 240 { 241 __le64 *wqe; 242 u64 hdr; 243 244 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 245 if (!wqe) 246 return -ENOSPC; 247 set_64bit_val(wqe, IRDMA_BYTE_8, info->reach_max); 248 249 set_64bit_val(wqe, IRDMA_BYTE_16, irdma_mac_to_u64(info->mac_addr)); 250 251 hdr = info->arp_index | 252 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_ARP) | 253 FIELD_PREP(IRDMA_CQPSQ_MAT_PERMANENT, info->permanent) | 254 FIELD_PREP(IRDMA_CQPSQ_MAT_ENTRYVALID, true) | 255 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 256 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 257 258 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 259 260 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "ARP_CACHE_ENTRY WQE", wqe, 261 IRDMA_CQP_WQE_SIZE * 8); 262 if (post_sq) 263 irdma_sc_cqp_post_sq(cqp); 264 265 return 0; 266 } 267 268 /** 269 * irdma_sc_del_arp_cache_entry - dele arp cache entry 270 * @cqp: struct for cqp hw 271 * @scratch: u64 saved to be used during cqp completion 272 * @arp_index: arp index to delete arp entry 273 * @post_sq: flag for cqp db to ring 274 */ 275 static int 276 irdma_sc_del_arp_cache_entry(struct irdma_sc_cqp *cqp, u64 scratch, 277 u16 arp_index, bool post_sq) 278 { 279 __le64 *wqe; 280 u64 hdr; 281 282 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 283 if (!wqe) 284 return -ENOSPC; 285 286 hdr = arp_index | 287 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_ARP) | 288 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 289 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 290 291 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 292 293 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "ARP_CACHE_DEL_ENTRY WQE", 294 wqe, IRDMA_CQP_WQE_SIZE * 8); 295 if (post_sq) 296 irdma_sc_cqp_post_sq(cqp); 297 298 return 0; 299 } 300 301 /** 302 * irdma_sc_manage_apbvt_entry - for adding and deleting apbvt entries 303 * @cqp: struct for cqp hw 304 * @info: info for apbvt entry to add or delete 305 * @scratch: u64 saved to be used during cqp completion 306 * @post_sq: flag for cqp db to ring 307 */ 308 static int 309 irdma_sc_manage_apbvt_entry(struct irdma_sc_cqp *cqp, 310 struct irdma_apbvt_info *info, 311 u64 scratch, bool post_sq) 312 { 313 __le64 *wqe; 314 u64 hdr; 315 316 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 317 if (!wqe) 318 return -ENOSPC; 319 320 set_64bit_val(wqe, IRDMA_BYTE_16, info->port); 321 322 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_APBVT) | 323 FIELD_PREP(IRDMA_CQPSQ_MAPT_ADDPORT, info->add) | 324 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 325 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 326 327 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 328 329 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "MANAGE_APBVT WQE", wqe, 330 IRDMA_CQP_WQE_SIZE * 8); 331 if (post_sq) 332 irdma_sc_cqp_post_sq(cqp); 333 334 return 0; 335 } 336 337 /** 338 * irdma_sc_manage_qhash_table_entry - manage quad hash entries 339 * @cqp: struct for cqp hw 340 * @info: info for quad hash to manage 341 * @scratch: u64 saved to be used during cqp completion 342 * @post_sq: flag for cqp db to ring 343 * 344 * This is called before connection establishment is started. 345 * For passive connections, when listener is created, it will 346 * call with entry type of IRDMA_QHASH_TYPE_TCP_SYN with local 347 * ip address and tcp port. When SYN is received (passive 348 * connections) or sent (active connections), this routine is 349 * called with entry type of IRDMA_QHASH_TYPE_TCP_ESTABLISHED 350 * and quad is passed in info. 351 * 352 * When iwarp connection is done and its state moves to RTS, the 353 * quad hash entry in the hardware will point to iwarp's qp 354 * number and requires no calls from the driver. 355 */ 356 static int 357 irdma_sc_manage_qhash_table_entry(struct irdma_sc_cqp *cqp, 358 struct irdma_qhash_table_info *info, 359 u64 scratch, bool post_sq) 360 { 361 __le64 *wqe; 362 u64 qw1 = 0; 363 u64 qw2 = 0; 364 u64 temp; 365 struct irdma_sc_vsi *vsi = info->vsi; 366 367 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 368 if (!wqe) 369 return -ENOSPC; 370 set_64bit_val(wqe, IRDMA_BYTE_0, irdma_mac_to_u64(info->mac_addr)); 371 372 qw1 = FIELD_PREP(IRDMA_CQPSQ_QHASH_QPN, info->qp_num) | 373 FIELD_PREP(IRDMA_CQPSQ_QHASH_DEST_PORT, info->dest_port); 374 if (info->ipv4_valid) { 375 set_64bit_val(wqe, IRDMA_BYTE_48, 376 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->dest_ip[0])); 377 } else { 378 set_64bit_val(wqe, IRDMA_BYTE_56, 379 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR0, info->dest_ip[0]) | 380 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR1, info->dest_ip[1])); 381 382 set_64bit_val(wqe, IRDMA_BYTE_48, 383 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR2, info->dest_ip[2]) | 384 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->dest_ip[3])); 385 } 386 qw2 = FIELD_PREP(IRDMA_CQPSQ_QHASH_QS_HANDLE, 387 vsi->qos[info->user_pri].qs_handle); 388 if (info->vlan_valid) 389 qw2 |= FIELD_PREP(IRDMA_CQPSQ_QHASH_VLANID, info->vlan_id); 390 set_64bit_val(wqe, IRDMA_BYTE_16, qw2); 391 if (info->entry_type == IRDMA_QHASH_TYPE_TCP_ESTABLISHED) { 392 qw1 |= FIELD_PREP(IRDMA_CQPSQ_QHASH_SRC_PORT, info->src_port); 393 if (!info->ipv4_valid) { 394 set_64bit_val(wqe, IRDMA_BYTE_40, 395 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR0, info->src_ip[0]) | 396 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR1, info->src_ip[1])); 397 set_64bit_val(wqe, IRDMA_BYTE_32, 398 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR2, info->src_ip[2]) | 399 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->src_ip[3])); 400 } else { 401 set_64bit_val(wqe, IRDMA_BYTE_32, 402 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->src_ip[0])); 403 } 404 } 405 406 set_64bit_val(wqe, IRDMA_BYTE_8, qw1); 407 temp = FIELD_PREP(IRDMA_CQPSQ_QHASH_WQEVALID, cqp->polarity) | 408 FIELD_PREP(IRDMA_CQPSQ_QHASH_OPCODE, 409 IRDMA_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY) | 410 FIELD_PREP(IRDMA_CQPSQ_QHASH_MANAGE, info->manage) | 411 FIELD_PREP(IRDMA_CQPSQ_QHASH_IPV4VALID, info->ipv4_valid) | 412 FIELD_PREP(IRDMA_CQPSQ_QHASH_VLANVALID, info->vlan_valid) | 413 FIELD_PREP(IRDMA_CQPSQ_QHASH_ENTRYTYPE, info->entry_type); 414 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 415 416 set_64bit_val(wqe, IRDMA_BYTE_24, temp); 417 418 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "MANAGE_QHASH WQE", wqe, 419 IRDMA_CQP_WQE_SIZE * 8); 420 if (post_sq) 421 irdma_sc_cqp_post_sq(cqp); 422 423 return 0; 424 } 425 426 /** 427 * irdma_sc_qp_init - initialize qp 428 * @qp: sc qp 429 * @info: initialization qp info 430 */ 431 int 432 irdma_sc_qp_init(struct irdma_sc_qp *qp, struct irdma_qp_init_info *info) 433 { 434 int ret_code; 435 u32 pble_obj_cnt; 436 u16 wqe_size; 437 438 if (info->qp_uk_init_info.max_sq_frag_cnt > 439 info->pd->dev->hw_attrs.uk_attrs.max_hw_wq_frags || 440 info->qp_uk_init_info.max_rq_frag_cnt > 441 info->pd->dev->hw_attrs.uk_attrs.max_hw_wq_frags) 442 return -EINVAL; 443 444 qp->dev = info->pd->dev; 445 qp->vsi = info->vsi; 446 qp->ieq_qp = info->vsi->exception_lan_q; 447 qp->sq_pa = info->sq_pa; 448 qp->rq_pa = info->rq_pa; 449 qp->hw_host_ctx_pa = info->host_ctx_pa; 450 qp->q2_pa = info->q2_pa; 451 qp->shadow_area_pa = info->shadow_area_pa; 452 qp->q2_buf = info->q2; 453 qp->pd = info->pd; 454 qp->hw_host_ctx = info->host_ctx; 455 info->qp_uk_init_info.wqe_alloc_db = qp->pd->dev->wqe_alloc_db; 456 ret_code = irdma_uk_qp_init(&qp->qp_uk, &info->qp_uk_init_info); 457 if (ret_code) 458 return ret_code; 459 460 qp->virtual_map = info->virtual_map; 461 pble_obj_cnt = info->pd->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; 462 463 if ((info->virtual_map && info->sq_pa >= pble_obj_cnt) || 464 (info->virtual_map && info->rq_pa >= pble_obj_cnt)) 465 return -EINVAL; 466 467 qp->llp_stream_handle = (void *)(-1); 468 qp->hw_sq_size = irdma_get_encoded_wqe_size(qp->qp_uk.sq_ring.size, 469 IRDMA_QUEUE_TYPE_SQ_RQ); 470 irdma_debug(qp->dev, IRDMA_DEBUG_WQE, 471 "hw_sq_size[%04d] sq_ring.size[%04d]\n", qp->hw_sq_size, 472 qp->qp_uk.sq_ring.size); 473 if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) 474 wqe_size = IRDMA_WQE_SIZE_128; 475 else 476 ret_code = irdma_fragcnt_to_wqesize_rq(qp->qp_uk.max_rq_frag_cnt, 477 &wqe_size); 478 if (ret_code) 479 return ret_code; 480 481 qp->hw_rq_size = 482 irdma_get_encoded_wqe_size(qp->qp_uk.rq_size * 483 (wqe_size / IRDMA_QP_WQE_MIN_SIZE), 484 IRDMA_QUEUE_TYPE_SQ_RQ); 485 irdma_debug(qp->dev, IRDMA_DEBUG_WQE, 486 "hw_rq_size[%04d] qp_uk.rq_size[%04d] wqe_size[%04d]\n", 487 qp->hw_rq_size, qp->qp_uk.rq_size, wqe_size); 488 489 qp->sq_tph_val = info->sq_tph_val; 490 qp->rq_tph_val = info->rq_tph_val; 491 qp->sq_tph_en = info->sq_tph_en; 492 qp->rq_tph_en = info->rq_tph_en; 493 qp->rcv_tph_en = info->rcv_tph_en; 494 qp->xmit_tph_en = info->xmit_tph_en; 495 qp->qp_uk.first_sq_wq = info->qp_uk_init_info.first_sq_wq; 496 qp->qs_handle = qp->vsi->qos[qp->user_pri].qs_handle; 497 498 return 0; 499 } 500 501 /** 502 * irdma_sc_qp_create - create qp 503 * @qp: sc qp 504 * @info: qp create info 505 * @scratch: u64 saved to be used during cqp completion 506 * @post_sq: flag for cqp db to ring 507 */ 508 int 509 irdma_sc_qp_create(struct irdma_sc_qp *qp, struct irdma_create_qp_info *info, 510 u64 scratch, bool post_sq) 511 { 512 struct irdma_sc_cqp *cqp; 513 __le64 *wqe; 514 u64 hdr; 515 516 cqp = qp->dev->cqp; 517 if (qp->qp_uk.qp_id < cqp->dev->hw_attrs.min_hw_qp_id || 518 qp->qp_uk.qp_id > (cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt - 1)) 519 return -EINVAL; 520 521 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 522 if (!wqe) 523 return -ENOSPC; 524 525 set_64bit_val(wqe, IRDMA_BYTE_16, qp->hw_host_ctx_pa); 526 set_64bit_val(wqe, IRDMA_BYTE_40, qp->shadow_area_pa); 527 528 hdr = qp->qp_uk.qp_id | 529 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_QP) | 530 FIELD_PREP(IRDMA_CQPSQ_QP_ORDVALID, info->ord_valid) | 531 FIELD_PREP(IRDMA_CQPSQ_QP_TOECTXVALID, info->tcp_ctx_valid) | 532 FIELD_PREP(IRDMA_CQPSQ_QP_MACVALID, info->mac_valid) | 533 FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp->qp_uk.qp_type) | 534 FIELD_PREP(IRDMA_CQPSQ_QP_VQ, qp->virtual_map) | 535 FIELD_PREP(IRDMA_CQPSQ_QP_FORCELOOPBACK, info->force_lpb) | 536 FIELD_PREP(IRDMA_CQPSQ_QP_CQNUMVALID, info->cq_num_valid) | 537 FIELD_PREP(IRDMA_CQPSQ_QP_ARPTABIDXVALID, 538 info->arp_cache_idx_valid) | 539 FIELD_PREP(IRDMA_CQPSQ_QP_NEXTIWSTATE, info->next_iwarp_state) | 540 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 541 542 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 543 544 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 545 546 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "QP_CREATE WQE", wqe, 547 IRDMA_CQP_WQE_SIZE * 8); 548 if (post_sq) 549 irdma_sc_cqp_post_sq(cqp); 550 551 return 0; 552 } 553 554 /** 555 * irdma_sc_qp_modify - modify qp cqp wqe 556 * @qp: sc qp 557 * @info: modify qp info 558 * @scratch: u64 saved to be used during cqp completion 559 * @post_sq: flag for cqp db to ring 560 */ 561 int 562 irdma_sc_qp_modify(struct irdma_sc_qp *qp, struct irdma_modify_qp_info *info, 563 u64 scratch, bool post_sq) 564 { 565 __le64 *wqe; 566 struct irdma_sc_cqp *cqp; 567 u64 hdr; 568 u8 term_actions = 0; 569 u8 term_len = 0; 570 571 cqp = qp->dev->cqp; 572 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 573 if (!wqe) 574 return -ENOSPC; 575 576 if (info->next_iwarp_state == IRDMA_QP_STATE_TERMINATE) { 577 if (info->dont_send_fin) 578 term_actions += IRDMAQP_TERM_SEND_TERM_ONLY; 579 if (info->dont_send_term) 580 term_actions += IRDMAQP_TERM_SEND_FIN_ONLY; 581 if (term_actions == IRDMAQP_TERM_SEND_TERM_AND_FIN || 582 term_actions == IRDMAQP_TERM_SEND_TERM_ONLY) 583 term_len = info->termlen; 584 } 585 586 set_64bit_val(wqe, IRDMA_BYTE_8, 587 FIELD_PREP(IRDMA_CQPSQ_QP_NEWMSS, info->new_mss) | 588 FIELD_PREP(IRDMA_CQPSQ_QP_TERMLEN, term_len)); 589 set_64bit_val(wqe, IRDMA_BYTE_16, qp->hw_host_ctx_pa); 590 set_64bit_val(wqe, IRDMA_BYTE_40, qp->shadow_area_pa); 591 592 hdr = qp->qp_uk.qp_id | 593 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MODIFY_QP) | 594 FIELD_PREP(IRDMA_CQPSQ_QP_ORDVALID, info->ord_valid) | 595 FIELD_PREP(IRDMA_CQPSQ_QP_TOECTXVALID, info->tcp_ctx_valid) | 596 FIELD_PREP(IRDMA_CQPSQ_QP_CACHEDVARVALID, 597 info->cached_var_valid) | 598 FIELD_PREP(IRDMA_CQPSQ_QP_VQ, qp->virtual_map) | 599 FIELD_PREP(IRDMA_CQPSQ_QP_FORCELOOPBACK, info->force_lpb) | 600 FIELD_PREP(IRDMA_CQPSQ_QP_CQNUMVALID, info->cq_num_valid) | 601 FIELD_PREP(IRDMA_CQPSQ_QP_MACVALID, info->mac_valid) | 602 FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp->qp_uk.qp_type) | 603 FIELD_PREP(IRDMA_CQPSQ_QP_MSSCHANGE, info->mss_change) | 604 FIELD_PREP(IRDMA_CQPSQ_QP_REMOVEHASHENTRY, 605 info->remove_hash_idx) | 606 FIELD_PREP(IRDMA_CQPSQ_QP_TERMACT, term_actions) | 607 FIELD_PREP(IRDMA_CQPSQ_QP_RESETCON, info->reset_tcp_conn) | 608 FIELD_PREP(IRDMA_CQPSQ_QP_ARPTABIDXVALID, 609 info->arp_cache_idx_valid) | 610 FIELD_PREP(IRDMA_CQPSQ_QP_NEXTIWSTATE, info->next_iwarp_state) | 611 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 612 613 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 614 615 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 616 617 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "QP_MODIFY WQE", wqe, 618 IRDMA_CQP_WQE_SIZE * 8); 619 if (post_sq) 620 irdma_sc_cqp_post_sq(cqp); 621 622 return 0; 623 } 624 625 /** 626 * irdma_sc_qp_destroy - cqp destroy qp 627 * @qp: sc qp 628 * @scratch: u64 saved to be used during cqp completion 629 * @remove_hash_idx: flag if to remove hash idx 630 * @ignore_mw_bnd: memory window bind flag 631 * @post_sq: flag for cqp db to ring 632 */ 633 int 634 irdma_sc_qp_destroy(struct irdma_sc_qp *qp, u64 scratch, 635 bool remove_hash_idx, bool ignore_mw_bnd, bool post_sq) 636 { 637 __le64 *wqe; 638 struct irdma_sc_cqp *cqp; 639 u64 hdr; 640 641 cqp = qp->dev->cqp; 642 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 643 if (!wqe) 644 return -ENOSPC; 645 646 set_64bit_val(wqe, IRDMA_BYTE_16, qp->hw_host_ctx_pa); 647 set_64bit_val(wqe, IRDMA_BYTE_40, qp->shadow_area_pa); 648 649 hdr = qp->qp_uk.qp_id | 650 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_QP) | 651 FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp->qp_uk.qp_type) | 652 FIELD_PREP(IRDMA_CQPSQ_QP_IGNOREMWBOUND, ignore_mw_bnd) | 653 FIELD_PREP(IRDMA_CQPSQ_QP_REMOVEHASHENTRY, remove_hash_idx) | 654 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 655 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 656 657 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 658 659 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "QP_DESTROY WQE", wqe, 660 IRDMA_CQP_WQE_SIZE * 8); 661 if (post_sq) 662 irdma_sc_cqp_post_sq(cqp); 663 664 return 0; 665 } 666 667 /** 668 * irdma_sc_get_encoded_ird_size - 669 * @ird_size: IRD size 670 * The ird from the connection is rounded to a supported HW setting and then encoded 671 * for ird_size field of qp_ctx. Consumers are expected to provide valid ird size based 672 * on hardware attributes. IRD size defaults to a value of 4 in case of invalid input 673 */ 674 static u8 irdma_sc_get_encoded_ird_size(u16 ird_size) { 675 switch (ird_size ? 676 roundup_pow_of_two(2 * ird_size) : 4) { 677 case 256: 678 return IRDMA_IRD_HW_SIZE_256; 679 case 128: 680 return IRDMA_IRD_HW_SIZE_128; 681 case 64: 682 case 32: 683 return IRDMA_IRD_HW_SIZE_64; 684 case 16: 685 case 8: 686 return IRDMA_IRD_HW_SIZE_16; 687 case 4: 688 default: 689 break; 690 } 691 692 return IRDMA_IRD_HW_SIZE_4; 693 } 694 695 /** 696 * irdma_sc_qp_setctx_roce - set qp's context 697 * @qp: sc qp 698 * @qp_ctx: context ptr 699 * @info: ctx info 700 */ 701 void 702 irdma_sc_qp_setctx_roce(struct irdma_sc_qp *qp, __le64 * qp_ctx, 703 struct irdma_qp_host_ctx_info *info) 704 { 705 struct irdma_roce_offload_info *roce_info; 706 struct irdma_udp_offload_info *udp; 707 u8 push_mode_en; 708 u32 push_idx; 709 710 roce_info = info->roce_info; 711 udp = info->udp_info; 712 713 qp->user_pri = info->user_pri; 714 if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) { 715 push_mode_en = 0; 716 push_idx = 0; 717 } else { 718 push_mode_en = 1; 719 push_idx = qp->push_idx; 720 } 721 set_64bit_val(qp_ctx, IRDMA_BYTE_0, 722 FIELD_PREP(IRDMAQPC_RQWQESIZE, qp->qp_uk.rq_wqe_size) | 723 FIELD_PREP(IRDMAQPC_RCVTPHEN, qp->rcv_tph_en) | 724 FIELD_PREP(IRDMAQPC_XMITTPHEN, qp->xmit_tph_en) | 725 FIELD_PREP(IRDMAQPC_RQTPHEN, qp->rq_tph_en) | 726 FIELD_PREP(IRDMAQPC_SQTPHEN, qp->sq_tph_en) | 727 FIELD_PREP(IRDMAQPC_PPIDX, push_idx) | 728 FIELD_PREP(IRDMAQPC_PMENA, push_mode_en) | 729 FIELD_PREP(IRDMAQPC_PDIDXHI, roce_info->pd_id >> 16) | 730 FIELD_PREP(IRDMAQPC_DC_TCP_EN, roce_info->dctcp_en) | 731 FIELD_PREP(IRDMAQPC_ERR_RQ_IDX_VALID, roce_info->err_rq_idx_valid) | 732 FIELD_PREP(IRDMAQPC_ISQP1, roce_info->is_qp1) | 733 FIELD_PREP(IRDMAQPC_ROCE_TVER, roce_info->roce_tver) | 734 FIELD_PREP(IRDMAQPC_IPV4, udp->ipv4) | 735 FIELD_PREP(IRDMAQPC_INSERTVLANTAG, udp->insert_vlan_tag)); 736 set_64bit_val(qp_ctx, IRDMA_BYTE_8, qp->sq_pa); 737 set_64bit_val(qp_ctx, IRDMA_BYTE_16, qp->rq_pa); 738 if (roce_info->dcqcn_en || roce_info->dctcp_en) { 739 udp->tos &= ~ECN_CODE_PT_MASK; 740 udp->tos |= ECN_CODE_PT_VAL; 741 } 742 743 set_64bit_val(qp_ctx, IRDMA_BYTE_24, 744 FIELD_PREP(IRDMAQPC_RQSIZE, qp->hw_rq_size) | 745 FIELD_PREP(IRDMAQPC_SQSIZE, qp->hw_sq_size) | 746 FIELD_PREP(IRDMAQPC_TTL, udp->ttl) | FIELD_PREP(IRDMAQPC_TOS, udp->tos) | 747 FIELD_PREP(IRDMAQPC_SRCPORTNUM, udp->src_port) | 748 FIELD_PREP(IRDMAQPC_DESTPORTNUM, udp->dst_port)); 749 set_64bit_val(qp_ctx, IRDMA_BYTE_32, 750 FIELD_PREP(IRDMAQPC_DESTIPADDR2, udp->dest_ip_addr[2]) | 751 FIELD_PREP(IRDMAQPC_DESTIPADDR3, udp->dest_ip_addr[3])); 752 set_64bit_val(qp_ctx, IRDMA_BYTE_40, 753 FIELD_PREP(IRDMAQPC_DESTIPADDR0, udp->dest_ip_addr[0]) | 754 FIELD_PREP(IRDMAQPC_DESTIPADDR1, udp->dest_ip_addr[1])); 755 set_64bit_val(qp_ctx, IRDMA_BYTE_48, 756 FIELD_PREP(IRDMAQPC_SNDMSS, udp->snd_mss) | 757 FIELD_PREP(IRDMAQPC_VLANTAG, udp->vlan_tag) | 758 FIELD_PREP(IRDMAQPC_ARPIDX, udp->arp_idx)); 759 set_64bit_val(qp_ctx, IRDMA_BYTE_56, 760 FIELD_PREP(IRDMAQPC_PKEY, roce_info->p_key) | 761 FIELD_PREP(IRDMAQPC_PDIDX, roce_info->pd_id) | 762 FIELD_PREP(IRDMAQPC_ACKCREDITS, roce_info->ack_credits) | 763 FIELD_PREP(IRDMAQPC_FLOWLABEL, udp->flow_label)); 764 set_64bit_val(qp_ctx, IRDMA_BYTE_64, 765 FIELD_PREP(IRDMAQPC_QKEY, roce_info->qkey) | 766 FIELD_PREP(IRDMAQPC_DESTQP, roce_info->dest_qp)); 767 set_64bit_val(qp_ctx, IRDMA_BYTE_80, 768 FIELD_PREP(IRDMAQPC_PSNNXT, udp->psn_nxt) | 769 FIELD_PREP(IRDMAQPC_LSN, udp->lsn)); 770 set_64bit_val(qp_ctx, IRDMA_BYTE_88, 771 FIELD_PREP(IRDMAQPC_EPSN, udp->epsn)); 772 set_64bit_val(qp_ctx, IRDMA_BYTE_96, 773 FIELD_PREP(IRDMAQPC_PSNMAX, udp->psn_max) | 774 FIELD_PREP(IRDMAQPC_PSNUNA, udp->psn_una)); 775 set_64bit_val(qp_ctx, IRDMA_BYTE_112, 776 FIELD_PREP(IRDMAQPC_CWNDROCE, udp->cwnd)); 777 set_64bit_val(qp_ctx, IRDMA_BYTE_128, 778 FIELD_PREP(IRDMAQPC_ERR_RQ_IDX, roce_info->err_rq_idx) | 779 FIELD_PREP(IRDMAQPC_RNRNAK_THRESH, udp->rnr_nak_thresh) | 780 FIELD_PREP(IRDMAQPC_REXMIT_THRESH, udp->rexmit_thresh) | 781 FIELD_PREP(IRDMAQPC_RTOMIN, roce_info->rtomin)); 782 set_64bit_val(qp_ctx, IRDMA_BYTE_136, 783 FIELD_PREP(IRDMAQPC_TXCQNUM, info->send_cq_num) | 784 FIELD_PREP(IRDMAQPC_RXCQNUM, info->rcv_cq_num)); 785 set_64bit_val(qp_ctx, IRDMA_BYTE_144, 786 FIELD_PREP(IRDMAQPC_STAT_INDEX, info->stats_idx)); 787 set_64bit_val(qp_ctx, IRDMA_BYTE_152, 788 FIELD_PREP(IRDMAQPC_MACADDRESS, 789 irdma_mac_to_u64(roce_info->mac_addr))); 790 set_64bit_val(qp_ctx, IRDMA_BYTE_160, 791 FIELD_PREP(IRDMAQPC_ORDSIZE, roce_info->ord_size) | 792 FIELD_PREP(IRDMAQPC_IRDSIZE, irdma_sc_get_encoded_ird_size(roce_info->ird_size)) | 793 FIELD_PREP(IRDMAQPC_WRRDRSPOK, roce_info->wr_rdresp_en) | 794 FIELD_PREP(IRDMAQPC_RDOK, roce_info->rd_en) | 795 FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info->stats_idx_valid) | 796 FIELD_PREP(IRDMAQPC_FASTREGEN, roce_info->fast_reg_en) | 797 FIELD_PREP(IRDMAQPC_DCQCNENABLE, roce_info->dcqcn_en) | 798 FIELD_PREP(IRDMAQPC_RCVNOICRC, roce_info->rcv_no_icrc) | 799 FIELD_PREP(IRDMAQPC_FW_CC_ENABLE, roce_info->fw_cc_enable) | 800 FIELD_PREP(IRDMAQPC_UDPRIVCQENABLE, roce_info->udprivcq_en) | 801 FIELD_PREP(IRDMAQPC_PRIVEN, roce_info->priv_mode_en) | 802 FIELD_PREP(IRDMAQPC_TIMELYENABLE, roce_info->timely_en)); 803 set_64bit_val(qp_ctx, IRDMA_BYTE_168, 804 FIELD_PREP(IRDMAQPC_QPCOMPCTX, info->qp_compl_ctx)); 805 set_64bit_val(qp_ctx, IRDMA_BYTE_176, 806 FIELD_PREP(IRDMAQPC_SQTPHVAL, qp->sq_tph_val) | 807 FIELD_PREP(IRDMAQPC_RQTPHVAL, qp->rq_tph_val) | 808 FIELD_PREP(IRDMAQPC_QSHANDLE, qp->qs_handle)); 809 set_64bit_val(qp_ctx, IRDMA_BYTE_184, 810 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR3, udp->local_ipaddr[3]) | 811 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR2, udp->local_ipaddr[2])); 812 set_64bit_val(qp_ctx, IRDMA_BYTE_192, 813 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR1, udp->local_ipaddr[1]) | 814 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR0, udp->local_ipaddr[0])); 815 set_64bit_val(qp_ctx, IRDMA_BYTE_200, 816 FIELD_PREP(IRDMAQPC_THIGH, roce_info->t_high) | 817 FIELD_PREP(IRDMAQPC_TLOW, roce_info->t_low)); 818 set_64bit_val(qp_ctx, IRDMA_BYTE_208, 819 FIELD_PREP(IRDMAQPC_REMENDPOINTIDX, info->rem_endpoint_idx)); 820 821 irdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, "QP_HOST CTX WQE", qp_ctx, 822 IRDMA_QP_CTX_SIZE); 823 } 824 825 /* 826 * irdma_sc_alloc_local_mac_entry - allocate a mac entry @cqp: struct for cqp hw @scratch: u64 saved to be used during 827 * cqp completion @post_sq: flag for cqp db to ring 828 */ 829 static int 830 irdma_sc_alloc_local_mac_entry(struct irdma_sc_cqp *cqp, u64 scratch, 831 bool post_sq) 832 { 833 __le64 *wqe; 834 u64 hdr; 835 836 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 837 if (!wqe) 838 return -ENOSPC; 839 840 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, 841 IRDMA_CQP_OP_ALLOCATE_LOC_MAC_TABLE_ENTRY) | 842 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 843 844 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 845 846 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 847 848 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "ALLOCATE_LOCAL_MAC WQE", 849 wqe, IRDMA_CQP_WQE_SIZE * 8); 850 851 if (post_sq) 852 irdma_sc_cqp_post_sq(cqp); 853 return 0; 854 } 855 856 /** 857 * irdma_sc_add_local_mac_entry - add mac enry 858 * @cqp: struct for cqp hw 859 * @info:mac addr info 860 * @scratch: u64 saved to be used during cqp completion 861 * @post_sq: flag for cqp db to ring 862 */ 863 static int 864 irdma_sc_add_local_mac_entry(struct irdma_sc_cqp *cqp, 865 struct irdma_local_mac_entry_info *info, 866 u64 scratch, bool post_sq) 867 { 868 __le64 *wqe; 869 u64 header; 870 871 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 872 if (!wqe) 873 return -ENOSPC; 874 875 set_64bit_val(wqe, IRDMA_BYTE_32, irdma_mac_to_u64(info->mac_addr)); 876 877 header = FIELD_PREP(IRDMA_CQPSQ_MLM_TABLEIDX, info->entry_idx) | 878 FIELD_PREP(IRDMA_CQPSQ_OPCODE, 879 IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE) | 880 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 881 882 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 883 884 set_64bit_val(wqe, IRDMA_BYTE_24, header); 885 886 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "ADD_LOCAL_MAC WQE", wqe, 887 IRDMA_CQP_WQE_SIZE * 8); 888 889 if (post_sq) 890 irdma_sc_cqp_post_sq(cqp); 891 return 0; 892 } 893 894 /** 895 * irdma_sc_del_local_mac_entry - cqp wqe to dele local mac 896 * @cqp: struct for cqp hw 897 * @scratch: u64 saved to be used during cqp completion 898 * @entry_idx: index of mac entry 899 * @ignore_ref_count: to force mac adde delete 900 * @post_sq: flag for cqp db to ring 901 */ 902 static int 903 irdma_sc_del_local_mac_entry(struct irdma_sc_cqp *cqp, u64 scratch, 904 u16 entry_idx, u8 ignore_ref_count, 905 bool post_sq) 906 { 907 __le64 *wqe; 908 u64 header; 909 910 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 911 if (!wqe) 912 return -ENOSPC; 913 header = FIELD_PREP(IRDMA_CQPSQ_MLM_TABLEIDX, entry_idx) | 914 FIELD_PREP(IRDMA_CQPSQ_OPCODE, 915 IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE) | 916 FIELD_PREP(IRDMA_CQPSQ_MLM_FREEENTRY, 1) | 917 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) | 918 FIELD_PREP(IRDMA_CQPSQ_MLM_IGNORE_REF_CNT, ignore_ref_count); 919 920 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 921 922 set_64bit_val(wqe, IRDMA_BYTE_24, header); 923 924 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "DEL_LOCAL_MAC_IPADDR WQE", 925 wqe, IRDMA_CQP_WQE_SIZE * 8); 926 927 if (post_sq) 928 irdma_sc_cqp_post_sq(cqp); 929 return 0; 930 } 931 932 /** 933 * irdma_sc_qp_setctx - set qp's context 934 * @qp: sc qp 935 * @qp_ctx: context ptr 936 * @info: ctx info 937 */ 938 void 939 irdma_sc_qp_setctx(struct irdma_sc_qp *qp, __le64 * qp_ctx, 940 struct irdma_qp_host_ctx_info *info) 941 { 942 struct irdma_iwarp_offload_info *iw; 943 struct irdma_tcp_offload_info *tcp; 944 struct irdma_sc_dev *dev; 945 u8 push_mode_en; 946 u32 push_idx; 947 u64 qw0, qw3, qw7 = 0, qw16 = 0; 948 u64 mac = 0; 949 950 iw = info->iwarp_info; 951 tcp = info->tcp_info; 952 dev = qp->dev; 953 if (iw->rcv_mark_en) { 954 qp->pfpdu.marker_len = 4; 955 qp->pfpdu.rcv_start_seq = tcp->rcv_nxt; 956 } 957 qp->user_pri = info->user_pri; 958 if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) { 959 push_mode_en = 0; 960 push_idx = 0; 961 } else { 962 push_mode_en = 1; 963 push_idx = qp->push_idx; 964 } 965 qw0 = FIELD_PREP(IRDMAQPC_RQWQESIZE, qp->qp_uk.rq_wqe_size) | 966 FIELD_PREP(IRDMAQPC_RCVTPHEN, qp->rcv_tph_en) | 967 FIELD_PREP(IRDMAQPC_XMITTPHEN, qp->xmit_tph_en) | 968 FIELD_PREP(IRDMAQPC_RQTPHEN, qp->rq_tph_en) | 969 FIELD_PREP(IRDMAQPC_SQTPHEN, qp->sq_tph_en) | 970 FIELD_PREP(IRDMAQPC_PPIDX, push_idx) | 971 FIELD_PREP(IRDMAQPC_PMENA, push_mode_en); 972 973 set_64bit_val(qp_ctx, IRDMA_BYTE_8, qp->sq_pa); 974 set_64bit_val(qp_ctx, IRDMA_BYTE_16, qp->rq_pa); 975 976 qw3 = FIELD_PREP(IRDMAQPC_RQSIZE, qp->hw_rq_size) | 977 FIELD_PREP(IRDMAQPC_SQSIZE, qp->hw_sq_size); 978 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 979 qw3 |= FIELD_PREP(IRDMAQPC_GEN1_SRCMACADDRIDX, 980 qp->src_mac_addr_idx); 981 set_64bit_val(qp_ctx, IRDMA_BYTE_136, 982 FIELD_PREP(IRDMAQPC_TXCQNUM, info->send_cq_num) | 983 FIELD_PREP(IRDMAQPC_RXCQNUM, info->rcv_cq_num)); 984 set_64bit_val(qp_ctx, IRDMA_BYTE_168, 985 FIELD_PREP(IRDMAQPC_QPCOMPCTX, info->qp_compl_ctx)); 986 set_64bit_val(qp_ctx, IRDMA_BYTE_176, 987 FIELD_PREP(IRDMAQPC_SQTPHVAL, qp->sq_tph_val) | 988 FIELD_PREP(IRDMAQPC_RQTPHVAL, qp->rq_tph_val) | 989 FIELD_PREP(IRDMAQPC_QSHANDLE, qp->qs_handle) | 990 FIELD_PREP(IRDMAQPC_EXCEPTION_LAN_QUEUE, qp->ieq_qp)); 991 if (info->iwarp_info_valid) { 992 qw0 |= FIELD_PREP(IRDMAQPC_DDP_VER, iw->ddp_ver) | 993 FIELD_PREP(IRDMAQPC_RDMAP_VER, iw->rdmap_ver) | 994 FIELD_PREP(IRDMAQPC_DC_TCP_EN, iw->dctcp_en) | 995 FIELD_PREP(IRDMAQPC_ECN_EN, iw->ecn_en) | 996 FIELD_PREP(IRDMAQPC_IBRDENABLE, iw->ib_rd_en) | 997 FIELD_PREP(IRDMAQPC_PDIDXHI, iw->pd_id >> 16) | 998 FIELD_PREP(IRDMAQPC_ERR_RQ_IDX_VALID, 999 iw->err_rq_idx_valid); 1000 qw7 |= FIELD_PREP(IRDMAQPC_PDIDX, iw->pd_id); 1001 qw16 |= FIELD_PREP(IRDMAQPC_ERR_RQ_IDX, iw->err_rq_idx) | 1002 FIELD_PREP(IRDMAQPC_RTOMIN, iw->rtomin); 1003 set_64bit_val(qp_ctx, IRDMA_BYTE_144, 1004 FIELD_PREP(IRDMAQPC_Q2ADDR, qp->q2_pa >> 8) | 1005 FIELD_PREP(IRDMAQPC_STAT_INDEX, info->stats_idx)); 1006 1007 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 1008 mac = FIELD_PREP(IRDMAQPC_MACADDRESS, 1009 irdma_mac_to_u64(iw->mac_addr)); 1010 1011 set_64bit_val(qp_ctx, IRDMA_BYTE_152, 1012 mac | FIELD_PREP(IRDMAQPC_LASTBYTESENT, iw->last_byte_sent)); 1013 set_64bit_val(qp_ctx, IRDMA_BYTE_160, 1014 FIELD_PREP(IRDMAQPC_ORDSIZE, iw->ord_size) | 1015 FIELD_PREP(IRDMAQPC_IRDSIZE, irdma_sc_get_encoded_ird_size(iw->ird_size)) | 1016 FIELD_PREP(IRDMAQPC_WRRDRSPOK, iw->wr_rdresp_en) | 1017 FIELD_PREP(IRDMAQPC_RDOK, iw->rd_en) | 1018 FIELD_PREP(IRDMAQPC_SNDMARKERS, iw->snd_mark_en) | 1019 FIELD_PREP(IRDMAQPC_FASTREGEN, iw->fast_reg_en) | 1020 FIELD_PREP(IRDMAQPC_PRIVEN, iw->priv_mode_en) | 1021 FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info->stats_idx_valid) | 1022 FIELD_PREP(IRDMAQPC_IWARPMODE, 1) | 1023 FIELD_PREP(IRDMAQPC_RCVMARKERS, iw->rcv_mark_en) | 1024 FIELD_PREP(IRDMAQPC_ALIGNHDRS, iw->align_hdrs) | 1025 FIELD_PREP(IRDMAQPC_RCVNOMPACRC, iw->rcv_no_mpa_crc) | 1026 FIELD_PREP(IRDMAQPC_RCVMARKOFFSET, iw->rcv_mark_offset) | 1027 FIELD_PREP(IRDMAQPC_SNDMARKOFFSET, iw->snd_mark_offset) | 1028 FIELD_PREP(IRDMAQPC_TIMELYENABLE, iw->timely_en)); 1029 } 1030 if (info->tcp_info_valid) { 1031 qw0 |= FIELD_PREP(IRDMAQPC_IPV4, tcp->ipv4) | 1032 FIELD_PREP(IRDMAQPC_NONAGLE, tcp->no_nagle) | 1033 FIELD_PREP(IRDMAQPC_INSERTVLANTAG, 1034 tcp->insert_vlan_tag) | 1035 FIELD_PREP(IRDMAQPC_TIMESTAMP, tcp->time_stamp) | 1036 FIELD_PREP(IRDMAQPC_LIMIT, tcp->cwnd_inc_limit) | 1037 FIELD_PREP(IRDMAQPC_DROPOOOSEG, tcp->drop_ooo_seg) | 1038 FIELD_PREP(IRDMAQPC_DUPACK_THRESH, tcp->dup_ack_thresh); 1039 1040 if (iw->ecn_en || iw->dctcp_en) { 1041 tcp->tos &= ~ECN_CODE_PT_MASK; 1042 tcp->tos |= ECN_CODE_PT_VAL; 1043 } 1044 1045 qw3 |= FIELD_PREP(IRDMAQPC_TTL, tcp->ttl) | 1046 FIELD_PREP(IRDMAQPC_AVOIDSTRETCHACK, tcp->avoid_stretch_ack) | 1047 FIELD_PREP(IRDMAQPC_TOS, tcp->tos) | 1048 FIELD_PREP(IRDMAQPC_SRCPORTNUM, tcp->src_port) | 1049 FIELD_PREP(IRDMAQPC_DESTPORTNUM, tcp->dst_port); 1050 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) { 1051 qw3 |= FIELD_PREP(IRDMAQPC_GEN1_SRCMACADDRIDX, tcp->src_mac_addr_idx); 1052 1053 qp->src_mac_addr_idx = tcp->src_mac_addr_idx; 1054 } 1055 set_64bit_val(qp_ctx, IRDMA_BYTE_32, 1056 FIELD_PREP(IRDMAQPC_DESTIPADDR2, tcp->dest_ip_addr[2]) | 1057 FIELD_PREP(IRDMAQPC_DESTIPADDR3, tcp->dest_ip_addr[3])); 1058 set_64bit_val(qp_ctx, IRDMA_BYTE_40, 1059 FIELD_PREP(IRDMAQPC_DESTIPADDR0, tcp->dest_ip_addr[0]) | 1060 FIELD_PREP(IRDMAQPC_DESTIPADDR1, tcp->dest_ip_addr[1])); 1061 set_64bit_val(qp_ctx, IRDMA_BYTE_48, 1062 FIELD_PREP(IRDMAQPC_SNDMSS, tcp->snd_mss) | 1063 FIELD_PREP(IRDMAQPC_SYN_RST_HANDLING, tcp->syn_rst_handling) | 1064 FIELD_PREP(IRDMAQPC_VLANTAG, tcp->vlan_tag) | 1065 FIELD_PREP(IRDMAQPC_ARPIDX, tcp->arp_idx)); 1066 qw7 |= FIELD_PREP(IRDMAQPC_FLOWLABEL, tcp->flow_label) | 1067 FIELD_PREP(IRDMAQPC_WSCALE, tcp->wscale) | 1068 FIELD_PREP(IRDMAQPC_IGNORE_TCP_OPT, 1069 tcp->ignore_tcp_opt) | 1070 FIELD_PREP(IRDMAQPC_IGNORE_TCP_UNS_OPT, 1071 tcp->ignore_tcp_uns_opt) | 1072 FIELD_PREP(IRDMAQPC_TCPSTATE, tcp->tcp_state) | 1073 FIELD_PREP(IRDMAQPC_RCVSCALE, tcp->rcv_wscale) | 1074 FIELD_PREP(IRDMAQPC_SNDSCALE, tcp->snd_wscale); 1075 set_64bit_val(qp_ctx, IRDMA_BYTE_72, 1076 FIELD_PREP(IRDMAQPC_TIMESTAMP_RECENT, tcp->time_stamp_recent) | 1077 FIELD_PREP(IRDMAQPC_TIMESTAMP_AGE, tcp->time_stamp_age)); 1078 set_64bit_val(qp_ctx, IRDMA_BYTE_80, 1079 FIELD_PREP(IRDMAQPC_SNDNXT, tcp->snd_nxt) | 1080 FIELD_PREP(IRDMAQPC_SNDWND, tcp->snd_wnd)); 1081 set_64bit_val(qp_ctx, IRDMA_BYTE_88, 1082 FIELD_PREP(IRDMAQPC_RCVNXT, tcp->rcv_nxt) | 1083 FIELD_PREP(IRDMAQPC_RCVWND, tcp->rcv_wnd)); 1084 set_64bit_val(qp_ctx, IRDMA_BYTE_96, 1085 FIELD_PREP(IRDMAQPC_SNDMAX, tcp->snd_max) | 1086 FIELD_PREP(IRDMAQPC_SNDUNA, tcp->snd_una)); 1087 set_64bit_val(qp_ctx, IRDMA_BYTE_104, 1088 FIELD_PREP(IRDMAQPC_SRTT, tcp->srtt) | 1089 FIELD_PREP(IRDMAQPC_RTTVAR, tcp->rtt_var)); 1090 set_64bit_val(qp_ctx, IRDMA_BYTE_112, 1091 FIELD_PREP(IRDMAQPC_SSTHRESH, tcp->ss_thresh) | 1092 FIELD_PREP(IRDMAQPC_CWND, tcp->cwnd)); 1093 set_64bit_val(qp_ctx, IRDMA_BYTE_120, 1094 FIELD_PREP(IRDMAQPC_SNDWL1, tcp->snd_wl1) | 1095 FIELD_PREP(IRDMAQPC_SNDWL2, tcp->snd_wl2)); 1096 qw16 |= FIELD_PREP(IRDMAQPC_MAXSNDWND, tcp->max_snd_window) | 1097 FIELD_PREP(IRDMAQPC_REXMIT_THRESH, tcp->rexmit_thresh); 1098 set_64bit_val(qp_ctx, IRDMA_BYTE_184, 1099 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR3, tcp->local_ipaddr[3]) | 1100 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR2, tcp->local_ipaddr[2])); 1101 set_64bit_val(qp_ctx, IRDMA_BYTE_192, 1102 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR1, tcp->local_ipaddr[1]) | 1103 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR0, tcp->local_ipaddr[0])); 1104 set_64bit_val(qp_ctx, IRDMA_BYTE_200, 1105 FIELD_PREP(IRDMAQPC_THIGH, iw->t_high) | 1106 FIELD_PREP(IRDMAQPC_TLOW, iw->t_low)); 1107 set_64bit_val(qp_ctx, IRDMA_BYTE_208, 1108 FIELD_PREP(IRDMAQPC_REMENDPOINTIDX, info->rem_endpoint_idx)); 1109 } 1110 1111 set_64bit_val(qp_ctx, IRDMA_BYTE_0, qw0); 1112 set_64bit_val(qp_ctx, IRDMA_BYTE_24, qw3); 1113 set_64bit_val(qp_ctx, IRDMA_BYTE_56, qw7); 1114 set_64bit_val(qp_ctx, IRDMA_BYTE_128, qw16); 1115 1116 irdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, "QP_HOST CTX", qp_ctx, 1117 IRDMA_QP_CTX_SIZE); 1118 } 1119 1120 /** 1121 * irdma_sc_alloc_stag - mr stag alloc 1122 * @dev: sc device struct 1123 * @info: stag info 1124 * @scratch: u64 saved to be used during cqp completion 1125 * @post_sq: flag for cqp db to ring 1126 */ 1127 static int 1128 irdma_sc_alloc_stag(struct irdma_sc_dev *dev, 1129 struct irdma_allocate_stag_info *info, 1130 u64 scratch, bool post_sq) 1131 { 1132 __le64 *wqe; 1133 struct irdma_sc_cqp *cqp; 1134 u64 hdr; 1135 enum irdma_page_size page_size; 1136 1137 if (!info->total_len && !info->all_memory) 1138 return -EINVAL; 1139 1140 if (info->page_size == 0x40000000) 1141 page_size = IRDMA_PAGE_SIZE_1G; 1142 else if (info->page_size == 0x200000) 1143 page_size = IRDMA_PAGE_SIZE_2M; 1144 else 1145 page_size = IRDMA_PAGE_SIZE_4K; 1146 1147 cqp = dev->cqp; 1148 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 1149 if (!wqe) 1150 return -ENOSPC; 1151 1152 set_64bit_val(wqe, IRDMA_BYTE_8, 1153 FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID) | 1154 FIELD_PREP(IRDMA_CQPSQ_STAG_STAGLEN, info->total_len)); 1155 set_64bit_val(wqe, IRDMA_BYTE_16, 1156 FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->stag_idx)); 1157 set_64bit_val(wqe, IRDMA_BYTE_40, 1158 FIELD_PREP(IRDMA_CQPSQ_STAG_HMCFNIDX, info->hmc_fcn_index)); 1159 1160 if (info->chunk_size) 1161 set_64bit_val(wqe, IRDMA_BYTE_48, 1162 FIELD_PREP(IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX, info->first_pm_pbl_idx)); 1163 1164 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_ALLOC_STAG) | 1165 FIELD_PREP(IRDMA_CQPSQ_STAG_MR, 1) | 1166 FIELD_PREP(IRDMA_CQPSQ_STAG_ARIGHTS, info->access_rights) | 1167 FIELD_PREP(IRDMA_CQPSQ_STAG_LPBLSIZE, info->chunk_size) | 1168 FIELD_PREP(IRDMA_CQPSQ_STAG_HPAGESIZE, page_size) | 1169 FIELD_PREP(IRDMA_CQPSQ_STAG_REMACCENABLED, info->remote_access) | 1170 FIELD_PREP(IRDMA_CQPSQ_STAG_USEHMCFNIDX, info->use_hmc_fcn_index) | 1171 FIELD_PREP(IRDMA_CQPSQ_STAG_USEPFRID, info->use_pf_rid) | 1172 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 1173 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 1174 1175 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 1176 1177 irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "ALLOC_STAG WQE", wqe, 1178 IRDMA_CQP_WQE_SIZE * 8); 1179 if (post_sq) 1180 irdma_sc_cqp_post_sq(cqp); 1181 1182 return 0; 1183 } 1184 1185 /** 1186 * irdma_sc_mr_reg_non_shared - non-shared mr registration 1187 * @dev: sc device struct 1188 * @info: mr info 1189 * @scratch: u64 saved to be used during cqp completion 1190 * @post_sq: flag for cqp db to ring 1191 */ 1192 static int 1193 irdma_sc_mr_reg_non_shared(struct irdma_sc_dev *dev, 1194 struct irdma_reg_ns_stag_info *info, 1195 u64 scratch, bool post_sq) 1196 { 1197 __le64 *wqe; 1198 u64 fbo; 1199 struct irdma_sc_cqp *cqp; 1200 u64 hdr; 1201 u32 pble_obj_cnt; 1202 bool remote_access; 1203 u8 addr_type; 1204 enum irdma_page_size page_size; 1205 1206 if (!info->total_len && !info->all_memory) 1207 return -EINVAL; 1208 1209 if (info->page_size == 0x40000000) 1210 page_size = IRDMA_PAGE_SIZE_1G; 1211 else if (info->page_size == 0x200000) 1212 page_size = IRDMA_PAGE_SIZE_2M; 1213 else if (info->page_size == 0x1000) 1214 page_size = IRDMA_PAGE_SIZE_4K; 1215 else 1216 return -EINVAL; 1217 1218 if (info->access_rights & (IRDMA_ACCESS_FLAGS_REMOTEREAD_ONLY | 1219 IRDMA_ACCESS_FLAGS_REMOTEWRITE_ONLY)) 1220 remote_access = true; 1221 else 1222 remote_access = false; 1223 1224 pble_obj_cnt = dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; 1225 if (info->chunk_size && info->first_pm_pbl_index >= pble_obj_cnt) 1226 return -EINVAL; 1227 1228 cqp = dev->cqp; 1229 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 1230 if (!wqe) 1231 return -ENOSPC; 1232 fbo = info->va & (info->page_size - 1); 1233 1234 set_64bit_val(wqe, IRDMA_BYTE_0, 1235 (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED ? 1236 info->va : fbo)); 1237 set_64bit_val(wqe, IRDMA_BYTE_8, 1238 FIELD_PREP(IRDMA_CQPSQ_STAG_STAGLEN, info->total_len) | 1239 FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID)); 1240 set_64bit_val(wqe, IRDMA_BYTE_16, 1241 FIELD_PREP(IRDMA_CQPSQ_STAG_KEY, info->stag_key) | 1242 FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->stag_idx)); 1243 if (!info->chunk_size) 1244 set_64bit_val(wqe, IRDMA_BYTE_32, info->reg_addr_pa); 1245 else 1246 set_64bit_val(wqe, IRDMA_BYTE_48, 1247 FIELD_PREP(IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX, info->first_pm_pbl_index)); 1248 1249 set_64bit_val(wqe, IRDMA_BYTE_40, info->hmc_fcn_index); 1250 1251 addr_type = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ? 1 : 0; 1252 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_REG_MR) | 1253 FIELD_PREP(IRDMA_CQPSQ_STAG_MR, 1) | 1254 FIELD_PREP(IRDMA_CQPSQ_STAG_LPBLSIZE, info->chunk_size) | 1255 FIELD_PREP(IRDMA_CQPSQ_STAG_HPAGESIZE, page_size) | 1256 FIELD_PREP(IRDMA_CQPSQ_STAG_ARIGHTS, info->access_rights) | 1257 FIELD_PREP(IRDMA_CQPSQ_STAG_REMACCENABLED, remote_access) | 1258 FIELD_PREP(IRDMA_CQPSQ_STAG_VABASEDTO, addr_type) | 1259 FIELD_PREP(IRDMA_CQPSQ_STAG_USEHMCFNIDX, info->use_hmc_fcn_index) | 1260 FIELD_PREP(IRDMA_CQPSQ_STAG_USEPFRID, info->use_pf_rid) | 1261 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 1262 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 1263 1264 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 1265 1266 irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "MR_REG_NS WQE", wqe, 1267 IRDMA_CQP_WQE_SIZE * 8); 1268 if (post_sq) 1269 irdma_sc_cqp_post_sq(cqp); 1270 1271 return 0; 1272 } 1273 1274 /** 1275 * irdma_sc_dealloc_stag - deallocate stag 1276 * @dev: sc device struct 1277 * @info: dealloc stag info 1278 * @scratch: u64 saved to be used during cqp completion 1279 * @post_sq: flag for cqp db to ring 1280 */ 1281 static int 1282 irdma_sc_dealloc_stag(struct irdma_sc_dev *dev, 1283 struct irdma_dealloc_stag_info *info, 1284 u64 scratch, bool post_sq) 1285 { 1286 u64 hdr; 1287 __le64 *wqe; 1288 struct irdma_sc_cqp *cqp; 1289 1290 cqp = dev->cqp; 1291 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 1292 if (!wqe) 1293 return -ENOSPC; 1294 1295 set_64bit_val(wqe, IRDMA_BYTE_8, 1296 FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID)); 1297 set_64bit_val(wqe, IRDMA_BYTE_16, 1298 FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->stag_idx)); 1299 1300 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DEALLOC_STAG) | 1301 FIELD_PREP(IRDMA_CQPSQ_STAG_MR, info->mr) | 1302 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 1303 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 1304 1305 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 1306 1307 irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "DEALLOC_STAG WQE", wqe, 1308 IRDMA_CQP_WQE_SIZE * 8); 1309 if (post_sq) 1310 irdma_sc_cqp_post_sq(cqp); 1311 1312 return 0; 1313 } 1314 1315 /** 1316 * irdma_sc_mw_alloc - mw allocate 1317 * @dev: sc device struct 1318 * @info: memory window allocation information 1319 * @scratch: u64 saved to be used during cqp completion 1320 * @post_sq: flag for cqp db to ring 1321 */ 1322 static int 1323 irdma_sc_mw_alloc(struct irdma_sc_dev *dev, 1324 struct irdma_mw_alloc_info *info, u64 scratch, 1325 bool post_sq) 1326 { 1327 u64 hdr; 1328 struct irdma_sc_cqp *cqp; 1329 __le64 *wqe; 1330 1331 cqp = dev->cqp; 1332 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 1333 if (!wqe) 1334 return -ENOSPC; 1335 1336 set_64bit_val(wqe, IRDMA_BYTE_8, 1337 FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID)); 1338 set_64bit_val(wqe, IRDMA_BYTE_16, 1339 FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->mw_stag_index)); 1340 1341 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_ALLOC_STAG) | 1342 FIELD_PREP(IRDMA_CQPSQ_STAG_MWTYPE, info->mw_wide) | 1343 FIELD_PREP(IRDMA_CQPSQ_STAG_MW1_BIND_DONT_VLDT_KEY, 1344 info->mw1_bind_dont_vldt_key) | 1345 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 1346 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 1347 1348 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 1349 1350 irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "MW_ALLOC WQE", wqe, 1351 IRDMA_CQP_WQE_SIZE * 8); 1352 if (post_sq) 1353 irdma_sc_cqp_post_sq(cqp); 1354 1355 return 0; 1356 } 1357 1358 /** 1359 * irdma_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp 1360 * @qp: sc qp struct 1361 * @info: fast mr info 1362 * @post_sq: flag for cqp db to ring 1363 */ 1364 int 1365 irdma_sc_mr_fast_register(struct irdma_sc_qp *qp, 1366 struct irdma_fast_reg_stag_info *info, 1367 bool post_sq) 1368 { 1369 u64 temp, hdr; 1370 __le64 *wqe; 1371 u32 wqe_idx; 1372 u16 quanta = IRDMA_QP_WQE_MIN_QUANTA; 1373 enum irdma_page_size page_size; 1374 struct irdma_post_sq_info sq_info = {0}; 1375 1376 if (info->page_size == 0x40000000) 1377 page_size = IRDMA_PAGE_SIZE_1G; 1378 else if (info->page_size == 0x200000) 1379 page_size = IRDMA_PAGE_SIZE_2M; 1380 else 1381 page_size = IRDMA_PAGE_SIZE_4K; 1382 1383 sq_info.wr_id = info->wr_id; 1384 sq_info.signaled = info->signaled; 1385 sq_info.push_wqe = info->push_wqe; 1386 1387 wqe = irdma_qp_get_next_send_wqe(&qp->qp_uk, &wqe_idx, &quanta, 0, &sq_info); 1388 if (!wqe) 1389 return -ENOSPC; 1390 1391 qp->qp_uk.sq_wrtrk_array[wqe_idx].signaled = info->signaled; 1392 irdma_debug(qp->dev, IRDMA_DEBUG_MR, 1393 "wr_id[%llxh] wqe_idx[%04d] location[%p]\n", 1394 (unsigned long long)info->wr_id, wqe_idx, 1395 &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid); 1396 1397 temp = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ? 1398 (uintptr_t)info->va : info->fbo; 1399 set_64bit_val(wqe, IRDMA_BYTE_0, temp); 1400 1401 temp = FIELD_GET(IRDMAQPSQ_FIRSTPMPBLIDXHI, 1402 info->first_pm_pbl_index >> 16); 1403 set_64bit_val(wqe, IRDMA_BYTE_8, 1404 FIELD_PREP(IRDMAQPSQ_FIRSTPMPBLIDXHI, temp) | 1405 FIELD_PREP(IRDMAQPSQ_PBLADDR, info->reg_addr_pa >> IRDMA_HW_PAGE_SHIFT)); 1406 set_64bit_val(wqe, IRDMA_BYTE_16, 1407 info->total_len | 1408 FIELD_PREP(IRDMAQPSQ_FIRSTPMPBLIDXLO, info->first_pm_pbl_index)); 1409 1410 hdr = FIELD_PREP(IRDMAQPSQ_STAGKEY, info->stag_key) | 1411 FIELD_PREP(IRDMAQPSQ_STAGINDEX, info->stag_idx) | 1412 FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_FAST_REGISTER) | 1413 FIELD_PREP(IRDMAQPSQ_LPBLSIZE, info->chunk_size) | 1414 FIELD_PREP(IRDMAQPSQ_HPAGESIZE, page_size) | 1415 FIELD_PREP(IRDMAQPSQ_STAGRIGHTS, info->access_rights) | 1416 FIELD_PREP(IRDMAQPSQ_VABASEDTO, info->addr_type) | 1417 FIELD_PREP(IRDMAQPSQ_PUSHWQE, (sq_info.push_wqe ? 1 : 0)) | 1418 FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) | 1419 FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) | 1420 FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | 1421 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity); 1422 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 1423 1424 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 1425 1426 irdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, "FAST_REG WQE", wqe, 1427 IRDMA_QP_WQE_MIN_SIZE); 1428 if (sq_info.push_wqe) 1429 irdma_qp_push_wqe(&qp->qp_uk, wqe, quanta, wqe_idx, post_sq); 1430 else if (post_sq) 1431 irdma_uk_qp_post_wr(&qp->qp_uk); 1432 1433 return 0; 1434 } 1435 1436 /** 1437 * irdma_sc_gen_rts_ae - request AE generated after RTS 1438 * @qp: sc qp struct 1439 */ 1440 static void 1441 irdma_sc_gen_rts_ae(struct irdma_sc_qp *qp) 1442 { 1443 __le64 *wqe; 1444 u64 hdr; 1445 struct irdma_qp_uk *qp_uk; 1446 1447 qp_uk = &qp->qp_uk; 1448 1449 wqe = qp_uk->sq_base[1].elem; 1450 1451 hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) | 1452 FIELD_PREP(IRDMAQPSQ_LOCALFENCE, 1) | 1453 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity); 1454 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 1455 1456 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 1457 irdma_debug_buf(qp->dev, IRDMA_DEBUG_QP, "NOP W/LOCAL FENCE WQE", wqe, 1458 IRDMA_QP_WQE_MIN_SIZE); 1459 1460 wqe = qp_uk->sq_base[2].elem; 1461 hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_GEN_RTS_AE) | 1462 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity); 1463 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 1464 1465 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 1466 irdma_debug_buf(qp->dev, IRDMA_DEBUG_QP, "CONN EST WQE", wqe, 1467 IRDMA_QP_WQE_MIN_SIZE); 1468 if (qp->qp_uk.start_wqe_idx) { 1469 wqe = qp_uk->sq_base[3].elem; 1470 hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) | 1471 FIELD_PREP(IRDMAQPSQ_LOCALFENCE, 1) | 1472 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity); 1473 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 1474 1475 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 1476 } 1477 } 1478 1479 /** 1480 * irdma_sc_send_lsmm - send last streaming mode message 1481 * @qp: sc qp struct 1482 * @lsmm_buf: buffer with lsmm message 1483 * @size: size of lsmm buffer 1484 * @stag: stag of lsmm buffer 1485 */ 1486 void 1487 irdma_sc_send_lsmm(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size, 1488 irdma_stag stag) 1489 { 1490 __le64 *wqe; 1491 u64 hdr; 1492 struct irdma_qp_uk *qp_uk; 1493 1494 qp_uk = &qp->qp_uk; 1495 wqe = qp_uk->sq_base->elem; 1496 1497 set_64bit_val(wqe, IRDMA_BYTE_0, (uintptr_t)lsmm_buf); 1498 if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) { 1499 set_64bit_val(wqe, IRDMA_BYTE_8, 1500 FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, size) | 1501 FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, stag)); 1502 } else { 1503 set_64bit_val(wqe, IRDMA_BYTE_8, 1504 FIELD_PREP(IRDMAQPSQ_FRAG_LEN, size) | 1505 FIELD_PREP(IRDMAQPSQ_FRAG_STAG, stag) | 1506 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity)); 1507 } 1508 set_64bit_val(wqe, IRDMA_BYTE_16, 0); 1509 1510 hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_SEND) | 1511 FIELD_PREP(IRDMAQPSQ_STREAMMODE, 1) | 1512 FIELD_PREP(IRDMAQPSQ_WAITFORRCVPDU, 1) | 1513 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity); 1514 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 1515 1516 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 1517 1518 irdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, "SEND_LSMM WQE", wqe, 1519 IRDMA_QP_WQE_MIN_SIZE); 1520 1521 if (qp->dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE) 1522 irdma_sc_gen_rts_ae(qp); 1523 } 1524 1525 /** 1526 * irdma_sc_send_rtt - send last read0 or write0 1527 * @qp: sc qp struct 1528 * @read: Do read0 or write0 1529 */ 1530 void 1531 irdma_sc_send_rtt(struct irdma_sc_qp *qp, bool read) 1532 { 1533 __le64 *wqe; 1534 u64 hdr; 1535 struct irdma_qp_uk *qp_uk; 1536 1537 qp_uk = &qp->qp_uk; 1538 wqe = qp_uk->sq_base->elem; 1539 1540 set_64bit_val(wqe, IRDMA_BYTE_0, 0); 1541 set_64bit_val(wqe, IRDMA_BYTE_16, 0); 1542 if (read) { 1543 if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) { 1544 set_64bit_val(wqe, IRDMA_BYTE_8, 1545 FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, 0xabcd)); 1546 } else { 1547 set_64bit_val(wqe, IRDMA_BYTE_8, 1548 (u64)0xabcd | FIELD_PREP(IRDMAQPSQ_VALID, 1549 qp->qp_uk.swqe_polarity)); 1550 } 1551 hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, 0x1234) | 1552 FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_READ) | 1553 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity); 1554 1555 } else { 1556 if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) { 1557 set_64bit_val(wqe, IRDMA_BYTE_8, 0); 1558 } else { 1559 set_64bit_val(wqe, IRDMA_BYTE_8, 1560 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity)); 1561 } 1562 hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_WRITE) | 1563 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity); 1564 } 1565 1566 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 1567 1568 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 1569 1570 irdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, "RTR WQE", wqe, 1571 IRDMA_QP_WQE_MIN_SIZE); 1572 1573 if (qp->dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE) 1574 irdma_sc_gen_rts_ae(qp); 1575 } 1576 1577 /** 1578 * irdma_iwarp_opcode - determine if incoming is rdma layer 1579 * @info: aeq info for the packet 1580 * @pkt: packet for error 1581 */ 1582 static u32 irdma_iwarp_opcode(struct irdma_aeqe_info *info, u8 *pkt){ 1583 BE16 *mpa; 1584 u32 opcode = 0xffffffff; 1585 1586 if (info->q2_data_written) { 1587 mpa = (BE16 *) pkt; 1588 opcode = IRDMA_NTOHS(mpa[1]) & 0xf; 1589 } 1590 1591 return opcode; 1592 } 1593 1594 /** 1595 * irdma_locate_mpa - return pointer to mpa in the pkt 1596 * @pkt: packet with data 1597 */ 1598 static u8 *irdma_locate_mpa(u8 *pkt) { 1599 /* skip over ethernet header */ 1600 pkt += IRDMA_MAC_HLEN; 1601 1602 /* Skip over IP and TCP headers */ 1603 pkt += 4 * (pkt[0] & 0x0f); 1604 pkt += 4 * ((pkt[12] >> 4) & 0x0f); 1605 1606 return pkt; 1607 } 1608 1609 /** 1610 * irdma_bld_termhdr_ctrl - setup terminate hdr control fields 1611 * @qp: sc qp ptr for pkt 1612 * @hdr: term hdr 1613 * @opcode: flush opcode for termhdr 1614 * @layer_etype: error layer + error type 1615 * @err: error cod ein the header 1616 */ 1617 static void 1618 irdma_bld_termhdr_ctrl(struct irdma_sc_qp *qp, 1619 struct irdma_terminate_hdr *hdr, 1620 enum irdma_flush_opcode opcode, 1621 u8 layer_etype, u8 err) 1622 { 1623 qp->flush_code = opcode; 1624 hdr->layer_etype = layer_etype; 1625 hdr->error_code = err; 1626 } 1627 1628 /** 1629 * irdma_bld_termhdr_ddp_rdma - setup ddp and rdma hdrs in terminate hdr 1630 * @pkt: ptr to mpa in offending pkt 1631 * @hdr: term hdr 1632 * @copy_len: offending pkt length to be copied to term hdr 1633 * @is_tagged: DDP tagged or untagged 1634 */ 1635 static void 1636 irdma_bld_termhdr_ddp_rdma(u8 *pkt, struct irdma_terminate_hdr *hdr, 1637 int *copy_len, u8 *is_tagged) 1638 { 1639 u16 ddp_seg_len; 1640 1641 ddp_seg_len = IRDMA_NTOHS(*(BE16 *) pkt); 1642 if (ddp_seg_len) { 1643 *copy_len = 2; 1644 hdr->hdrct = DDP_LEN_FLAG; 1645 if (pkt[2] & 0x80) { 1646 *is_tagged = 1; 1647 if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) { 1648 *copy_len += TERM_DDP_LEN_TAGGED; 1649 hdr->hdrct |= DDP_HDR_FLAG; 1650 } 1651 } else { 1652 if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) { 1653 *copy_len += TERM_DDP_LEN_UNTAGGED; 1654 hdr->hdrct |= DDP_HDR_FLAG; 1655 } 1656 if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN) && 1657 ((pkt[3] & RDMA_OPCODE_M) == RDMA_READ_REQ_OPCODE)) { 1658 *copy_len += TERM_RDMA_LEN; 1659 hdr->hdrct |= RDMA_HDR_FLAG; 1660 } 1661 } 1662 } 1663 } 1664 1665 /** 1666 * irdma_bld_terminate_hdr - build terminate message header 1667 * @qp: qp associated with received terminate AE 1668 * @info: the struct contiaing AE information 1669 */ 1670 static int 1671 irdma_bld_terminate_hdr(struct irdma_sc_qp *qp, 1672 struct irdma_aeqe_info *info) 1673 { 1674 u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET; 1675 int copy_len = 0; 1676 u8 is_tagged = 0; 1677 u32 opcode; 1678 struct irdma_terminate_hdr *termhdr; 1679 1680 termhdr = (struct irdma_terminate_hdr *)qp->q2_buf; 1681 memset(termhdr, 0, Q2_BAD_FRAME_OFFSET); 1682 1683 if (info->q2_data_written) { 1684 pkt = irdma_locate_mpa(pkt); 1685 irdma_bld_termhdr_ddp_rdma(pkt, termhdr, ©_len, &is_tagged); 1686 } 1687 1688 opcode = irdma_iwarp_opcode(info, pkt); 1689 qp->event_type = IRDMA_QP_EVENT_CATASTROPHIC; 1690 qp->sq_flush_code = info->sq; 1691 qp->rq_flush_code = info->rq; 1692 1693 switch (info->ae_id) { 1694 case IRDMA_AE_AMP_UNALLOCATED_STAG: 1695 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; 1696 if (opcode == IRDMA_OP_TYPE_RDMA_WRITE) 1697 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_PROT_ERR, 1698 (LAYER_DDP << 4) | DDP_TAGGED_BUF, 1699 DDP_TAGGED_INV_STAG); 1700 else 1701 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR, 1702 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, 1703 RDMAP_INV_STAG); 1704 break; 1705 case IRDMA_AE_AMP_BOUNDS_VIOLATION: 1706 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; 1707 if (info->q2_data_written) 1708 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_PROT_ERR, 1709 (LAYER_DDP << 4) | DDP_TAGGED_BUF, 1710 DDP_TAGGED_BOUNDS); 1711 else 1712 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR, 1713 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, 1714 RDMAP_INV_BOUNDS); 1715 break; 1716 case IRDMA_AE_AMP_BAD_PD: 1717 switch (opcode) { 1718 case IRDMA_OP_TYPE_RDMA_WRITE: 1719 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_PROT_ERR, 1720 (LAYER_DDP << 4) | DDP_TAGGED_BUF, 1721 DDP_TAGGED_UNASSOC_STAG); 1722 break; 1723 case IRDMA_OP_TYPE_SEND_INV: 1724 case IRDMA_OP_TYPE_SEND_SOL_INV: 1725 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR, 1726 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, 1727 RDMAP_CANT_INV_STAG); 1728 break; 1729 default: 1730 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR, 1731 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, 1732 RDMAP_UNASSOC_STAG); 1733 } 1734 break; 1735 case IRDMA_AE_AMP_INVALID_STAG: 1736 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; 1737 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR, 1738 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, 1739 RDMAP_INV_STAG); 1740 break; 1741 case IRDMA_AE_AMP_BAD_QP: 1742 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_LOC_QP_OP_ERR, 1743 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF, 1744 DDP_UNTAGGED_INV_QN); 1745 break; 1746 case IRDMA_AE_AMP_BAD_STAG_KEY: 1747 case IRDMA_AE_AMP_BAD_STAG_INDEX: 1748 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; 1749 switch (opcode) { 1750 case IRDMA_OP_TYPE_SEND_INV: 1751 case IRDMA_OP_TYPE_SEND_SOL_INV: 1752 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_OP_ERR, 1753 (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, 1754 RDMAP_CANT_INV_STAG); 1755 break; 1756 default: 1757 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR, 1758 (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, 1759 RDMAP_INV_STAG); 1760 } 1761 break; 1762 case IRDMA_AE_AMP_RIGHTS_VIOLATION: 1763 case IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS: 1764 case IRDMA_AE_PRIV_OPERATION_DENIED: 1765 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; 1766 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR, 1767 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, 1768 RDMAP_ACCESS); 1769 break; 1770 case IRDMA_AE_AMP_TO_WRAP: 1771 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; 1772 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR, 1773 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, 1774 RDMAP_TO_WRAP); 1775 break; 1776 case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR: 1777 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR, 1778 (LAYER_MPA << 4) | DDP_LLP, MPA_CRC); 1779 break; 1780 case IRDMA_AE_LLP_SEGMENT_TOO_SMALL: 1781 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_LOC_LEN_ERR, 1782 (LAYER_DDP << 4) | DDP_CATASTROPHIC, 1783 DDP_CATASTROPHIC_LOCAL); 1784 break; 1785 case IRDMA_AE_LCE_QP_CATASTROPHIC: 1786 case IRDMA_AE_DDP_NO_L_BIT: 1787 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_FATAL_ERR, 1788 (LAYER_DDP << 4) | DDP_CATASTROPHIC, 1789 DDP_CATASTROPHIC_LOCAL); 1790 break; 1791 case IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN: 1792 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR, 1793 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF, 1794 DDP_UNTAGGED_INV_MSN_RANGE); 1795 break; 1796 case IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER: 1797 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; 1798 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_LOC_LEN_ERR, 1799 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF, 1800 DDP_UNTAGGED_INV_TOO_LONG); 1801 break; 1802 case IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION: 1803 if (is_tagged) 1804 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR, 1805 (LAYER_DDP << 4) | DDP_TAGGED_BUF, 1806 DDP_TAGGED_INV_DDP_VER); 1807 else 1808 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR, 1809 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF, 1810 DDP_UNTAGGED_INV_DDP_VER); 1811 break; 1812 case IRDMA_AE_DDP_UBE_INVALID_MO: 1813 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR, 1814 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF, 1815 DDP_UNTAGGED_INV_MO); 1816 break; 1817 case IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE: 1818 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_OP_ERR, 1819 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF, 1820 DDP_UNTAGGED_INV_MSN_NO_BUF); 1821 break; 1822 case IRDMA_AE_DDP_UBE_INVALID_QN: 1823 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR, 1824 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF, 1825 DDP_UNTAGGED_INV_QN); 1826 break; 1827 case IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION: 1828 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR, 1829 (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, 1830 RDMAP_INV_RDMAP_VER); 1831 break; 1832 default: 1833 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_FATAL_ERR, 1834 (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, 1835 RDMAP_UNSPECIFIED); 1836 break; 1837 } 1838 1839 if (copy_len) 1840 irdma_memcpy(termhdr + 1, pkt, copy_len); 1841 1842 return sizeof(*termhdr) + copy_len; 1843 } 1844 1845 /** 1846 * irdma_terminate_send_fin() - Send fin for terminate message 1847 * @qp: qp associated with received terminate AE 1848 */ 1849 void 1850 irdma_terminate_send_fin(struct irdma_sc_qp *qp) 1851 { 1852 irdma_term_modify_qp(qp, IRDMA_QP_STATE_TERMINATE, 1853 IRDMAQP_TERM_SEND_FIN_ONLY, 0); 1854 } 1855 1856 /** 1857 * irdma_terminate_connection() - Bad AE and send terminate to remote QP 1858 * @qp: qp associated with received terminate AE 1859 * @info: the struct contiaing AE information 1860 */ 1861 void 1862 irdma_terminate_connection(struct irdma_sc_qp *qp, 1863 struct irdma_aeqe_info *info) 1864 { 1865 u8 termlen = 0; 1866 1867 if (qp->term_flags & IRDMA_TERM_SENT) 1868 return; 1869 1870 termlen = irdma_bld_terminate_hdr(qp, info); 1871 irdma_terminate_start_timer(qp); 1872 qp->term_flags |= IRDMA_TERM_SENT; 1873 irdma_term_modify_qp(qp, IRDMA_QP_STATE_TERMINATE, 1874 IRDMAQP_TERM_SEND_TERM_ONLY, termlen); 1875 } 1876 1877 /** 1878 * irdma_terminate_received - handle terminate received AE 1879 * @qp: qp associated with received terminate AE 1880 * @info: the struct contiaing AE information 1881 */ 1882 void 1883 irdma_terminate_received(struct irdma_sc_qp *qp, 1884 struct irdma_aeqe_info *info) 1885 { 1886 u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET; 1887 BE32 *mpa; 1888 u8 ddp_ctl; 1889 u8 rdma_ctl; 1890 u16 aeq_id = 0; 1891 struct irdma_terminate_hdr *termhdr; 1892 1893 mpa = (BE32 *) irdma_locate_mpa(pkt); 1894 if (info->q2_data_written) { 1895 /* did not validate the frame - do it now */ 1896 ddp_ctl = (ntohl(mpa[0]) >> 8) & 0xff; 1897 rdma_ctl = ntohl(mpa[0]) & 0xff; 1898 if ((ddp_ctl & 0xc0) != 0x40) 1899 aeq_id = IRDMA_AE_LCE_QP_CATASTROPHIC; 1900 else if ((ddp_ctl & 0x03) != 1) 1901 aeq_id = IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION; 1902 else if (ntohl(mpa[2]) != 2) 1903 aeq_id = IRDMA_AE_DDP_UBE_INVALID_QN; 1904 else if (ntohl(mpa[3]) != 1) 1905 aeq_id = IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN; 1906 else if (ntohl(mpa[4]) != 0) 1907 aeq_id = IRDMA_AE_DDP_UBE_INVALID_MO; 1908 else if ((rdma_ctl & 0xc0) != 0x40) 1909 aeq_id = IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION; 1910 1911 info->ae_id = aeq_id; 1912 if (info->ae_id) { 1913 /* Bad terminate recvd - send back a terminate */ 1914 irdma_terminate_connection(qp, info); 1915 return; 1916 } 1917 } 1918 1919 qp->term_flags |= IRDMA_TERM_RCVD; 1920 qp->event_type = IRDMA_QP_EVENT_CATASTROPHIC; 1921 termhdr = (struct irdma_terminate_hdr *)&mpa[5]; 1922 if (termhdr->layer_etype == RDMAP_REMOTE_PROT || 1923 termhdr->layer_etype == RDMAP_REMOTE_OP) { 1924 irdma_terminate_done(qp, 0); 1925 } else { 1926 irdma_terminate_start_timer(qp); 1927 irdma_terminate_send_fin(qp); 1928 } 1929 } 1930 1931 static int 1932 irdma_null_ws_add(struct irdma_sc_vsi *vsi, u8 user_pri) 1933 { 1934 return 0; 1935 } 1936 1937 static void 1938 irdma_null_ws_remove(struct irdma_sc_vsi *vsi, u8 user_pri) 1939 { 1940 /* do nothing */ 1941 } 1942 1943 static void 1944 irdma_null_ws_reset(struct irdma_sc_vsi *vsi) 1945 { 1946 /* do nothing */ 1947 } 1948 1949 /** 1950 * irdma_sc_vsi_init - Init the vsi structure 1951 * @vsi: pointer to vsi structure to initialize 1952 * @info: the info used to initialize the vsi struct 1953 */ 1954 void 1955 irdma_sc_vsi_init(struct irdma_sc_vsi *vsi, 1956 struct irdma_vsi_init_info *info) 1957 { 1958 u8 i; 1959 1960 vsi->dev = info->dev; 1961 vsi->back_vsi = info->back_vsi; 1962 vsi->register_qset = info->register_qset; 1963 vsi->unregister_qset = info->unregister_qset; 1964 vsi->mtu = info->params->mtu; 1965 vsi->exception_lan_q = info->exception_lan_q; 1966 vsi->vsi_idx = info->pf_data_vsi_num; 1967 1968 irdma_set_qos_info(vsi, info->params); 1969 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) { 1970 mutex_init(&vsi->qos[i].qos_mutex); 1971 INIT_LIST_HEAD(&vsi->qos[i].qplist); 1972 } 1973 if (vsi->register_qset) { 1974 vsi->dev->ws_add = irdma_ws_add; 1975 vsi->dev->ws_remove = irdma_ws_remove; 1976 vsi->dev->ws_reset = irdma_ws_reset; 1977 } else { 1978 vsi->dev->ws_add = irdma_null_ws_add; 1979 vsi->dev->ws_remove = irdma_null_ws_remove; 1980 vsi->dev->ws_reset = irdma_null_ws_reset; 1981 } 1982 } 1983 1984 /** 1985 * irdma_get_stats_idx - Return stats index 1986 * @vsi: pointer to the vsi 1987 */ 1988 static u16 irdma_get_stats_idx(struct irdma_sc_vsi *vsi){ 1989 struct irdma_stats_inst_info stats_info = {0}; 1990 struct irdma_sc_dev *dev = vsi->dev; 1991 1992 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1993 if (!irdma_cqp_stats_inst_cmd(vsi, IRDMA_OP_STATS_ALLOCATE, 1994 &stats_info)) 1995 return stats_info.stats_idx; 1996 } 1997 1998 return IRDMA_INVALID_STATS_IDX; 1999 } 2000 2001 /** 2002 * irdma_vsi_stats_init - Initialize the vsi statistics 2003 * @vsi: pointer to the vsi structure 2004 * @info: The info structure used for initialization 2005 */ 2006 int 2007 irdma_vsi_stats_init(struct irdma_sc_vsi *vsi, 2008 struct irdma_vsi_stats_info *info) 2009 { 2010 struct irdma_dma_mem *stats_buff_mem; 2011 2012 vsi->pestat = info->pestat; 2013 vsi->pestat->hw = vsi->dev->hw; 2014 vsi->pestat->vsi = vsi; 2015 2016 stats_buff_mem = &vsi->pestat->gather_info.stats_buff_mem; 2017 stats_buff_mem->size = IRDMA_GATHER_STATS_BUF_SIZE * 2; 2018 stats_buff_mem->va = irdma_allocate_dma_mem(vsi->pestat->hw, 2019 stats_buff_mem, 2020 stats_buff_mem->size, 1); 2021 if (!stats_buff_mem->va) 2022 return -ENOMEM; 2023 2024 vsi->pestat->gather_info.gather_stats_va = stats_buff_mem->va; 2025 vsi->pestat->gather_info.last_gather_stats_va = 2026 (void *)((uintptr_t)stats_buff_mem->va + 2027 IRDMA_GATHER_STATS_BUF_SIZE); 2028 2029 irdma_hw_stats_start_timer(vsi); 2030 2031 /* when stat allocation is not required default to fcn_id. */ 2032 vsi->stats_idx = info->fcn_id; 2033 if (info->alloc_stats_inst) { 2034 u16 stats_idx = irdma_get_stats_idx(vsi); 2035 2036 if (stats_idx != IRDMA_INVALID_STATS_IDX) { 2037 vsi->stats_inst_alloc = true; 2038 vsi->stats_idx = stats_idx; 2039 vsi->pestat->gather_info.use_stats_inst = true; 2040 vsi->pestat->gather_info.stats_inst_index = stats_idx; 2041 } 2042 } 2043 2044 return 0; 2045 } 2046 2047 /** 2048 * irdma_vsi_stats_free - Free the vsi stats 2049 * @vsi: pointer to the vsi structure 2050 */ 2051 void 2052 irdma_vsi_stats_free(struct irdma_sc_vsi *vsi) 2053 { 2054 struct irdma_stats_inst_info stats_info = {0}; 2055 struct irdma_sc_dev *dev = vsi->dev; 2056 2057 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 2058 if (vsi->stats_inst_alloc) { 2059 stats_info.stats_idx = vsi->stats_idx; 2060 irdma_cqp_stats_inst_cmd(vsi, IRDMA_OP_STATS_FREE, 2061 &stats_info); 2062 } 2063 } 2064 2065 if (!vsi->pestat) 2066 return; 2067 2068 irdma_hw_stats_stop_timer(vsi); 2069 irdma_free_dma_mem(vsi->pestat->hw, 2070 &vsi->pestat->gather_info.stats_buff_mem); 2071 } 2072 2073 /** 2074 * irdma_get_encoded_wqe_size - given wq size, returns hardware encoded size 2075 * @wqsize: size of the wq (sq, rq) to encoded_size 2076 * @queue_type: queue type selected for the calculation algorithm 2077 */ 2078 u8 2079 irdma_get_encoded_wqe_size(u32 wqsize, enum irdma_queue_type queue_type) 2080 { 2081 u8 encoded_size = 0; 2082 2083 /* 2084 * cqp sq's hw coded value starts from 1 for size of 4 while it starts from 0 for qp' wq's. 2085 */ 2086 if (queue_type == IRDMA_QUEUE_TYPE_CQP) 2087 encoded_size = 1; 2088 wqsize >>= 2; 2089 while (wqsize >>= 1) 2090 encoded_size++; 2091 2092 return encoded_size; 2093 } 2094 2095 /** 2096 * irdma_sc_gather_stats - collect the statistics 2097 * @cqp: struct for cqp hw 2098 * @info: gather stats info structure 2099 * @scratch: u64 saved to be used during cqp completion 2100 */ 2101 static int 2102 irdma_sc_gather_stats(struct irdma_sc_cqp *cqp, 2103 struct irdma_stats_gather_info *info, 2104 u64 scratch) 2105 { 2106 __le64 *wqe; 2107 u64 temp; 2108 2109 if (info->stats_buff_mem.size < IRDMA_GATHER_STATS_BUF_SIZE) 2110 return -ENOSPC; 2111 2112 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 2113 if (!wqe) 2114 return -ENOSPC; 2115 2116 set_64bit_val(wqe, IRDMA_BYTE_40, 2117 FIELD_PREP(IRDMA_CQPSQ_STATS_HMC_FCN_INDEX, info->hmc_fcn_index)); 2118 set_64bit_val(wqe, IRDMA_BYTE_32, info->stats_buff_mem.pa); 2119 2120 temp = FIELD_PREP(IRDMA_CQPSQ_STATS_WQEVALID, cqp->polarity) | 2121 FIELD_PREP(IRDMA_CQPSQ_STATS_USE_INST, info->use_stats_inst) | 2122 FIELD_PREP(IRDMA_CQPSQ_STATS_INST_INDEX, 2123 info->stats_inst_index) | 2124 FIELD_PREP(IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX, 2125 info->use_hmc_fcn_index) | 2126 FIELD_PREP(IRDMA_CQPSQ_STATS_OP, IRDMA_CQP_OP_GATHER_STATS); 2127 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 2128 2129 set_64bit_val(wqe, IRDMA_BYTE_24, temp); 2130 2131 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_STATS, "GATHER_STATS WQE", wqe, 2132 IRDMA_CQP_WQE_SIZE * 8); 2133 2134 irdma_sc_cqp_post_sq(cqp); 2135 return 0; 2136 } 2137 2138 /** 2139 * irdma_sc_manage_stats_inst - allocate or free stats instance 2140 * @cqp: struct for cqp hw 2141 * @info: stats info structure 2142 * @alloc: alloc vs. delete flag 2143 * @scratch: u64 saved to be used during cqp completion 2144 */ 2145 static int 2146 irdma_sc_manage_stats_inst(struct irdma_sc_cqp *cqp, 2147 struct irdma_stats_inst_info *info, 2148 bool alloc, u64 scratch) 2149 { 2150 __le64 *wqe; 2151 u64 temp; 2152 2153 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 2154 if (!wqe) 2155 return -ENOSPC; 2156 2157 set_64bit_val(wqe, IRDMA_BYTE_40, 2158 FIELD_PREP(IRDMA_CQPSQ_STATS_HMC_FCN_INDEX, info->hmc_fn_id)); 2159 temp = FIELD_PREP(IRDMA_CQPSQ_STATS_WQEVALID, cqp->polarity) | 2160 FIELD_PREP(IRDMA_CQPSQ_STATS_ALLOC_INST, alloc) | 2161 FIELD_PREP(IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX, 2162 info->use_hmc_fcn_index) | 2163 FIELD_PREP(IRDMA_CQPSQ_STATS_INST_INDEX, info->stats_idx) | 2164 FIELD_PREP(IRDMA_CQPSQ_STATS_OP, IRDMA_CQP_OP_MANAGE_STATS); 2165 2166 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 2167 2168 set_64bit_val(wqe, IRDMA_BYTE_24, temp); 2169 2170 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "MANAGE_STATS WQE", wqe, 2171 IRDMA_CQP_WQE_SIZE * 8); 2172 2173 irdma_sc_cqp_post_sq(cqp); 2174 return 0; 2175 } 2176 2177 /** 2178 * irdma_sc_set_up_map - set the up map table 2179 * @cqp: struct for cqp hw 2180 * @info: User priority map info 2181 * @scratch: u64 saved to be used during cqp completion 2182 */ 2183 static int 2184 irdma_sc_set_up_map(struct irdma_sc_cqp *cqp, 2185 struct irdma_up_info *info, u64 scratch) 2186 { 2187 __le64 *wqe; 2188 u64 temp = 0; 2189 int i; 2190 2191 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 2192 if (!wqe) 2193 return -ENOSPC; 2194 2195 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) 2196 temp |= (u64)info->map[i] << (i * 8); 2197 2198 set_64bit_val(wqe, IRDMA_BYTE_0, temp); 2199 set_64bit_val(wqe, IRDMA_BYTE_40, 2200 FIELD_PREP(IRDMA_CQPSQ_UP_CNPOVERRIDE, info->cnp_up_override) | 2201 FIELD_PREP(IRDMA_CQPSQ_UP_HMCFCNIDX, info->hmc_fcn_idx)); 2202 2203 temp = FIELD_PREP(IRDMA_CQPSQ_UP_WQEVALID, cqp->polarity) | 2204 FIELD_PREP(IRDMA_CQPSQ_UP_USEVLAN, info->use_vlan) | 2205 FIELD_PREP(IRDMA_CQPSQ_UP_USEOVERRIDE, 2206 info->use_cnp_up_override) | 2207 FIELD_PREP(IRDMA_CQPSQ_UP_OP, IRDMA_CQP_OP_UP_MAP); 2208 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 2209 2210 set_64bit_val(wqe, IRDMA_BYTE_24, temp); 2211 2212 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "UPMAP WQE", wqe, 2213 IRDMA_CQP_WQE_SIZE * 8); 2214 irdma_sc_cqp_post_sq(cqp); 2215 2216 return 0; 2217 } 2218 2219 /** 2220 * irdma_sc_manage_ws_node - create/modify/destroy WS node 2221 * @cqp: struct for cqp hw 2222 * @info: node info structure 2223 * @node_op: 0 for add 1 for modify, 2 for delete 2224 * @scratch: u64 saved to be used during cqp completion 2225 */ 2226 static int 2227 irdma_sc_manage_ws_node(struct irdma_sc_cqp *cqp, 2228 struct irdma_ws_node_info *info, 2229 enum irdma_ws_node_op node_op, u64 scratch) 2230 { 2231 __le64 *wqe; 2232 u64 temp = 0; 2233 2234 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 2235 if (!wqe) 2236 return -ENOSPC; 2237 2238 set_64bit_val(wqe, IRDMA_BYTE_32, 2239 FIELD_PREP(IRDMA_CQPSQ_WS_VSI, info->vsi) | 2240 FIELD_PREP(IRDMA_CQPSQ_WS_WEIGHT, info->weight)); 2241 2242 temp = FIELD_PREP(IRDMA_CQPSQ_WS_WQEVALID, cqp->polarity) | 2243 FIELD_PREP(IRDMA_CQPSQ_WS_NODEOP, node_op) | 2244 FIELD_PREP(IRDMA_CQPSQ_WS_ENABLENODE, info->enable) | 2245 FIELD_PREP(IRDMA_CQPSQ_WS_NODETYPE, info->type_leaf) | 2246 FIELD_PREP(IRDMA_CQPSQ_WS_PRIOTYPE, info->prio_type) | 2247 FIELD_PREP(IRDMA_CQPSQ_WS_TC, info->tc) | 2248 FIELD_PREP(IRDMA_CQPSQ_WS_OP, IRDMA_CQP_OP_WORK_SCHED_NODE) | 2249 FIELD_PREP(IRDMA_CQPSQ_WS_PARENTID, info->parent_id) | 2250 FIELD_PREP(IRDMA_CQPSQ_WS_NODEID, info->id); 2251 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 2252 2253 set_64bit_val(wqe, IRDMA_BYTE_24, temp); 2254 2255 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "MANAGE_WS WQE", wqe, 2256 IRDMA_CQP_WQE_SIZE * 8); 2257 irdma_sc_cqp_post_sq(cqp); 2258 2259 return 0; 2260 } 2261 2262 /** 2263 * irdma_sc_qp_flush_wqes - flush qp's wqe 2264 * @qp: sc qp 2265 * @info: dlush information 2266 * @scratch: u64 saved to be used during cqp completion 2267 * @post_sq: flag for cqp db to ring 2268 */ 2269 int 2270 irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp, 2271 struct irdma_qp_flush_info *info, u64 scratch, 2272 bool post_sq) 2273 { 2274 u64 temp = 0; 2275 __le64 *wqe; 2276 struct irdma_sc_cqp *cqp; 2277 u64 hdr; 2278 bool flush_sq = false, flush_rq = false; 2279 2280 if (info->rq && !qp->flush_rq) 2281 flush_rq = true; 2282 if (info->sq && !qp->flush_sq) 2283 flush_sq = true; 2284 qp->flush_sq |= flush_sq; 2285 qp->flush_rq |= flush_rq; 2286 2287 if (!flush_sq && !flush_rq) { 2288 irdma_debug(qp->dev, IRDMA_DEBUG_CQP, 2289 "Additional flush request ignored for qp %x\n", 2290 qp->qp_uk.qp_id); 2291 return -EALREADY; 2292 } 2293 2294 cqp = qp->pd->dev->cqp; 2295 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 2296 if (!wqe) 2297 return -ENOSPC; 2298 2299 if (info->userflushcode) { 2300 if (flush_rq) 2301 temp |= FIELD_PREP(IRDMA_CQPSQ_FWQE_RQMNERR, 2302 info->rq_minor_code) | 2303 FIELD_PREP(IRDMA_CQPSQ_FWQE_RQMJERR, 2304 info->rq_major_code); 2305 if (flush_sq) 2306 temp |= FIELD_PREP(IRDMA_CQPSQ_FWQE_SQMNERR, 2307 info->sq_minor_code) | 2308 FIELD_PREP(IRDMA_CQPSQ_FWQE_SQMJERR, 2309 info->sq_major_code); 2310 } 2311 set_64bit_val(wqe, IRDMA_BYTE_16, temp); 2312 2313 temp = (info->generate_ae) ? 2314 info->ae_code | FIELD_PREP(IRDMA_CQPSQ_FWQE_AESOURCE, 2315 info->ae_src) : 0; 2316 set_64bit_val(wqe, IRDMA_BYTE_8, temp); 2317 hdr = qp->qp_uk.qp_id | 2318 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_FLUSH_WQES) | 2319 FIELD_PREP(IRDMA_CQPSQ_FWQE_GENERATE_AE, info->generate_ae) | 2320 FIELD_PREP(IRDMA_CQPSQ_FWQE_USERFLCODE, info->userflushcode) | 2321 FIELD_PREP(IRDMA_CQPSQ_FWQE_FLUSHSQ, flush_sq) | 2322 FIELD_PREP(IRDMA_CQPSQ_FWQE_FLUSHRQ, flush_rq) | 2323 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 2324 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 2325 2326 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 2327 2328 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "QP_FLUSH WQE", wqe, 2329 IRDMA_CQP_WQE_SIZE * 8); 2330 if (post_sq) 2331 irdma_sc_cqp_post_sq(cqp); 2332 2333 return 0; 2334 } 2335 2336 /** 2337 * irdma_sc_gen_ae - generate AE, uses flush WQE CQP OP 2338 * @qp: sc qp 2339 * @info: gen ae information 2340 * @scratch: u64 saved to be used during cqp completion 2341 * @post_sq: flag for cqp db to ring 2342 */ 2343 static int 2344 irdma_sc_gen_ae(struct irdma_sc_qp *qp, 2345 struct irdma_gen_ae_info *info, u64 scratch, 2346 bool post_sq) 2347 { 2348 u64 temp; 2349 __le64 *wqe; 2350 struct irdma_sc_cqp *cqp; 2351 u64 hdr; 2352 2353 cqp = qp->pd->dev->cqp; 2354 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 2355 if (!wqe) 2356 return -ENOSPC; 2357 2358 temp = info->ae_code | FIELD_PREP(IRDMA_CQPSQ_FWQE_AESOURCE, 2359 info->ae_src); 2360 set_64bit_val(wqe, IRDMA_BYTE_8, temp); 2361 2362 hdr = qp->qp_uk.qp_id | FIELD_PREP(IRDMA_CQPSQ_OPCODE, 2363 IRDMA_CQP_OP_GEN_AE) | 2364 FIELD_PREP(IRDMA_CQPSQ_FWQE_GENERATE_AE, 1) | 2365 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 2366 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 2367 2368 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 2369 2370 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "GEN_AE WQE", wqe, 2371 IRDMA_CQP_WQE_SIZE * 8); 2372 if (post_sq) 2373 irdma_sc_cqp_post_sq(cqp); 2374 2375 return 0; 2376 } 2377 2378 /*** irdma_sc_qp_upload_context - upload qp's context 2379 * @dev: sc device struct 2380 * @info: upload context info ptr for return 2381 * @scratch: u64 saved to be used during cqp completion 2382 * @post_sq: flag for cqp db to ring 2383 */ 2384 static int 2385 irdma_sc_qp_upload_context(struct irdma_sc_dev *dev, 2386 struct irdma_upload_context_info *info, 2387 u64 scratch, bool post_sq) 2388 { 2389 __le64 *wqe; 2390 struct irdma_sc_cqp *cqp; 2391 u64 hdr; 2392 2393 cqp = dev->cqp; 2394 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 2395 if (!wqe) 2396 return -ENOSPC; 2397 2398 set_64bit_val(wqe, IRDMA_BYTE_16, info->buf_pa); 2399 2400 hdr = FIELD_PREP(IRDMA_CQPSQ_UCTX_QPID, info->qp_id) | 2401 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_UPLOAD_CONTEXT) | 2402 FIELD_PREP(IRDMA_CQPSQ_UCTX_QPTYPE, info->qp_type) | 2403 FIELD_PREP(IRDMA_CQPSQ_UCTX_RAWFORMAT, info->raw_format) | 2404 FIELD_PREP(IRDMA_CQPSQ_UCTX_FREEZEQP, info->freeze_qp) | 2405 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 2406 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 2407 2408 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 2409 2410 irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "QP_UPLOAD_CTX WQE", wqe, 2411 IRDMA_CQP_WQE_SIZE * 8); 2412 if (post_sq) 2413 irdma_sc_cqp_post_sq(cqp); 2414 2415 return 0; 2416 } 2417 2418 /** 2419 * irdma_sc_manage_push_page - Handle push page 2420 * @cqp: struct for cqp hw 2421 * @info: push page info 2422 * @scratch: u64 saved to be used during cqp completion 2423 * @post_sq: flag for cqp db to ring 2424 */ 2425 static int 2426 irdma_sc_manage_push_page(struct irdma_sc_cqp *cqp, 2427 struct irdma_cqp_manage_push_page_info *info, 2428 u64 scratch, bool post_sq) 2429 { 2430 __le64 *wqe; 2431 u64 hdr; 2432 2433 if (info->free_page && 2434 info->push_idx >= cqp->dev->hw_attrs.max_hw_device_pages) 2435 return -EINVAL; 2436 2437 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 2438 if (!wqe) 2439 return -ENOSPC; 2440 2441 set_64bit_val(wqe, IRDMA_BYTE_16, info->qs_handle); 2442 hdr = FIELD_PREP(IRDMA_CQPSQ_MPP_PPIDX, info->push_idx) | 2443 FIELD_PREP(IRDMA_CQPSQ_MPP_PPTYPE, info->push_page_type) | 2444 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_PUSH_PAGES) | 2445 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) | 2446 FIELD_PREP(IRDMA_CQPSQ_MPP_FREE_PAGE, info->free_page); 2447 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 2448 2449 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 2450 2451 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "MANAGE_PUSH_PAGES WQE", wqe, 2452 IRDMA_CQP_WQE_SIZE * 8); 2453 if (post_sq) 2454 irdma_sc_cqp_post_sq(cqp); 2455 2456 return 0; 2457 } 2458 2459 /** 2460 * irdma_sc_suspend_qp - suspend qp for param change 2461 * @cqp: struct for cqp hw 2462 * @qp: sc qp struct 2463 * @scratch: u64 saved to be used during cqp completion 2464 */ 2465 static int 2466 irdma_sc_suspend_qp(struct irdma_sc_cqp *cqp, struct irdma_sc_qp *qp, 2467 u64 scratch) 2468 { 2469 u64 hdr; 2470 __le64 *wqe; 2471 2472 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 2473 if (!wqe) 2474 return -ENOSPC; 2475 2476 hdr = FIELD_PREP(IRDMA_CQPSQ_SUSPENDQP_QPID, qp->qp_uk.qp_id) | 2477 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_SUSPEND_QP) | 2478 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 2479 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 2480 2481 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 2482 2483 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "SUSPEND_QP WQE", wqe, 2484 IRDMA_CQP_WQE_SIZE * 8); 2485 irdma_sc_cqp_post_sq(cqp); 2486 2487 return 0; 2488 } 2489 2490 /** 2491 * irdma_sc_resume_qp - resume qp after suspend 2492 * @cqp: struct for cqp hw 2493 * @qp: sc qp struct 2494 * @scratch: u64 saved to be used during cqp completion 2495 */ 2496 static int 2497 irdma_sc_resume_qp(struct irdma_sc_cqp *cqp, struct irdma_sc_qp *qp, 2498 u64 scratch) 2499 { 2500 u64 hdr; 2501 __le64 *wqe; 2502 2503 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 2504 if (!wqe) 2505 return -ENOSPC; 2506 2507 set_64bit_val(wqe, IRDMA_BYTE_16, 2508 FIELD_PREP(IRDMA_CQPSQ_RESUMEQP_QSHANDLE, qp->qs_handle)); 2509 2510 hdr = FIELD_PREP(IRDMA_CQPSQ_RESUMEQP_QPID, qp->qp_uk.qp_id) | 2511 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_RESUME_QP) | 2512 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 2513 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 2514 2515 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 2516 2517 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "RESUME_QP WQE", wqe, 2518 IRDMA_CQP_WQE_SIZE * 8); 2519 irdma_sc_cqp_post_sq(cqp); 2520 2521 return 0; 2522 } 2523 2524 /** 2525 * irdma_sc_cq_ack - acknowledge completion q 2526 * @cq: cq struct 2527 */ 2528 static inline void 2529 irdma_sc_cq_ack(struct irdma_sc_cq *cq) 2530 { 2531 db_wr32(cq->cq_uk.cq_id, cq->cq_uk.cq_ack_db); 2532 } 2533 2534 /** 2535 * irdma_sc_cq_init - initialize completion q 2536 * @cq: cq struct 2537 * @info: cq initialization info 2538 */ 2539 int 2540 irdma_sc_cq_init(struct irdma_sc_cq *cq, struct irdma_cq_init_info *info) 2541 { 2542 int ret_code; 2543 u32 pble_obj_cnt; 2544 2545 pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; 2546 if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt) 2547 return -EINVAL; 2548 2549 cq->cq_pa = info->cq_base_pa; 2550 cq->dev = info->dev; 2551 cq->ceq_id = info->ceq_id; 2552 info->cq_uk_init_info.cqe_alloc_db = cq->dev->cq_arm_db; 2553 info->cq_uk_init_info.cq_ack_db = cq->dev->cq_ack_db; 2554 ret_code = irdma_uk_cq_init(&cq->cq_uk, &info->cq_uk_init_info); 2555 if (ret_code) 2556 return ret_code; 2557 2558 cq->virtual_map = info->virtual_map; 2559 cq->pbl_chunk_size = info->pbl_chunk_size; 2560 cq->ceqe_mask = info->ceqe_mask; 2561 cq->cq_type = (info->type) ? info->type : IRDMA_CQ_TYPE_IWARP; 2562 cq->shadow_area_pa = info->shadow_area_pa; 2563 cq->shadow_read_threshold = info->shadow_read_threshold; 2564 cq->ceq_id_valid = info->ceq_id_valid; 2565 cq->tph_en = info->tph_en; 2566 cq->tph_val = info->tph_val; 2567 cq->first_pm_pbl_idx = info->first_pm_pbl_idx; 2568 cq->vsi = info->vsi; 2569 2570 return 0; 2571 } 2572 2573 /** 2574 * irdma_sc_cq_create - create completion q 2575 * @cq: cq struct 2576 * @scratch: u64 saved to be used during cqp completion 2577 * @check_overflow: flag for overflow check 2578 * @post_sq: flag for cqp db to ring 2579 */ 2580 static int 2581 irdma_sc_cq_create(struct irdma_sc_cq *cq, u64 scratch, 2582 bool check_overflow, bool post_sq) 2583 { 2584 __le64 *wqe; 2585 struct irdma_sc_cqp *cqp; 2586 u64 hdr; 2587 struct irdma_sc_ceq *ceq; 2588 int ret_code = 0; 2589 2590 cqp = cq->dev->cqp; 2591 if (cq->cq_uk.cq_id > (cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].max_cnt - 1)) 2592 return -EINVAL; 2593 2594 if (cq->ceq_id > (cq->dev->hmc_fpm_misc.max_ceqs - 1)) 2595 return -EINVAL; 2596 2597 ceq = cq->dev->ceq[cq->ceq_id]; 2598 if (ceq && ceq->reg_cq) { 2599 ret_code = irdma_sc_add_cq_ctx(ceq, cq); 2600 if (ret_code) 2601 return ret_code; 2602 } 2603 2604 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 2605 if (!wqe) { 2606 if (ceq && ceq->reg_cq) 2607 irdma_sc_remove_cq_ctx(ceq, cq); 2608 return -ENOSPC; 2609 } 2610 2611 set_64bit_val(wqe, IRDMA_BYTE_0, cq->cq_uk.cq_size); 2612 set_64bit_val(wqe, IRDMA_BYTE_8, RS_64_1(cq, 1)); 2613 set_64bit_val(wqe, IRDMA_BYTE_16, 2614 FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, 2615 cq->shadow_read_threshold)); 2616 set_64bit_val(wqe, IRDMA_BYTE_32, cq->virtual_map ? 0 : cq->cq_pa); 2617 set_64bit_val(wqe, IRDMA_BYTE_40, cq->shadow_area_pa); 2618 set_64bit_val(wqe, IRDMA_BYTE_48, 2619 FIELD_PREP(IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX, 2620 cq->virtual_map ? cq->first_pm_pbl_idx : 0)); 2621 set_64bit_val(wqe, IRDMA_BYTE_56, 2622 FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq->tph_val) | 2623 FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq->vsi->vsi_idx)); 2624 hdr = FLD_LS_64(cq->dev, cq->cq_uk.cq_id, IRDMA_CQPSQ_CQ_CQID) | 2625 FLD_LS_64(cq->dev, cq->ceq_id_valid ? cq->ceq_id : 0, 2626 IRDMA_CQPSQ_CQ_CEQID) | 2627 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_CQ) | 2628 FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, cq->pbl_chunk_size) | 2629 FIELD_PREP(IRDMA_CQPSQ_CQ_CHKOVERFLOW, check_overflow) | 2630 FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, cq->virtual_map) | 2631 FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) | 2632 FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, cq->ceq_id_valid) | 2633 FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) | 2634 FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT, 2635 cq->cq_uk.avoid_mem_cflct) | 2636 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 2637 2638 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 2639 2640 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 2641 2642 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "CQ_CREATE WQE", wqe, 2643 IRDMA_CQP_WQE_SIZE * 8); 2644 if (post_sq) 2645 irdma_sc_cqp_post_sq(cqp); 2646 2647 return 0; 2648 } 2649 2650 /** 2651 * irdma_sc_cq_destroy - destroy completion q 2652 * @cq: cq struct 2653 * @scratch: u64 saved to be used during cqp completion 2654 * @post_sq: flag for cqp db to ring 2655 */ 2656 int 2657 irdma_sc_cq_destroy(struct irdma_sc_cq *cq, u64 scratch, bool post_sq) 2658 { 2659 struct irdma_sc_cqp *cqp; 2660 __le64 *wqe; 2661 u64 hdr; 2662 struct irdma_sc_ceq *ceq; 2663 2664 cqp = cq->dev->cqp; 2665 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 2666 if (!wqe) 2667 return -ENOSPC; 2668 2669 ceq = cq->dev->ceq[cq->ceq_id]; 2670 if (ceq && ceq->reg_cq) 2671 irdma_sc_remove_cq_ctx(ceq, cq); 2672 2673 set_64bit_val(wqe, IRDMA_BYTE_0, cq->cq_uk.cq_size); 2674 set_64bit_val(wqe, IRDMA_BYTE_8, RS_64_1(cq, 1)); 2675 set_64bit_val(wqe, IRDMA_BYTE_40, cq->shadow_area_pa); 2676 set_64bit_val(wqe, IRDMA_BYTE_48, 2677 (cq->virtual_map ? cq->first_pm_pbl_idx : 0)); 2678 2679 hdr = cq->cq_uk.cq_id | 2680 FLD_LS_64(cq->dev, (cq->ceq_id_valid ? cq->ceq_id : 0), 2681 IRDMA_CQPSQ_CQ_CEQID) | 2682 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_CQ) | 2683 FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, cq->pbl_chunk_size) | 2684 FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, cq->virtual_map) | 2685 FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) | 2686 FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, cq->ceq_id_valid) | 2687 FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) | 2688 FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT, cq->cq_uk.avoid_mem_cflct) | 2689 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 2690 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 2691 2692 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 2693 2694 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "CQ_DESTROY WQE", wqe, 2695 IRDMA_CQP_WQE_SIZE * 8); 2696 if (post_sq) 2697 irdma_sc_cqp_post_sq(cqp); 2698 2699 return 0; 2700 } 2701 2702 /** 2703 * irdma_sc_cq_resize - set resized cq buffer info 2704 * @cq: resized cq 2705 * @info: resized cq buffer info 2706 */ 2707 void 2708 irdma_sc_cq_resize(struct irdma_sc_cq *cq, struct irdma_modify_cq_info *info) 2709 { 2710 cq->virtual_map = info->virtual_map; 2711 cq->cq_pa = info->cq_pa; 2712 cq->first_pm_pbl_idx = info->first_pm_pbl_idx; 2713 cq->pbl_chunk_size = info->pbl_chunk_size; 2714 irdma_uk_cq_resize(&cq->cq_uk, info->cq_base, info->cq_size); 2715 } 2716 2717 /** 2718 * irdma_sc_cq_modify - modify a Completion Queue 2719 * @cq: cq struct 2720 * @info: modification info struct 2721 * @scratch: u64 saved to be used during cqp completion 2722 * @post_sq: flag to post to sq 2723 */ 2724 static int 2725 irdma_sc_cq_modify(struct irdma_sc_cq *cq, 2726 struct irdma_modify_cq_info *info, u64 scratch, 2727 bool post_sq) 2728 { 2729 struct irdma_sc_cqp *cqp; 2730 __le64 *wqe; 2731 u64 hdr; 2732 u32 pble_obj_cnt; 2733 2734 pble_obj_cnt = cq->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; 2735 if (info->cq_resize && info->virtual_map && 2736 info->first_pm_pbl_idx >= pble_obj_cnt) 2737 return -EINVAL; 2738 2739 cqp = cq->dev->cqp; 2740 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 2741 if (!wqe) 2742 return -ENOSPC; 2743 2744 set_64bit_val(wqe, IRDMA_BYTE_0, info->cq_size); 2745 set_64bit_val(wqe, IRDMA_BYTE_8, RS_64_1(cq, 1)); 2746 set_64bit_val(wqe, IRDMA_BYTE_16, 2747 FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, info->shadow_read_threshold)); 2748 set_64bit_val(wqe, IRDMA_BYTE_32, info->cq_pa); 2749 set_64bit_val(wqe, IRDMA_BYTE_40, cq->shadow_area_pa); 2750 set_64bit_val(wqe, IRDMA_BYTE_48, info->first_pm_pbl_idx); 2751 set_64bit_val(wqe, IRDMA_BYTE_56, 2752 FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq->tph_val) | 2753 FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq->vsi->vsi_idx)); 2754 2755 hdr = cq->cq_uk.cq_id | 2756 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MODIFY_CQ) | 2757 FIELD_PREP(IRDMA_CQPSQ_CQ_CQRESIZE, info->cq_resize) | 2758 FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, info->pbl_chunk_size) | 2759 FIELD_PREP(IRDMA_CQPSQ_CQ_CHKOVERFLOW, info->check_overflow) | 2760 FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, info->virtual_map) | 2761 FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) | 2762 FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) | 2763 FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT, 2764 cq->cq_uk.avoid_mem_cflct) | 2765 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 2766 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 2767 2768 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 2769 2770 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "CQ_MODIFY WQE", wqe, 2771 IRDMA_CQP_WQE_SIZE * 8); 2772 if (post_sq) 2773 irdma_sc_cqp_post_sq(cqp); 2774 2775 return 0; 2776 } 2777 2778 /** 2779 * irdma_check_cqp_progress - check cqp processing progress 2780 * @timeout: timeout info struct 2781 * @dev: sc device struct 2782 */ 2783 void 2784 irdma_check_cqp_progress(struct irdma_cqp_timeout *timeout, 2785 struct irdma_sc_dev *dev) 2786 { 2787 u64 completed_ops = atomic64_read(&dev->cqp->completed_ops); 2788 2789 if (timeout->compl_cqp_cmds != completed_ops) { 2790 timeout->compl_cqp_cmds = completed_ops; 2791 timeout->count = 0; 2792 } else if (timeout->compl_cqp_cmds != dev->cqp->requested_ops) { 2793 timeout->count++; 2794 } 2795 } 2796 2797 /** 2798 * irdma_get_cqp_reg_info - get head and tail for cqp using registers 2799 * @cqp: struct for cqp hw 2800 * @val: cqp tail register value 2801 * @tail: wqtail register value 2802 * @error: cqp processing err 2803 */ 2804 static inline void 2805 irdma_get_cqp_reg_info(struct irdma_sc_cqp *cqp, u32 *val, 2806 u32 *tail, u32 *error) 2807 { 2808 *val = readl(cqp->dev->hw_regs[IRDMA_CQPTAIL]); 2809 *tail = FIELD_GET(IRDMA_CQPTAIL_WQTAIL, *val); 2810 *error = FIELD_GET(IRDMA_CQPTAIL_CQP_OP_ERR, *val); 2811 } 2812 2813 /** 2814 * irdma_cqp_poll_registers - poll cqp registers 2815 * @cqp: struct for cqp hw 2816 * @tail: wqtail register value 2817 * @count: how many times to try for completion 2818 */ 2819 static int 2820 irdma_cqp_poll_registers(struct irdma_sc_cqp *cqp, u32 tail, 2821 u32 count) 2822 { 2823 u32 i = 0; 2824 u32 newtail, error, val; 2825 2826 while (i++ < count) { 2827 irdma_get_cqp_reg_info(cqp, &val, &newtail, &error); 2828 if (error) { 2829 error = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]); 2830 irdma_debug(cqp->dev, IRDMA_DEBUG_CQP, 2831 "CQPERRCODES error_code[x%08X]\n", error); 2832 return -EIO; 2833 } 2834 if (newtail != tail) { 2835 /* SUCCESS */ 2836 IRDMA_RING_MOVE_TAIL(cqp->sq_ring); 2837 atomic64_inc(&cqp->completed_ops); 2838 return 0; 2839 } 2840 irdma_usec_delay(cqp->dev->hw_attrs.max_sleep_count); 2841 } 2842 2843 return -ETIMEDOUT; 2844 } 2845 2846 /** 2847 * irdma_sc_decode_fpm_commit - decode a 64 bit value into count and base 2848 * @dev: sc device struct 2849 * @buf: pointer to commit buffer 2850 * @buf_idx: buffer index 2851 * @obj_info: object info pointer 2852 * @rsrc_idx: indexs of memory resource 2853 */ 2854 static u64 irdma_sc_decode_fpm_commit(struct irdma_sc_dev *dev, __le64 * buf, 2855 u32 buf_idx, struct irdma_hmc_obj_info *obj_info, 2856 u32 rsrc_idx){ 2857 u64 temp; 2858 2859 get_64bit_val(buf, buf_idx, &temp); 2860 2861 switch (rsrc_idx) { 2862 case IRDMA_HMC_IW_QP: 2863 obj_info[rsrc_idx].cnt = (u32)FIELD_GET(IRDMA_COMMIT_FPM_QPCNT, temp); 2864 break; 2865 case IRDMA_HMC_IW_CQ: 2866 obj_info[rsrc_idx].cnt = (u32)FLD_RS_64(dev, temp, IRDMA_COMMIT_FPM_CQCNT); 2867 break; 2868 case IRDMA_HMC_IW_APBVT_ENTRY: 2869 if (dev->hw_attrs.uk_attrs.hw_rev <= IRDMA_GEN_2) 2870 obj_info[rsrc_idx].cnt = 1; 2871 else 2872 obj_info[rsrc_idx].cnt = 0; 2873 break; 2874 default: 2875 obj_info[rsrc_idx].cnt = (u32)temp; 2876 break; 2877 } 2878 2879 obj_info[rsrc_idx].base = (u64)RS_64_1(temp, IRDMA_COMMIT_FPM_BASE_S) * 512; 2880 2881 return temp; 2882 } 2883 2884 /** 2885 * irdma_sc_parse_fpm_commit_buf - parse fpm commit buffer 2886 * @dev: pointer to dev struct 2887 * @buf: ptr to fpm commit buffer 2888 * @info: ptr to irdma_hmc_obj_info struct 2889 * @sd: number of SDs for HMC objects 2890 * 2891 * parses fpm commit info and copy base value 2892 * of hmc objects in hmc_info 2893 */ 2894 static void 2895 irdma_sc_parse_fpm_commit_buf(struct irdma_sc_dev *dev, __le64 * buf, 2896 struct irdma_hmc_obj_info *info, 2897 u32 *sd) 2898 { 2899 u64 size; 2900 u32 i; 2901 u64 max_base = 0; 2902 u32 last_hmc_obj = 0; 2903 2904 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_0, info, 2905 IRDMA_HMC_IW_QP); 2906 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_8, info, 2907 IRDMA_HMC_IW_CQ); 2908 /* skiping RSRVD */ 2909 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_24, info, 2910 IRDMA_HMC_IW_HTE); 2911 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_32, info, 2912 IRDMA_HMC_IW_ARP); 2913 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_40, info, 2914 IRDMA_HMC_IW_APBVT_ENTRY); 2915 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_48, info, 2916 IRDMA_HMC_IW_MR); 2917 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_56, info, 2918 IRDMA_HMC_IW_XF); 2919 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_64, info, 2920 IRDMA_HMC_IW_XFFL); 2921 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_72, info, 2922 IRDMA_HMC_IW_Q1); 2923 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_80, info, 2924 IRDMA_HMC_IW_Q1FL); 2925 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_88, info, 2926 IRDMA_HMC_IW_TIMER); 2927 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_112, info, 2928 IRDMA_HMC_IW_PBLE); 2929 /* skipping RSVD. */ 2930 if (dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1) { 2931 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_96, info, 2932 IRDMA_HMC_IW_FSIMC); 2933 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_104, info, 2934 IRDMA_HMC_IW_FSIAV); 2935 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_128, info, 2936 IRDMA_HMC_IW_RRF); 2937 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_136, info, 2938 IRDMA_HMC_IW_RRFFL); 2939 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_144, info, 2940 IRDMA_HMC_IW_HDR); 2941 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_152, info, 2942 IRDMA_HMC_IW_MD); 2943 if (dev->cqp->protocol_used == IRDMA_IWARP_PROTOCOL_ONLY) { 2944 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_160, info, 2945 IRDMA_HMC_IW_OOISC); 2946 irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_168, info, 2947 IRDMA_HMC_IW_OOISCFFL); 2948 } 2949 } 2950 2951 /* searching for the last object in HMC to find the size of the HMC area. */ 2952 for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++) { 2953 if (info[i].base > max_base) { 2954 max_base = info[i].base; 2955 last_hmc_obj = i; 2956 } 2957 } 2958 2959 size = info[last_hmc_obj].cnt * info[last_hmc_obj].size + 2960 info[last_hmc_obj].base; 2961 2962 if (size & 0x1FFFFF) 2963 *sd = (u32)((size >> 21) + 1); /* add 1 for remainder */ 2964 else 2965 *sd = (u32)(size >> 21); 2966 2967 } 2968 2969 /** 2970 * irdma_sc_decode_fpm_query() - Decode a 64 bit value into max count and size 2971 * @buf: ptr to fpm query buffer 2972 * @buf_idx: index into buf 2973 * @obj_info: ptr to irdma_hmc_obj_info struct 2974 * @rsrc_idx: resource index into info 2975 * 2976 * Decode a 64 bit value from fpm query buffer into max count and size 2977 */ 2978 static u64 irdma_sc_decode_fpm_query(__le64 * buf, u32 buf_idx, 2979 struct irdma_hmc_obj_info *obj_info, 2980 u32 rsrc_idx){ 2981 u64 temp; 2982 u32 size; 2983 2984 get_64bit_val(buf, buf_idx, &temp); 2985 obj_info[rsrc_idx].max_cnt = (u32)temp; 2986 size = (u32)RS_64_1(temp, 32); 2987 obj_info[rsrc_idx].size = LS_64_1(1, size); 2988 2989 return temp; 2990 } 2991 2992 /** 2993 * irdma_sc_parse_fpm_query_buf() - parses fpm query buffer 2994 * @dev: ptr to shared code device 2995 * @buf: ptr to fpm query buffer 2996 * @hmc_info: ptr to irdma_hmc_obj_info struct 2997 * @hmc_fpm_misc: ptr to fpm data 2998 * 2999 * parses fpm query buffer and copy max_cnt and 3000 * size value of hmc objects in hmc_info 3001 */ 3002 static int 3003 irdma_sc_parse_fpm_query_buf(struct irdma_sc_dev *dev, __le64 * buf, 3004 struct irdma_hmc_info *hmc_info, 3005 struct irdma_hmc_fpm_misc *hmc_fpm_misc) 3006 { 3007 struct irdma_hmc_obj_info *obj_info; 3008 u64 temp; 3009 u32 size; 3010 u16 max_pe_sds; 3011 3012 obj_info = hmc_info->hmc_obj; 3013 3014 get_64bit_val(buf, IRDMA_BYTE_0, &temp); 3015 hmc_info->first_sd_index = (u16)FIELD_GET(IRDMA_QUERY_FPM_FIRST_PE_SD_INDEX, temp); 3016 max_pe_sds = (u16)FIELD_GET(IRDMA_QUERY_FPM_MAX_PE_SDS, temp); 3017 3018 hmc_fpm_misc->max_sds = max_pe_sds; 3019 hmc_info->sd_table.sd_cnt = max_pe_sds + hmc_info->first_sd_index; 3020 get_64bit_val(buf, 8, &temp); 3021 obj_info[IRDMA_HMC_IW_QP].max_cnt = (u32)FIELD_GET(IRDMA_QUERY_FPM_MAX_QPS, temp); 3022 size = (u32)RS_64_1(temp, 32); 3023 obj_info[IRDMA_HMC_IW_QP].size = LS_64_1(1, size); 3024 3025 get_64bit_val(buf, 16, &temp); 3026 obj_info[IRDMA_HMC_IW_CQ].max_cnt = (u32)FIELD_GET(IRDMA_QUERY_FPM_MAX_CQS, temp); 3027 size = (u32)RS_64_1(temp, 32); 3028 obj_info[IRDMA_HMC_IW_CQ].size = LS_64_1(1, size); 3029 3030 irdma_sc_decode_fpm_query(buf, 32, obj_info, IRDMA_HMC_IW_HTE); 3031 irdma_sc_decode_fpm_query(buf, 40, obj_info, IRDMA_HMC_IW_ARP); 3032 3033 obj_info[IRDMA_HMC_IW_APBVT_ENTRY].size = 8192; 3034 obj_info[IRDMA_HMC_IW_APBVT_ENTRY].max_cnt = 1; 3035 3036 irdma_sc_decode_fpm_query(buf, 48, obj_info, IRDMA_HMC_IW_MR); 3037 irdma_sc_decode_fpm_query(buf, 56, obj_info, IRDMA_HMC_IW_XF); 3038 3039 get_64bit_val(buf, 64, &temp); 3040 obj_info[IRDMA_HMC_IW_XFFL].max_cnt = (u32)temp; 3041 obj_info[IRDMA_HMC_IW_XFFL].size = 4; 3042 hmc_fpm_misc->xf_block_size = FIELD_GET(IRDMA_QUERY_FPM_XFBLOCKSIZE, temp); 3043 if (!hmc_fpm_misc->xf_block_size) 3044 return -EINVAL; 3045 3046 irdma_sc_decode_fpm_query(buf, 72, obj_info, IRDMA_HMC_IW_Q1); 3047 get_64bit_val(buf, 80, &temp); 3048 obj_info[IRDMA_HMC_IW_Q1FL].max_cnt = (u32)temp; 3049 obj_info[IRDMA_HMC_IW_Q1FL].size = 4; 3050 3051 hmc_fpm_misc->q1_block_size = FIELD_GET(IRDMA_QUERY_FPM_Q1BLOCKSIZE, temp); 3052 if (!hmc_fpm_misc->q1_block_size) 3053 return -EINVAL; 3054 3055 irdma_sc_decode_fpm_query(buf, 88, obj_info, IRDMA_HMC_IW_TIMER); 3056 3057 get_64bit_val(buf, 112, &temp); 3058 obj_info[IRDMA_HMC_IW_PBLE].max_cnt = (u32)temp; 3059 obj_info[IRDMA_HMC_IW_PBLE].size = 8; 3060 3061 get_64bit_val(buf, 120, &temp); 3062 hmc_fpm_misc->max_ceqs = FIELD_GET(IRDMA_QUERY_FPM_MAX_CEQS, temp); 3063 hmc_fpm_misc->ht_multiplier = FIELD_GET(IRDMA_QUERY_FPM_HTMULTIPLIER, temp); 3064 hmc_fpm_misc->timer_bucket = FIELD_GET(IRDMA_QUERY_FPM_TIMERBUCKET, temp); 3065 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 3066 return 0; 3067 irdma_sc_decode_fpm_query(buf, 96, obj_info, IRDMA_HMC_IW_FSIMC); 3068 irdma_sc_decode_fpm_query(buf, 104, obj_info, IRDMA_HMC_IW_FSIAV); 3069 irdma_sc_decode_fpm_query(buf, 128, obj_info, IRDMA_HMC_IW_RRF); 3070 3071 get_64bit_val(buf, IRDMA_BYTE_136, &temp); 3072 obj_info[IRDMA_HMC_IW_RRFFL].max_cnt = (u32)temp; 3073 obj_info[IRDMA_HMC_IW_RRFFL].size = 4; 3074 hmc_fpm_misc->rrf_block_size = FIELD_GET(IRDMA_QUERY_FPM_RRFBLOCKSIZE, temp); 3075 if (!hmc_fpm_misc->rrf_block_size && 3076 obj_info[IRDMA_HMC_IW_RRFFL].max_cnt) 3077 return -EINVAL; 3078 3079 irdma_sc_decode_fpm_query(buf, 144, obj_info, IRDMA_HMC_IW_HDR); 3080 irdma_sc_decode_fpm_query(buf, 152, obj_info, IRDMA_HMC_IW_MD); 3081 3082 if (dev->cqp->protocol_used == IRDMA_IWARP_PROTOCOL_ONLY) { 3083 irdma_sc_decode_fpm_query(buf, 160, obj_info, IRDMA_HMC_IW_OOISC); 3084 3085 get_64bit_val(buf, IRDMA_BYTE_168, &temp); 3086 obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt = (u32)temp; 3087 obj_info[IRDMA_HMC_IW_OOISCFFL].size = 4; 3088 hmc_fpm_misc->ooiscf_block_size = FIELD_GET(IRDMA_QUERY_FPM_OOISCFBLOCKSIZE, temp); 3089 if (!hmc_fpm_misc->ooiscf_block_size && 3090 obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt) 3091 return -EINVAL; 3092 } 3093 3094 return 0; 3095 } 3096 3097 /** 3098 * irdma_sc_find_reg_cq - find cq ctx index 3099 * @ceq: ceq sc structure 3100 * @cq: cq sc structure 3101 */ 3102 static u32 irdma_sc_find_reg_cq(struct irdma_sc_ceq *ceq, 3103 struct irdma_sc_cq *cq){ 3104 u32 i; 3105 3106 for (i = 0; i < ceq->reg_cq_size; i++) { 3107 if (cq == ceq->reg_cq[i]) 3108 return i; 3109 } 3110 3111 return IRDMA_INVALID_CQ_IDX; 3112 } 3113 3114 /** 3115 * irdma_sc_add_cq_ctx - add cq ctx tracking for ceq 3116 * @ceq: ceq sc structure 3117 * @cq: cq sc structure 3118 */ 3119 int 3120 irdma_sc_add_cq_ctx(struct irdma_sc_ceq *ceq, struct irdma_sc_cq *cq) 3121 { 3122 unsigned long flags; 3123 3124 spin_lock_irqsave(&ceq->req_cq_lock, flags); 3125 3126 if (ceq->reg_cq_size == ceq->elem_cnt) { 3127 spin_unlock_irqrestore(&ceq->req_cq_lock, flags); 3128 return -ENOSPC; 3129 } 3130 3131 ceq->reg_cq[ceq->reg_cq_size++] = cq; 3132 3133 spin_unlock_irqrestore(&ceq->req_cq_lock, flags); 3134 3135 return 0; 3136 } 3137 3138 /** 3139 * irdma_sc_remove_cq_ctx - remove cq ctx tracking for ceq 3140 * @ceq: ceq sc structure 3141 * @cq: cq sc structure 3142 */ 3143 void 3144 irdma_sc_remove_cq_ctx(struct irdma_sc_ceq *ceq, struct irdma_sc_cq *cq) 3145 { 3146 unsigned long flags; 3147 u32 cq_ctx_idx; 3148 3149 spin_lock_irqsave(&ceq->req_cq_lock, flags); 3150 cq_ctx_idx = irdma_sc_find_reg_cq(ceq, cq); 3151 if (cq_ctx_idx == IRDMA_INVALID_CQ_IDX) 3152 goto exit; 3153 3154 ceq->reg_cq_size--; 3155 if (cq_ctx_idx != ceq->reg_cq_size) 3156 ceq->reg_cq[cq_ctx_idx] = ceq->reg_cq[ceq->reg_cq_size]; 3157 ceq->reg_cq[ceq->reg_cq_size] = NULL; 3158 3159 exit: 3160 spin_unlock_irqrestore(&ceq->req_cq_lock, flags); 3161 } 3162 3163 /** 3164 * irdma_sc_cqp_init - Initialize buffers for a control Queue Pair 3165 * @cqp: IWARP control queue pair pointer 3166 * @info: IWARP control queue pair init info pointer 3167 * 3168 * Initializes the object and context buffers for a control Queue Pair. 3169 */ 3170 int 3171 irdma_sc_cqp_init(struct irdma_sc_cqp *cqp, 3172 struct irdma_cqp_init_info *info) 3173 { 3174 u8 hw_sq_size; 3175 3176 if (info->sq_size > IRDMA_CQP_SW_SQSIZE_2048 || 3177 info->sq_size < IRDMA_CQP_SW_SQSIZE_4 || 3178 ((info->sq_size & (info->sq_size - 1)))) 3179 return -EINVAL; 3180 3181 hw_sq_size = irdma_get_encoded_wqe_size(info->sq_size, 3182 IRDMA_QUEUE_TYPE_CQP); 3183 cqp->size = sizeof(*cqp); 3184 cqp->sq_size = info->sq_size; 3185 cqp->hw_sq_size = hw_sq_size; 3186 cqp->sq_base = info->sq; 3187 cqp->host_ctx = info->host_ctx; 3188 cqp->sq_pa = info->sq_pa; 3189 cqp->host_ctx_pa = info->host_ctx_pa; 3190 cqp->dev = info->dev; 3191 cqp->struct_ver = info->struct_ver; 3192 cqp->hw_maj_ver = info->hw_maj_ver; 3193 cqp->hw_min_ver = info->hw_min_ver; 3194 cqp->scratch_array = info->scratch_array; 3195 cqp->polarity = 0; 3196 cqp->en_datacenter_tcp = info->en_datacenter_tcp; 3197 cqp->ena_vf_count = info->ena_vf_count; 3198 cqp->hmc_profile = info->hmc_profile; 3199 cqp->ceqs_per_vf = info->ceqs_per_vf; 3200 cqp->disable_packed = info->disable_packed; 3201 cqp->rocev2_rto_policy = info->rocev2_rto_policy; 3202 cqp->protocol_used = info->protocol_used; 3203 irdma_memcpy(&cqp->dcqcn_params, &info->dcqcn_params, sizeof(cqp->dcqcn_params)); 3204 cqp->en_rem_endpoint_trk = info->en_rem_endpoint_trk; 3205 info->dev->cqp = cqp; 3206 3207 IRDMA_RING_INIT(cqp->sq_ring, cqp->sq_size); 3208 cqp->requested_ops = 0; 3209 atomic64_set(&cqp->completed_ops, 0); 3210 /* for the cqp commands backlog. */ 3211 INIT_LIST_HEAD(&cqp->dev->cqp_cmd_head); 3212 3213 writel(0, cqp->dev->hw_regs[IRDMA_CQPTAIL]); 3214 writel(0, cqp->dev->hw_regs[IRDMA_CQPDB]); 3215 writel(0, cqp->dev->hw_regs[IRDMA_CCQPSTATUS]); 3216 3217 irdma_debug(cqp->dev, IRDMA_DEBUG_WQE, 3218 "sq_size[%04d] hw_sq_size[%04d] sq_base[%p] sq_pa[%llxh] cqp[%p] polarity[x%04x]\n", 3219 cqp->sq_size, cqp->hw_sq_size, cqp->sq_base, 3220 (unsigned long long)cqp->sq_pa, cqp, cqp->polarity); 3221 return 0; 3222 } 3223 3224 /** 3225 * irdma_sc_cqp_create - create cqp during bringup 3226 * @cqp: struct for cqp hw 3227 * @maj_err: If error, major err number 3228 * @min_err: If error, minor err number 3229 */ 3230 int 3231 irdma_sc_cqp_create(struct irdma_sc_cqp *cqp, u16 *maj_err, u16 *min_err) 3232 { 3233 u64 temp; 3234 u8 hw_rev; 3235 u32 cnt = 0, p1, p2, val = 0, err_code; 3236 int ret_code; 3237 3238 hw_rev = cqp->dev->hw_attrs.uk_attrs.hw_rev; 3239 cqp->sdbuf.size = IRDMA_UPDATE_SD_BUFF_SIZE * cqp->sq_size; 3240 cqp->sdbuf.va = irdma_allocate_dma_mem(cqp->dev->hw, &cqp->sdbuf, 3241 cqp->sdbuf.size, 3242 IRDMA_SD_BUF_ALIGNMENT); 3243 if (!cqp->sdbuf.va) 3244 return -ENOMEM; 3245 3246 spin_lock_init(&cqp->dev->cqp_lock); 3247 3248 temp = FIELD_PREP(IRDMA_CQPHC_SQSIZE, cqp->hw_sq_size) | 3249 FIELD_PREP(IRDMA_CQPHC_SVER, cqp->struct_ver) | 3250 FIELD_PREP(IRDMA_CQPHC_DISABLE_PFPDUS, cqp->disable_packed) | 3251 FIELD_PREP(IRDMA_CQPHC_CEQPERVF, cqp->ceqs_per_vf); 3252 if (hw_rev >= IRDMA_GEN_2) { 3253 temp |= FIELD_PREP(IRDMA_CQPHC_ROCEV2_RTO_POLICY, 3254 cqp->rocev2_rto_policy) | 3255 FIELD_PREP(IRDMA_CQPHC_PROTOCOL_USED, 3256 cqp->protocol_used); 3257 } 3258 3259 set_64bit_val(cqp->host_ctx, IRDMA_BYTE_0, temp); 3260 set_64bit_val(cqp->host_ctx, IRDMA_BYTE_8, cqp->sq_pa); 3261 3262 temp = FIELD_PREP(IRDMA_CQPHC_ENABLED_VFS, cqp->ena_vf_count) | 3263 FIELD_PREP(IRDMA_CQPHC_HMC_PROFILE, cqp->hmc_profile); 3264 3265 if (hw_rev >= IRDMA_GEN_2) 3266 temp |= FIELD_PREP(IRDMA_CQPHC_EN_REM_ENDPOINT_TRK, 3267 cqp->en_rem_endpoint_trk); 3268 set_64bit_val(cqp->host_ctx, IRDMA_BYTE_16, temp); 3269 set_64bit_val(cqp->host_ctx, IRDMA_BYTE_24, (uintptr_t)cqp); 3270 temp = FIELD_PREP(IRDMA_CQPHC_HW_MAJVER, cqp->hw_maj_ver) | 3271 FIELD_PREP(IRDMA_CQPHC_HW_MINVER, cqp->hw_min_ver); 3272 if (hw_rev >= IRDMA_GEN_2) { 3273 temp |= FIELD_PREP(IRDMA_CQPHC_MIN_RATE, cqp->dcqcn_params.min_rate) | 3274 FIELD_PREP(IRDMA_CQPHC_MIN_DEC_FACTOR, cqp->dcqcn_params.min_dec_factor); 3275 } 3276 set_64bit_val(cqp->host_ctx, IRDMA_BYTE_32, temp); 3277 set_64bit_val(cqp->host_ctx, IRDMA_BYTE_40, 0); 3278 temp = 0; 3279 if (hw_rev >= IRDMA_GEN_2) { 3280 temp |= FIELD_PREP(IRDMA_CQPHC_DCQCN_T, cqp->dcqcn_params.dcqcn_t) | 3281 FIELD_PREP(IRDMA_CQPHC_RAI_FACTOR, cqp->dcqcn_params.rai_factor) | 3282 FIELD_PREP(IRDMA_CQPHC_HAI_FACTOR, cqp->dcqcn_params.hai_factor); 3283 } 3284 set_64bit_val(cqp->host_ctx, IRDMA_BYTE_48, temp); 3285 temp = 0; 3286 if (hw_rev >= IRDMA_GEN_2) { 3287 temp |= FIELD_PREP(IRDMA_CQPHC_DCQCN_B, cqp->dcqcn_params.dcqcn_b) | 3288 FIELD_PREP(IRDMA_CQPHC_DCQCN_F, cqp->dcqcn_params.dcqcn_f) | 3289 FIELD_PREP(IRDMA_CQPHC_CC_CFG_VALID, cqp->dcqcn_params.cc_cfg_valid) | 3290 FIELD_PREP(IRDMA_CQPHC_RREDUCE_MPERIOD, cqp->dcqcn_params.rreduce_mperiod); 3291 } 3292 set_64bit_val(cqp->host_ctx, IRDMA_BYTE_56, temp); 3293 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "CQP_HOST_CTX WQE", 3294 cqp->host_ctx, IRDMA_CQP_CTX_SIZE * 8); 3295 p1 = RS_32_1(cqp->host_ctx_pa, 32); 3296 p2 = (u32)cqp->host_ctx_pa; 3297 3298 writel(p1, cqp->dev->hw_regs[IRDMA_CCQPHIGH]); 3299 writel(p2, cqp->dev->hw_regs[IRDMA_CCQPLOW]); 3300 3301 do { 3302 if (cnt++ > cqp->dev->hw_attrs.max_done_count) { 3303 ret_code = -ETIMEDOUT; 3304 goto err; 3305 } 3306 irdma_usec_delay(cqp->dev->hw_attrs.max_sleep_count); 3307 val = readl(cqp->dev->hw_regs[IRDMA_CCQPSTATUS]); 3308 } while (!val); 3309 3310 if (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_ERR)) { 3311 ret_code = -EOPNOTSUPP; 3312 goto err; 3313 } 3314 3315 cqp->process_cqp_sds = irdma_update_sds_noccq; 3316 return 0; 3317 3318 err: 3319 spin_lock_destroy(&cqp->dev->cqp_lock); 3320 irdma_free_dma_mem(cqp->dev->hw, &cqp->sdbuf); 3321 err_code = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]); 3322 *min_err = FIELD_GET(IRDMA_CQPERRCODES_CQP_MINOR_CODE, err_code); 3323 *maj_err = FIELD_GET(IRDMA_CQPERRCODES_CQP_MAJOR_CODE, err_code); 3324 return ret_code; 3325 } 3326 3327 /** 3328 * irdma_sc_cqp_post_sq - post of cqp's sq 3329 * @cqp: struct for cqp hw 3330 */ 3331 void 3332 irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp) 3333 { 3334 db_wr32(IRDMA_RING_CURRENT_HEAD(cqp->sq_ring), cqp->dev->cqp_db); 3335 3336 irdma_debug(cqp->dev, IRDMA_DEBUG_WQE, 3337 "CQP SQ head 0x%x tail 0x%x size 0x%x\n", cqp->sq_ring.head, 3338 cqp->sq_ring.tail, cqp->sq_ring.size); 3339 } 3340 3341 /** 3342 * irdma_sc_cqp_get_next_send_wqe_idx - get next wqe on cqp sq 3343 * and pass back index 3344 * @cqp: CQP HW structure 3345 * @scratch: private data for CQP WQE 3346 * @wqe_idx: WQE index of CQP SQ 3347 */ 3348 __le64 * 3349 irdma_sc_cqp_get_next_send_wqe_idx(struct irdma_sc_cqp *cqp, u64 scratch, 3350 u32 *wqe_idx) 3351 { 3352 __le64 *wqe = NULL; 3353 int ret_code; 3354 3355 if (IRDMA_RING_FULL_ERR(cqp->sq_ring)) { 3356 irdma_debug(cqp->dev, IRDMA_DEBUG_WQE, 3357 "CQP SQ is full, head 0x%x tail 0x%x size 0x%x\n", 3358 cqp->sq_ring.head, cqp->sq_ring.tail, 3359 cqp->sq_ring.size); 3360 return NULL; 3361 } 3362 IRDMA_ATOMIC_RING_MOVE_HEAD(cqp->sq_ring, *wqe_idx, ret_code); 3363 if (ret_code) 3364 return NULL; 3365 3366 cqp->requested_ops++; 3367 if (!*wqe_idx) 3368 cqp->polarity = !cqp->polarity; 3369 wqe = cqp->sq_base[*wqe_idx].elem; 3370 cqp->scratch_array[*wqe_idx] = scratch; 3371 3372 memset(&wqe[0], 0, 24); 3373 memset(&wqe[4], 0, 32); 3374 3375 return wqe; 3376 } 3377 3378 /** 3379 * irdma_sc_cqp_destroy - destroy cqp during close 3380 * @cqp: struct for cqp hw 3381 * @free_hwcqp: true for regular cqp destroy; false for reset path 3382 */ 3383 int 3384 irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp, bool free_hwcqp) 3385 { 3386 u32 cnt = 0, val; 3387 int ret_code = 0; 3388 3389 if (free_hwcqp) { 3390 writel(0, cqp->dev->hw_regs[IRDMA_CCQPHIGH]); 3391 writel(0, cqp->dev->hw_regs[IRDMA_CCQPLOW]); 3392 do { 3393 if (cnt++ > cqp->dev->hw_attrs.max_done_count) { 3394 ret_code = -ETIMEDOUT; 3395 break; 3396 } 3397 irdma_usec_delay(cqp->dev->hw_attrs.max_sleep_count); 3398 val = readl(cqp->dev->hw_regs[IRDMA_CCQPSTATUS]); 3399 } while (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_DONE)); 3400 } 3401 irdma_free_dma_mem(cqp->dev->hw, &cqp->sdbuf); 3402 spin_lock_destroy(&cqp->dev->cqp_lock); 3403 return ret_code; 3404 } 3405 3406 /** 3407 * irdma_sc_ccq_arm - enable intr for control cq 3408 * @ccq: ccq sc struct 3409 */ 3410 void 3411 irdma_sc_ccq_arm(struct irdma_sc_cq *ccq) 3412 { 3413 unsigned long flags; 3414 u64 temp_val; 3415 u16 sw_cq_sel; 3416 u8 arm_next_se; 3417 u8 arm_seq_num; 3418 3419 spin_lock_irqsave(&ccq->dev->cqp_lock, flags); 3420 get_64bit_val(ccq->cq_uk.shadow_area, IRDMA_BYTE_32, &temp_val); 3421 sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val); 3422 arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val); 3423 arm_seq_num = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val); 3424 arm_seq_num++; 3425 temp_val = FIELD_PREP(IRDMA_CQ_DBSA_ARM_SEQ_NUM, arm_seq_num) | 3426 FIELD_PREP(IRDMA_CQ_DBSA_SW_CQ_SELECT, sw_cq_sel) | 3427 FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) | 3428 FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, 1); 3429 set_64bit_val(ccq->cq_uk.shadow_area, IRDMA_BYTE_32, temp_val); 3430 spin_unlock_irqrestore(&ccq->dev->cqp_lock, flags); 3431 3432 irdma_wmb(); /* make sure shadow area is updated before arming */ 3433 3434 db_wr32(ccq->cq_uk.cq_id, ccq->dev->cq_arm_db); 3435 } 3436 3437 /** 3438 * irdma_sc_ccq_get_cqe_info - get ccq's cq entry 3439 * @ccq: ccq sc struct 3440 * @info: completion q entry to return 3441 */ 3442 int 3443 irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq, 3444 struct irdma_ccq_cqe_info *info) 3445 { 3446 u64 qp_ctx, temp, temp1; 3447 __le64 *cqe; 3448 struct irdma_sc_cqp *cqp; 3449 u32 wqe_idx; 3450 u32 error; 3451 u8 polarity; 3452 int ret_code = 0; 3453 unsigned long flags; 3454 3455 if (ccq->cq_uk.avoid_mem_cflct) 3456 cqe = IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(&ccq->cq_uk); 3457 else 3458 cqe = IRDMA_GET_CURRENT_CQ_ELEM(&ccq->cq_uk); 3459 3460 get_64bit_val(cqe, IRDMA_BYTE_24, &temp); 3461 polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, temp); 3462 if (polarity != ccq->cq_uk.polarity) 3463 return -ENOENT; 3464 3465 /* Ensure CEQE contents are read after valid bit is checked */ 3466 rmb(); 3467 3468 get_64bit_val(cqe, IRDMA_BYTE_8, &qp_ctx); 3469 cqp = (struct irdma_sc_cqp *)(irdma_uintptr) qp_ctx; 3470 info->error = (bool)FIELD_GET(IRDMA_CQ_ERROR, temp); 3471 info->maj_err_code = IRDMA_CQPSQ_MAJ_NO_ERROR; 3472 info->min_err_code = (u16)FIELD_GET(IRDMA_CQ_MINERR, temp); 3473 if (info->error) { 3474 info->maj_err_code = (u16)FIELD_GET(IRDMA_CQ_MAJERR, temp); 3475 error = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]); 3476 irdma_debug(cqp->dev, IRDMA_DEBUG_CQP, 3477 "CQPERRCODES error_code[x%08X]\n", error); 3478 } 3479 3480 wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, temp); 3481 info->scratch = cqp->scratch_array[wqe_idx]; 3482 3483 get_64bit_val(cqe, IRDMA_BYTE_16, &temp1); 3484 info->op_ret_val = (u32)FIELD_GET(IRDMA_CCQ_OPRETVAL, temp1); 3485 3486 get_64bit_val(cqp->sq_base[wqe_idx].elem, IRDMA_BYTE_24, &temp1); 3487 info->op_code = (u8)FIELD_GET(IRDMA_CQPSQ_OPCODE, temp1); 3488 info->cqp = cqp; 3489 3490 /* move the head for cq */ 3491 IRDMA_RING_MOVE_HEAD(ccq->cq_uk.cq_ring, ret_code); 3492 if (!IRDMA_RING_CURRENT_HEAD(ccq->cq_uk.cq_ring)) 3493 ccq->cq_uk.polarity ^= 1; 3494 3495 /* update cq tail in cq shadow memory also */ 3496 IRDMA_RING_MOVE_TAIL(ccq->cq_uk.cq_ring); 3497 set_64bit_val(ccq->cq_uk.shadow_area, IRDMA_BYTE_0, 3498 IRDMA_RING_CURRENT_HEAD(ccq->cq_uk.cq_ring)); 3499 3500 irdma_wmb(); /* make sure shadow area is updated before moving tail */ 3501 3502 spin_lock_irqsave(&cqp->dev->cqp_lock, flags); 3503 IRDMA_RING_MOVE_TAIL(cqp->sq_ring); 3504 spin_unlock_irqrestore(&cqp->dev->cqp_lock, flags); 3505 atomic64_inc(&cqp->completed_ops); 3506 3507 return ret_code; 3508 } 3509 3510 /** 3511 * irdma_sc_poll_for_cqp_op_done - Waits for last write to complete in CQP SQ 3512 * @cqp: struct for cqp hw 3513 * @op_code: cqp opcode for completion 3514 * @compl_info: completion q entry to return 3515 */ 3516 int 3517 irdma_sc_poll_for_cqp_op_done(struct irdma_sc_cqp *cqp, u8 op_code, 3518 struct irdma_ccq_cqe_info *compl_info) 3519 { 3520 struct irdma_ccq_cqe_info info = {0}; 3521 struct irdma_sc_cq *ccq; 3522 int ret_code = 0; 3523 u32 cnt = 0; 3524 3525 ccq = cqp->dev->ccq; 3526 while (1) { 3527 if (cnt++ > 100 * cqp->dev->hw_attrs.max_done_count) 3528 return -ETIMEDOUT; 3529 3530 if (irdma_sc_ccq_get_cqe_info(ccq, &info)) { 3531 irdma_usec_delay(cqp->dev->hw_attrs.max_sleep_count); 3532 continue; 3533 } 3534 if (info.error && info.op_code != IRDMA_CQP_OP_QUERY_STAG) { 3535 ret_code = -EIO; 3536 break; 3537 } 3538 /* make sure op code matches */ 3539 if (op_code == info.op_code) 3540 break; 3541 irdma_debug(cqp->dev, IRDMA_DEBUG_WQE, 3542 "opcode mismatch for my op code 0x%x, returned opcode %x\n", 3543 op_code, info.op_code); 3544 } 3545 3546 if (compl_info) 3547 irdma_memcpy(compl_info, &info, sizeof(*compl_info)); 3548 3549 return ret_code; 3550 } 3551 3552 /** 3553 * irdma_sc_manage_hmc_pm_func_table - manage of function table 3554 * @cqp: struct for cqp hw 3555 * @scratch: u64 saved to be used during cqp completion 3556 * @info: info for the manage function table operation 3557 * @post_sq: flag for cqp db to ring 3558 */ 3559 static int 3560 irdma_sc_manage_hmc_pm_func_table(struct irdma_sc_cqp *cqp, 3561 struct irdma_hmc_fcn_info *info, 3562 u64 scratch, bool post_sq) 3563 { 3564 __le64 *wqe; 3565 u64 hdr; 3566 3567 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 3568 if (!wqe) 3569 return -ENOSPC; 3570 3571 hdr = FIELD_PREP(IRDMA_CQPSQ_MHMC_VFIDX, info->vf_id) | 3572 FIELD_PREP(IRDMA_CQPSQ_OPCODE, 3573 IRDMA_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE) | 3574 FIELD_PREP(IRDMA_CQPSQ_MHMC_FREEPMFN, info->free_fcn) | 3575 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 3576 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 3577 3578 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 3579 3580 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, 3581 "MANAGE_HMC_PM_FUNC_TABLE WQE", wqe, 3582 IRDMA_CQP_WQE_SIZE * 8); 3583 if (post_sq) 3584 irdma_sc_cqp_post_sq(cqp); 3585 3586 return 0; 3587 } 3588 3589 /** 3590 * irdma_sc_commit_fpm_val_done - wait for cqp eqe completion 3591 * for fpm commit 3592 * @cqp: struct for cqp hw 3593 */ 3594 static int 3595 irdma_sc_commit_fpm_val_done(struct irdma_sc_cqp *cqp) 3596 { 3597 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_COMMIT_FPM_VAL, 3598 NULL); 3599 } 3600 3601 /** 3602 * irdma_sc_commit_fpm_val - cqp wqe for commit fpm values 3603 * @cqp: struct for cqp hw 3604 * @scratch: u64 saved to be used during cqp completion 3605 * @hmc_fn_id: hmc function id 3606 * @commit_fpm_mem: Memory for fpm values 3607 * @post_sq: flag for cqp db to ring 3608 * @wait_type: poll ccq or cqp registers for cqp completion 3609 */ 3610 static int 3611 irdma_sc_commit_fpm_val(struct irdma_sc_cqp *cqp, u64 scratch, 3612 u16 hmc_fn_id, 3613 struct irdma_dma_mem *commit_fpm_mem, 3614 bool post_sq, u8 wait_type) 3615 { 3616 __le64 *wqe; 3617 u64 hdr; 3618 u32 tail, val, error; 3619 int ret_code = 0; 3620 3621 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 3622 if (!wqe) 3623 return -ENOSPC; 3624 3625 set_64bit_val(wqe, IRDMA_BYTE_16, hmc_fn_id); 3626 set_64bit_val(wqe, IRDMA_BYTE_32, commit_fpm_mem->pa); 3627 3628 hdr = FIELD_PREP(IRDMA_CQPSQ_BUFSIZE, IRDMA_COMMIT_FPM_BUF_SIZE) | 3629 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_COMMIT_FPM_VAL) | 3630 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 3631 3632 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 3633 3634 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 3635 3636 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "COMMIT_FPM_VAL WQE", wqe, 3637 IRDMA_CQP_WQE_SIZE * 8); 3638 irdma_get_cqp_reg_info(cqp, &val, &tail, &error); 3639 3640 if (post_sq) { 3641 irdma_sc_cqp_post_sq(cqp); 3642 if (wait_type == IRDMA_CQP_WAIT_POLL_REGS) 3643 ret_code = irdma_cqp_poll_registers(cqp, tail, 3644 cqp->dev->hw_attrs.max_done_count); 3645 else if (wait_type == IRDMA_CQP_WAIT_POLL_CQ) 3646 ret_code = irdma_sc_commit_fpm_val_done(cqp); 3647 } 3648 3649 return ret_code; 3650 } 3651 3652 /** 3653 * irdma_sc_query_fpm_val_done - poll for cqp wqe completion for 3654 * query fpm 3655 * @cqp: struct for cqp hw 3656 */ 3657 static int 3658 irdma_sc_query_fpm_val_done(struct irdma_sc_cqp *cqp) 3659 { 3660 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_QUERY_FPM_VAL, 3661 NULL); 3662 } 3663 3664 /** 3665 * irdma_sc_query_fpm_val - cqp wqe query fpm values 3666 * @cqp: struct for cqp hw 3667 * @scratch: u64 saved to be used during cqp completion 3668 * @hmc_fn_id: hmc function id 3669 * @query_fpm_mem: memory for return fpm values 3670 * @post_sq: flag for cqp db to ring 3671 * @wait_type: poll ccq or cqp registers for cqp completion 3672 */ 3673 static int 3674 irdma_sc_query_fpm_val(struct irdma_sc_cqp *cqp, u64 scratch, 3675 u16 hmc_fn_id, 3676 struct irdma_dma_mem *query_fpm_mem, 3677 bool post_sq, u8 wait_type) 3678 { 3679 __le64 *wqe; 3680 u64 hdr; 3681 u32 tail, val, error; 3682 int ret_code = 0; 3683 3684 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 3685 if (!wqe) 3686 return -ENOSPC; 3687 3688 set_64bit_val(wqe, IRDMA_BYTE_16, hmc_fn_id); 3689 set_64bit_val(wqe, IRDMA_BYTE_32, query_fpm_mem->pa); 3690 3691 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_QUERY_FPM_VAL) | 3692 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 3693 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 3694 3695 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 3696 3697 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "QUERY_FPM WQE", wqe, 3698 IRDMA_CQP_WQE_SIZE * 8); 3699 irdma_get_cqp_reg_info(cqp, &val, &tail, &error); 3700 3701 if (post_sq) { 3702 irdma_sc_cqp_post_sq(cqp); 3703 if (wait_type == IRDMA_CQP_WAIT_POLL_REGS) 3704 ret_code = irdma_cqp_poll_registers(cqp, tail, 3705 cqp->dev->hw_attrs.max_done_count); 3706 else if (wait_type == IRDMA_CQP_WAIT_POLL_CQ) 3707 ret_code = irdma_sc_query_fpm_val_done(cqp); 3708 } 3709 3710 return ret_code; 3711 } 3712 3713 /** 3714 * irdma_sc_ceq_init - initialize ceq 3715 * @ceq: ceq sc structure 3716 * @info: ceq initialization info 3717 */ 3718 int 3719 irdma_sc_ceq_init(struct irdma_sc_ceq *ceq, 3720 struct irdma_ceq_init_info *info) 3721 { 3722 u32 pble_obj_cnt; 3723 3724 if (info->elem_cnt < info->dev->hw_attrs.min_hw_ceq_size || 3725 info->elem_cnt > info->dev->hw_attrs.max_hw_ceq_size) 3726 return -EINVAL; 3727 3728 if (info->ceq_id > (info->dev->hmc_fpm_misc.max_ceqs - 1)) 3729 return -EINVAL; 3730 pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; 3731 3732 if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt) 3733 return -EINVAL; 3734 3735 ceq->size = sizeof(*ceq); 3736 ceq->ceqe_base = (struct irdma_ceqe *)info->ceqe_base; 3737 ceq->ceq_id = info->ceq_id; 3738 ceq->dev = info->dev; 3739 ceq->elem_cnt = info->elem_cnt; 3740 ceq->ceq_elem_pa = info->ceqe_pa; 3741 ceq->virtual_map = info->virtual_map; 3742 ceq->itr_no_expire = info->itr_no_expire; 3743 ceq->reg_cq = info->reg_cq; 3744 ceq->reg_cq_size = 0; 3745 spin_lock_init(&ceq->req_cq_lock); 3746 ceq->pbl_chunk_size = (ceq->virtual_map ? info->pbl_chunk_size : 0); 3747 ceq->first_pm_pbl_idx = (ceq->virtual_map ? info->first_pm_pbl_idx : 0); 3748 ceq->pbl_list = (ceq->virtual_map ? info->pbl_list : NULL); 3749 ceq->tph_en = info->tph_en; 3750 ceq->tph_val = info->tph_val; 3751 ceq->vsi = info->vsi; 3752 ceq->polarity = 1; 3753 IRDMA_RING_INIT(ceq->ceq_ring, ceq->elem_cnt); 3754 ceq->dev->ceq[info->ceq_id] = ceq; 3755 3756 return 0; 3757 } 3758 3759 /** 3760 * irdma_sc_ceq_create - create ceq wqe 3761 * @ceq: ceq sc structure 3762 * @scratch: u64 saved to be used during cqp completion 3763 * @post_sq: flag for cqp db to ring 3764 */ 3765 static int 3766 irdma_sc_ceq_create(struct irdma_sc_ceq *ceq, u64 scratch, 3767 bool post_sq) 3768 { 3769 struct irdma_sc_cqp *cqp; 3770 __le64 *wqe; 3771 u64 hdr; 3772 3773 cqp = ceq->dev->cqp; 3774 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 3775 if (!wqe) 3776 return -ENOSPC; 3777 set_64bit_val(wqe, IRDMA_BYTE_16, ceq->elem_cnt); 3778 set_64bit_val(wqe, IRDMA_BYTE_32, 3779 (ceq->virtual_map ? 0 : ceq->ceq_elem_pa)); 3780 set_64bit_val(wqe, IRDMA_BYTE_48, 3781 (ceq->virtual_map ? ceq->first_pm_pbl_idx : 0)); 3782 set_64bit_val(wqe, IRDMA_BYTE_56, 3783 FIELD_PREP(IRDMA_CQPSQ_TPHVAL, ceq->tph_val) | 3784 FIELD_PREP(IRDMA_CQPSQ_VSIIDX, ceq->vsi->vsi_idx)); 3785 hdr = FIELD_PREP(IRDMA_CQPSQ_CEQ_CEQID, ceq->ceq_id) | 3786 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_CEQ) | 3787 FIELD_PREP(IRDMA_CQPSQ_CEQ_LPBLSIZE, ceq->pbl_chunk_size) | 3788 FIELD_PREP(IRDMA_CQPSQ_CEQ_VMAP, ceq->virtual_map) | 3789 FIELD_PREP(IRDMA_CQPSQ_CEQ_ITRNOEXPIRE, ceq->itr_no_expire) | 3790 FIELD_PREP(IRDMA_CQPSQ_TPHEN, ceq->tph_en) | 3791 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 3792 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 3793 3794 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 3795 3796 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "CEQ_CREATE WQE", wqe, 3797 IRDMA_CQP_WQE_SIZE * 8); 3798 if (post_sq) 3799 irdma_sc_cqp_post_sq(cqp); 3800 3801 return 0; 3802 } 3803 3804 /** 3805 * irdma_sc_cceq_create_done - poll for control ceq wqe to complete 3806 * @ceq: ceq sc structure 3807 */ 3808 static int 3809 irdma_sc_cceq_create_done(struct irdma_sc_ceq *ceq) 3810 { 3811 struct irdma_sc_cqp *cqp; 3812 3813 cqp = ceq->dev->cqp; 3814 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_CREATE_CEQ, 3815 NULL); 3816 } 3817 3818 /** 3819 * irdma_sc_cceq_destroy_done - poll for destroy cceq to complete 3820 * @ceq: ceq sc structure 3821 */ 3822 int 3823 irdma_sc_cceq_destroy_done(struct irdma_sc_ceq *ceq) 3824 { 3825 struct irdma_sc_cqp *cqp; 3826 3827 if (ceq->reg_cq) 3828 irdma_sc_remove_cq_ctx(ceq, ceq->dev->ccq); 3829 cqp = ceq->dev->cqp; 3830 cqp->process_cqp_sds = irdma_update_sds_noccq; 3831 3832 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_DESTROY_CEQ, 3833 NULL); 3834 } 3835 3836 /** 3837 * irdma_sc_cceq_create - create cceq 3838 * @ceq: ceq sc structure 3839 * @scratch: u64 saved to be used during cqp completion 3840 */ 3841 int 3842 irdma_sc_cceq_create(struct irdma_sc_ceq *ceq, u64 scratch) 3843 { 3844 int ret_code; 3845 struct irdma_sc_dev *dev = ceq->dev; 3846 3847 dev->ccq->vsi = ceq->vsi; 3848 if (ceq->reg_cq) { 3849 ret_code = irdma_sc_add_cq_ctx(ceq, ceq->dev->ccq); 3850 if (ret_code) 3851 return ret_code; 3852 } 3853 ret_code = irdma_sc_ceq_create(ceq, scratch, true); 3854 if (!ret_code) 3855 return irdma_sc_cceq_create_done(ceq); 3856 3857 return ret_code; 3858 } 3859 3860 /** 3861 * irdma_sc_ceq_destroy - destroy ceq 3862 * @ceq: ceq sc structure 3863 * @scratch: u64 saved to be used during cqp completion 3864 * @post_sq: flag for cqp db to ring 3865 */ 3866 int 3867 irdma_sc_ceq_destroy(struct irdma_sc_ceq *ceq, u64 scratch, bool post_sq) 3868 { 3869 struct irdma_sc_cqp *cqp; 3870 __le64 *wqe; 3871 u64 hdr; 3872 3873 cqp = ceq->dev->cqp; 3874 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 3875 if (!wqe) 3876 return -ENOSPC; 3877 3878 set_64bit_val(wqe, IRDMA_BYTE_16, ceq->elem_cnt); 3879 set_64bit_val(wqe, IRDMA_BYTE_48, ceq->first_pm_pbl_idx); 3880 hdr = ceq->ceq_id | 3881 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_CEQ) | 3882 FIELD_PREP(IRDMA_CQPSQ_CEQ_LPBLSIZE, ceq->pbl_chunk_size) | 3883 FIELD_PREP(IRDMA_CQPSQ_CEQ_VMAP, ceq->virtual_map) | 3884 FIELD_PREP(IRDMA_CQPSQ_TPHEN, ceq->tph_en) | 3885 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 3886 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 3887 3888 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 3889 3890 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "CEQ_DESTROY WQE", wqe, 3891 IRDMA_CQP_WQE_SIZE * 8); 3892 ceq->dev->ceq[ceq->ceq_id] = NULL; 3893 if (post_sq) 3894 irdma_sc_cqp_post_sq(cqp); 3895 3896 return 0; 3897 } 3898 3899 /** 3900 * irdma_sc_process_ceq - process ceq 3901 * @dev: sc device struct 3902 * @ceq: ceq sc structure 3903 * 3904 * It is expected caller serializes this function with cleanup_ceqes() 3905 * because these functions manipulate the same ceq 3906 */ 3907 void * 3908 irdma_sc_process_ceq(struct irdma_sc_dev *dev, struct irdma_sc_ceq *ceq) 3909 { 3910 u64 temp; 3911 __le64 *ceqe; 3912 struct irdma_sc_cq *cq = NULL; 3913 struct irdma_sc_cq *temp_cq; 3914 u8 polarity; 3915 u32 cq_idx; 3916 unsigned long flags; 3917 3918 do { 3919 cq_idx = 0; 3920 ceqe = IRDMA_GET_CURRENT_CEQ_ELEM(ceq); 3921 get_64bit_val(ceqe, IRDMA_BYTE_0, &temp); 3922 polarity = (u8)FIELD_GET(IRDMA_CEQE_VALID, temp); 3923 if (polarity != ceq->polarity) 3924 return NULL; 3925 3926 temp_cq = (struct irdma_sc_cq *)(irdma_uintptr) LS_64_1(temp, 1); 3927 if (!temp_cq) { 3928 cq_idx = IRDMA_INVALID_CQ_IDX; 3929 IRDMA_RING_MOVE_TAIL(ceq->ceq_ring); 3930 3931 if (!IRDMA_RING_CURRENT_TAIL(ceq->ceq_ring)) 3932 ceq->polarity ^= 1; 3933 continue; 3934 } 3935 3936 cq = temp_cq; 3937 if (ceq->reg_cq) { 3938 spin_lock_irqsave(&ceq->req_cq_lock, flags); 3939 cq_idx = irdma_sc_find_reg_cq(ceq, cq); 3940 spin_unlock_irqrestore(&ceq->req_cq_lock, flags); 3941 } 3942 IRDMA_RING_MOVE_TAIL(ceq->ceq_ring); 3943 if (!IRDMA_RING_CURRENT_TAIL(ceq->ceq_ring)) 3944 ceq->polarity ^= 1; 3945 } while (cq_idx == IRDMA_INVALID_CQ_IDX); 3946 3947 if (cq) 3948 irdma_sc_cq_ack(cq); 3949 return cq; 3950 } 3951 3952 /** 3953 * irdma_sc_cleanup_ceqes - clear the valid ceqes ctx matching the cq 3954 * @cq: cq for which the ceqes need to be cleaned up 3955 * @ceq: ceq ptr 3956 * 3957 * The function is called after the cq is destroyed to cleanup 3958 * its pending ceqe entries. It is expected caller serializes this 3959 * function with process_ceq() in interrupt context. 3960 */ 3961 void 3962 irdma_sc_cleanup_ceqes(struct irdma_sc_cq *cq, struct irdma_sc_ceq *ceq) 3963 { 3964 struct irdma_sc_cq *next_cq; 3965 u8 ceq_polarity = ceq->polarity; 3966 __le64 *ceqe; 3967 u8 polarity; 3968 u64 temp; 3969 int next; 3970 u32 i; 3971 3972 next = IRDMA_RING_GET_NEXT_TAIL(ceq->ceq_ring, 0); 3973 3974 for (i = 1; i <= IRDMA_RING_SIZE(*ceq); i++) { 3975 ceqe = IRDMA_GET_CEQ_ELEM_AT_POS(ceq, next); 3976 3977 get_64bit_val(ceqe, IRDMA_BYTE_0, &temp); 3978 polarity = (u8)FIELD_GET(IRDMA_CEQE_VALID, temp); 3979 if (polarity != ceq_polarity) 3980 return; 3981 3982 next_cq = (struct irdma_sc_cq *)(irdma_uintptr) LS_64_1(temp, 1); 3983 if (cq == next_cq) 3984 set_64bit_val(ceqe, IRDMA_BYTE_0, temp & IRDMA_CEQE_VALID); 3985 3986 next = IRDMA_RING_GET_NEXT_TAIL(ceq->ceq_ring, i); 3987 if (!next) 3988 ceq_polarity ^= 1; 3989 } 3990 } 3991 3992 /** 3993 * irdma_sc_aeq_init - initialize aeq 3994 * @aeq: aeq structure ptr 3995 * @info: aeq initialization info 3996 */ 3997 int 3998 irdma_sc_aeq_init(struct irdma_sc_aeq *aeq, 3999 struct irdma_aeq_init_info *info) 4000 { 4001 u32 pble_obj_cnt; 4002 4003 if (info->elem_cnt < info->dev->hw_attrs.min_hw_aeq_size || 4004 info->elem_cnt > info->dev->hw_attrs.max_hw_aeq_size) 4005 return -EINVAL; 4006 4007 pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; 4008 4009 if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt) 4010 return -EINVAL; 4011 4012 aeq->size = sizeof(*aeq); 4013 aeq->polarity = 1; 4014 aeq->aeqe_base = (struct irdma_sc_aeqe *)info->aeqe_base; 4015 aeq->dev = info->dev; 4016 aeq->elem_cnt = info->elem_cnt; 4017 aeq->aeq_elem_pa = info->aeq_elem_pa; 4018 IRDMA_RING_INIT(aeq->aeq_ring, aeq->elem_cnt); 4019 aeq->virtual_map = info->virtual_map; 4020 aeq->pbl_list = (aeq->virtual_map ? info->pbl_list : NULL); 4021 aeq->pbl_chunk_size = (aeq->virtual_map ? info->pbl_chunk_size : 0); 4022 aeq->first_pm_pbl_idx = (aeq->virtual_map ? info->first_pm_pbl_idx : 0); 4023 aeq->msix_idx = info->msix_idx; 4024 info->dev->aeq = aeq; 4025 4026 return 0; 4027 } 4028 4029 /** 4030 * irdma_sc_aeq_create - create aeq 4031 * @aeq: aeq structure ptr 4032 * @scratch: u64 saved to be used during cqp completion 4033 * @post_sq: flag for cqp db to ring 4034 */ 4035 static int 4036 irdma_sc_aeq_create(struct irdma_sc_aeq *aeq, u64 scratch, 4037 bool post_sq) 4038 { 4039 __le64 *wqe; 4040 struct irdma_sc_cqp *cqp; 4041 u64 hdr; 4042 4043 cqp = aeq->dev->cqp; 4044 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 4045 if (!wqe) 4046 return -ENOSPC; 4047 set_64bit_val(wqe, IRDMA_BYTE_16, aeq->elem_cnt); 4048 set_64bit_val(wqe, IRDMA_BYTE_32, 4049 (aeq->virtual_map ? 0 : aeq->aeq_elem_pa)); 4050 set_64bit_val(wqe, IRDMA_BYTE_48, 4051 (aeq->virtual_map ? aeq->first_pm_pbl_idx : 0)); 4052 4053 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_AEQ) | 4054 FIELD_PREP(IRDMA_CQPSQ_AEQ_LPBLSIZE, aeq->pbl_chunk_size) | 4055 FIELD_PREP(IRDMA_CQPSQ_AEQ_VMAP, aeq->virtual_map) | 4056 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 4057 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 4058 4059 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 4060 4061 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "AEQ_CREATE WQE", wqe, 4062 IRDMA_CQP_WQE_SIZE * 8); 4063 if (post_sq) 4064 irdma_sc_cqp_post_sq(cqp); 4065 4066 return 0; 4067 } 4068 4069 /** 4070 * irdma_sc_aeq_destroy - destroy aeq during close 4071 * @aeq: aeq structure ptr 4072 * @scratch: u64 saved to be used during cqp completion 4073 * @post_sq: flag for cqp db to ring 4074 */ 4075 int 4076 irdma_sc_aeq_destroy(struct irdma_sc_aeq *aeq, u64 scratch, bool post_sq) 4077 { 4078 __le64 *wqe; 4079 struct irdma_sc_cqp *cqp; 4080 struct irdma_sc_dev *dev; 4081 u64 hdr; 4082 4083 dev = aeq->dev; 4084 writel(0, dev->hw_regs[IRDMA_PFINT_AEQCTL]); 4085 4086 cqp = dev->cqp; 4087 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 4088 if (!wqe) 4089 return -ENOSPC; 4090 set_64bit_val(wqe, IRDMA_BYTE_16, aeq->elem_cnt); 4091 set_64bit_val(wqe, IRDMA_BYTE_48, aeq->first_pm_pbl_idx); 4092 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_AEQ) | 4093 FIELD_PREP(IRDMA_CQPSQ_AEQ_LPBLSIZE, aeq->pbl_chunk_size) | 4094 FIELD_PREP(IRDMA_CQPSQ_AEQ_VMAP, aeq->virtual_map) | 4095 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 4096 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 4097 4098 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 4099 4100 irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "AEQ_DESTROY WQE", wqe, 4101 IRDMA_CQP_WQE_SIZE * 8); 4102 if (post_sq) 4103 irdma_sc_cqp_post_sq(cqp); 4104 return 0; 4105 } 4106 4107 /** 4108 * irdma_sc_get_next_aeqe - get next aeq entry 4109 * @aeq: aeq structure ptr 4110 * @info: aeqe info to be returned 4111 */ 4112 int 4113 irdma_sc_get_next_aeqe(struct irdma_sc_aeq *aeq, 4114 struct irdma_aeqe_info *info) 4115 { 4116 u64 temp, compl_ctx; 4117 __le64 *aeqe; 4118 u8 ae_src; 4119 u8 polarity; 4120 4121 aeqe = IRDMA_GET_CURRENT_AEQ_ELEM(aeq); 4122 get_64bit_val(aeqe, IRDMA_BYTE_8, &temp); 4123 polarity = (u8)FIELD_GET(IRDMA_AEQE_VALID, temp); 4124 4125 if (aeq->polarity != polarity) 4126 return -ENOENT; 4127 4128 /* Ensure AEQE contents are read after valid bit is checked */ 4129 rmb(); 4130 4131 get_64bit_val(aeqe, IRDMA_BYTE_0, &compl_ctx); 4132 4133 irdma_debug_buf(aeq->dev, IRDMA_DEBUG_WQE, "AEQ_ENTRY WQE", aeqe, 16); 4134 4135 ae_src = (u8)FIELD_GET(IRDMA_AEQE_AESRC, temp); 4136 info->wqe_idx = (u16)FIELD_GET(IRDMA_AEQE_WQDESCIDX, temp); 4137 info->qp_cq_id = (u32)FIELD_GET(IRDMA_AEQE_QPCQID_LOW, temp) | 4138 ((u32)FIELD_GET(IRDMA_AEQE_QPCQID_HI, temp) << 18); 4139 info->ae_id = (u16)FIELD_GET(IRDMA_AEQE_AECODE, temp); 4140 info->tcp_state = (u8)FIELD_GET(IRDMA_AEQE_TCPSTATE, temp); 4141 info->iwarp_state = (u8)FIELD_GET(IRDMA_AEQE_IWSTATE, temp); 4142 info->q2_data_written = (u8)FIELD_GET(IRDMA_AEQE_Q2DATA, temp); 4143 info->aeqe_overflow = (bool)FIELD_GET(IRDMA_AEQE_OVERFLOW, temp); 4144 4145 info->ae_src = ae_src; 4146 switch (info->ae_id) { 4147 case IRDMA_AE_PRIV_OPERATION_DENIED: 4148 case IRDMA_AE_AMP_INVALIDATE_TYPE1_MW: 4149 case IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW: 4150 case IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG: 4151 case IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH: 4152 case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG: 4153 case IRDMA_AE_UDA_XMIT_BAD_PD: 4154 case IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT: 4155 case IRDMA_AE_BAD_CLOSE: 4156 case IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO: 4157 case IRDMA_AE_STAG_ZERO_INVALID: 4158 case IRDMA_AE_IB_RREQ_AND_Q1_FULL: 4159 case IRDMA_AE_IB_INVALID_REQUEST: 4160 case IRDMA_AE_WQE_UNEXPECTED_OPCODE: 4161 case IRDMA_AE_IB_REMOTE_ACCESS_ERROR: 4162 case IRDMA_AE_IB_REMOTE_OP_ERROR: 4163 case IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION: 4164 case IRDMA_AE_DDP_UBE_INVALID_MO: 4165 case IRDMA_AE_DDP_UBE_INVALID_QN: 4166 case IRDMA_AE_DDP_NO_L_BIT: 4167 case IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION: 4168 case IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE: 4169 case IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST: 4170 case IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP: 4171 case IRDMA_AE_ROCE_RSP_LENGTH_ERROR: 4172 case IRDMA_AE_ROCE_REQ_LENGTH_ERROR: 4173 case IRDMA_AE_INVALID_ARP_ENTRY: 4174 case IRDMA_AE_INVALID_TCP_OPTION_RCVD: 4175 case IRDMA_AE_STALE_ARP_ENTRY: 4176 case IRDMA_AE_INVALID_AH_ENTRY: 4177 case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR: 4178 case IRDMA_AE_LLP_SEGMENT_TOO_SMALL: 4179 case IRDMA_AE_LLP_TOO_MANY_RETRIES: 4180 case IRDMA_AE_LCE_QP_CATASTROPHIC: 4181 case IRDMA_AE_LLP_DOUBT_REACHABILITY: 4182 case IRDMA_AE_LLP_CONNECTION_ESTABLISHED: 4183 case IRDMA_AE_RESET_SENT: 4184 case IRDMA_AE_TERMINATE_SENT: 4185 case IRDMA_AE_RESET_NOT_SENT: 4186 case IRDMA_AE_QP_SUSPEND_COMPLETE: 4187 case IRDMA_AE_UDA_L4LEN_INVALID: 4188 info->qp = true; 4189 info->compl_ctx = compl_ctx; 4190 break; 4191 case IRDMA_AE_LCE_CQ_CATASTROPHIC: 4192 info->cq = true; 4193 info->compl_ctx = LS_64_1(compl_ctx, 1); 4194 ae_src = IRDMA_AE_SOURCE_RSVD; 4195 break; 4196 case IRDMA_AE_ROCE_EMPTY_MCG: 4197 case IRDMA_AE_ROCE_BAD_MC_IP_ADDR: 4198 case IRDMA_AE_ROCE_BAD_MC_QPID: 4199 case IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH: 4200 /* fallthrough */ 4201 case IRDMA_AE_LLP_CONNECTION_RESET: 4202 case IRDMA_AE_LLP_SYN_RECEIVED: 4203 case IRDMA_AE_LLP_FIN_RECEIVED: 4204 case IRDMA_AE_LLP_CLOSE_COMPLETE: 4205 case IRDMA_AE_LLP_TERMINATE_RECEIVED: 4206 case IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE: 4207 ae_src = IRDMA_AE_SOURCE_RSVD; 4208 info->qp = true; 4209 info->compl_ctx = compl_ctx; 4210 break; 4211 case IRDMA_AE_RESOURCE_EXHAUSTION: 4212 /* 4213 * ae_src contains the exhausted resource with a unique decoding. Set RSVD here to prevent matching 4214 * with a CQ or QP. 4215 */ 4216 ae_src = IRDMA_AE_SOURCE_RSVD; 4217 break; 4218 default: 4219 break; 4220 } 4221 4222 switch (ae_src) { 4223 case IRDMA_AE_SOURCE_RQ: 4224 case IRDMA_AE_SOURCE_RQ_0011: 4225 info->qp = true; 4226 info->rq = true; 4227 info->compl_ctx = compl_ctx; 4228 info->err_rq_idx_valid = true; 4229 break; 4230 case IRDMA_AE_SOURCE_CQ: 4231 case IRDMA_AE_SOURCE_CQ_0110: 4232 case IRDMA_AE_SOURCE_CQ_1010: 4233 case IRDMA_AE_SOURCE_CQ_1110: 4234 info->cq = true; 4235 info->compl_ctx = LS_64_1(compl_ctx, 1); 4236 break; 4237 case IRDMA_AE_SOURCE_SQ: 4238 case IRDMA_AE_SOURCE_SQ_0111: 4239 info->qp = true; 4240 info->sq = true; 4241 info->compl_ctx = compl_ctx; 4242 break; 4243 case IRDMA_AE_SOURCE_IN_WR: 4244 info->qp = true; 4245 info->compl_ctx = compl_ctx; 4246 info->in_rdrsp_wr = true; 4247 break; 4248 case IRDMA_AE_SOURCE_IN_RR: 4249 info->qp = true; 4250 info->compl_ctx = compl_ctx; 4251 info->in_rdrsp_wr = true; 4252 break; 4253 case IRDMA_AE_SOURCE_OUT_RR: 4254 case IRDMA_AE_SOURCE_OUT_RR_1111: 4255 info->qp = true; 4256 info->compl_ctx = compl_ctx; 4257 info->out_rdrsp = true; 4258 break; 4259 case IRDMA_AE_SOURCE_RSVD: 4260 default: 4261 break; 4262 } 4263 4264 IRDMA_RING_MOVE_TAIL(aeq->aeq_ring); 4265 if (!IRDMA_RING_CURRENT_TAIL(aeq->aeq_ring)) 4266 aeq->polarity ^= 1; 4267 4268 return 0; 4269 } 4270 4271 /** 4272 * irdma_sc_repost_aeq_entries - repost completed aeq entries 4273 * @dev: sc device struct 4274 * @count: allocate count 4275 */ 4276 void 4277 irdma_sc_repost_aeq_entries(struct irdma_sc_dev *dev, u32 count) 4278 { 4279 db_wr32(count, dev->aeq_alloc_db); 4280 4281 } 4282 4283 /** 4284 * irdma_sc_ccq_init - initialize control cq 4285 * @cq: sc's cq ctruct 4286 * @info: info for control cq initialization 4287 */ 4288 int 4289 irdma_sc_ccq_init(struct irdma_sc_cq *cq, struct irdma_ccq_init_info *info) 4290 { 4291 u32 pble_obj_cnt; 4292 4293 if (info->num_elem < info->dev->hw_attrs.uk_attrs.min_hw_cq_size || 4294 info->num_elem > info->dev->hw_attrs.uk_attrs.max_hw_cq_size) 4295 return -EINVAL; 4296 4297 if (info->ceq_id > (info->dev->hmc_fpm_misc.max_ceqs - 1)) 4298 return -EINVAL; 4299 4300 pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt; 4301 4302 if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt) 4303 return -EINVAL; 4304 4305 cq->cq_pa = info->cq_pa; 4306 cq->cq_uk.cq_base = info->cq_base; 4307 cq->shadow_area_pa = info->shadow_area_pa; 4308 cq->cq_uk.shadow_area = info->shadow_area; 4309 cq->shadow_read_threshold = info->shadow_read_threshold; 4310 cq->dev = info->dev; 4311 cq->ceq_id = info->ceq_id; 4312 cq->cq_uk.cq_size = info->num_elem; 4313 cq->cq_type = IRDMA_CQ_TYPE_CQP; 4314 cq->ceqe_mask = info->ceqe_mask; 4315 IRDMA_RING_INIT(cq->cq_uk.cq_ring, info->num_elem); 4316 cq->cq_uk.cq_id = 0; /* control cq is id 0 always */ 4317 cq->ceq_id_valid = info->ceq_id_valid; 4318 cq->tph_en = info->tph_en; 4319 cq->tph_val = info->tph_val; 4320 cq->cq_uk.avoid_mem_cflct = info->avoid_mem_cflct; 4321 cq->pbl_list = info->pbl_list; 4322 cq->virtual_map = info->virtual_map; 4323 cq->pbl_chunk_size = info->pbl_chunk_size; 4324 cq->first_pm_pbl_idx = info->first_pm_pbl_idx; 4325 cq->cq_uk.polarity = true; 4326 cq->vsi = info->vsi; 4327 cq->cq_uk.cq_ack_db = cq->dev->cq_ack_db; 4328 4329 /* Only applicable to CQs other than CCQ so initialize to zero */ 4330 cq->cq_uk.cqe_alloc_db = NULL; 4331 4332 info->dev->ccq = cq; 4333 return 0; 4334 } 4335 4336 /** 4337 * irdma_sc_ccq_create_done - poll cqp for ccq create 4338 * @ccq: ccq sc struct 4339 */ 4340 static inline int 4341 irdma_sc_ccq_create_done(struct irdma_sc_cq *ccq) 4342 { 4343 struct irdma_sc_cqp *cqp; 4344 4345 cqp = ccq->dev->cqp; 4346 4347 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_CREATE_CQ, NULL); 4348 } 4349 4350 /** 4351 * irdma_sc_ccq_create - create control cq 4352 * @ccq: ccq sc struct 4353 * @scratch: u64 saved to be used during cqp completion 4354 * @check_overflow: overlow flag for ccq 4355 * @post_sq: flag for cqp db to ring 4356 */ 4357 int 4358 irdma_sc_ccq_create(struct irdma_sc_cq *ccq, u64 scratch, 4359 bool check_overflow, bool post_sq) 4360 { 4361 int ret_code; 4362 4363 ret_code = irdma_sc_cq_create(ccq, scratch, check_overflow, post_sq); 4364 if (ret_code) 4365 return ret_code; 4366 4367 if (post_sq) { 4368 ret_code = irdma_sc_ccq_create_done(ccq); 4369 if (ret_code) 4370 return ret_code; 4371 } 4372 ccq->dev->cqp->process_cqp_sds = irdma_cqp_sds_cmd; 4373 4374 return 0; 4375 } 4376 4377 /** 4378 * irdma_sc_ccq_destroy - destroy ccq during close 4379 * @ccq: ccq sc struct 4380 * @scratch: u64 saved to be used during cqp completion 4381 * @post_sq: flag for cqp db to ring 4382 */ 4383 int 4384 irdma_sc_ccq_destroy(struct irdma_sc_cq *ccq, u64 scratch, bool post_sq) 4385 { 4386 struct irdma_sc_cqp *cqp; 4387 __le64 *wqe; 4388 u64 hdr; 4389 int ret_code = 0; 4390 u32 tail, val, error; 4391 4392 cqp = ccq->dev->cqp; 4393 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 4394 if (!wqe) 4395 return -ENOSPC; 4396 4397 set_64bit_val(wqe, IRDMA_BYTE_0, ccq->cq_uk.cq_size); 4398 set_64bit_val(wqe, IRDMA_BYTE_8, RS_64_1(ccq, 1)); 4399 set_64bit_val(wqe, IRDMA_BYTE_40, ccq->shadow_area_pa); 4400 4401 hdr = ccq->cq_uk.cq_id | 4402 FLD_LS_64(ccq->dev, (ccq->ceq_id_valid ? ccq->ceq_id : 0), 4403 IRDMA_CQPSQ_CQ_CEQID) | 4404 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_CQ) | 4405 FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, ccq->ceqe_mask) | 4406 FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, ccq->ceq_id_valid) | 4407 FIELD_PREP(IRDMA_CQPSQ_TPHEN, ccq->tph_en) | 4408 FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT, ccq->cq_uk.avoid_mem_cflct) | 4409 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 4410 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 4411 4412 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 4413 4414 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "CCQ_DESTROY WQE", wqe, 4415 IRDMA_CQP_WQE_SIZE * 8); 4416 irdma_get_cqp_reg_info(cqp, &val, &tail, &error); 4417 4418 if (post_sq) { 4419 irdma_sc_cqp_post_sq(cqp); 4420 ret_code = irdma_cqp_poll_registers(cqp, tail, 4421 cqp->dev->hw_attrs.max_done_count); 4422 } 4423 4424 cqp->process_cqp_sds = irdma_update_sds_noccq; 4425 4426 return ret_code; 4427 } 4428 4429 /** 4430 * irdma_sc_init_iw_hmc() - queries fpm values using cqp and populates hmc_info 4431 * @dev : ptr to irdma_dev struct 4432 * @hmc_fn_id: hmc function id 4433 */ 4434 int 4435 irdma_sc_init_iw_hmc(struct irdma_sc_dev *dev, u16 hmc_fn_id) 4436 { 4437 struct irdma_hmc_info *hmc_info; 4438 struct irdma_hmc_fpm_misc *hmc_fpm_misc; 4439 struct irdma_dma_mem query_fpm_mem; 4440 int ret_code = 0; 4441 u8 wait_type; 4442 4443 hmc_info = dev->hmc_info; 4444 hmc_fpm_misc = &dev->hmc_fpm_misc; 4445 query_fpm_mem.pa = dev->fpm_query_buf_pa; 4446 query_fpm_mem.va = dev->fpm_query_buf; 4447 hmc_info->hmc_fn_id = hmc_fn_id; 4448 wait_type = (u8)IRDMA_CQP_WAIT_POLL_REGS; 4449 4450 ret_code = irdma_sc_query_fpm_val(dev->cqp, 0, hmc_info->hmc_fn_id, 4451 &query_fpm_mem, true, wait_type); 4452 if (ret_code) 4453 return ret_code; 4454 4455 /* parse the fpm_query_buf and fill hmc obj info */ 4456 ret_code = irdma_sc_parse_fpm_query_buf(dev, query_fpm_mem.va, hmc_info, 4457 hmc_fpm_misc); 4458 4459 irdma_debug_buf(dev, IRDMA_DEBUG_HMC, "QUERY FPM BUFFER", 4460 query_fpm_mem.va, IRDMA_QUERY_FPM_BUF_SIZE); 4461 return ret_code; 4462 } 4463 4464 /** 4465 * irdma_sc_cfg_iw_fpm() - commits hmc obj cnt values using cqp 4466 * command and populates fpm base address in hmc_info 4467 * @dev : ptr to irdma_dev struct 4468 * @hmc_fn_id: hmc function id 4469 */ 4470 static int 4471 irdma_sc_cfg_iw_fpm(struct irdma_sc_dev *dev, u16 hmc_fn_id) 4472 { 4473 struct irdma_hmc_obj_info *obj_info; 4474 __le64 *buf; 4475 struct irdma_hmc_info *hmc_info; 4476 struct irdma_dma_mem commit_fpm_mem; 4477 int ret_code = 0; 4478 u8 wait_type; 4479 4480 hmc_info = dev->hmc_info; 4481 obj_info = hmc_info->hmc_obj; 4482 buf = dev->fpm_commit_buf; 4483 4484 set_64bit_val(buf, IRDMA_BYTE_0, (u64)obj_info[IRDMA_HMC_IW_QP].cnt); 4485 set_64bit_val(buf, IRDMA_BYTE_8, (u64)obj_info[IRDMA_HMC_IW_CQ].cnt); 4486 set_64bit_val(buf, IRDMA_BYTE_16, (u64)0); /* RSRVD */ 4487 set_64bit_val(buf, IRDMA_BYTE_24, (u64)obj_info[IRDMA_HMC_IW_HTE].cnt); 4488 set_64bit_val(buf, IRDMA_BYTE_32, (u64)obj_info[IRDMA_HMC_IW_ARP].cnt); 4489 set_64bit_val(buf, IRDMA_BYTE_40, (u64)0); /* RSVD */ 4490 set_64bit_val(buf, IRDMA_BYTE_48, (u64)obj_info[IRDMA_HMC_IW_MR].cnt); 4491 set_64bit_val(buf, IRDMA_BYTE_56, (u64)obj_info[IRDMA_HMC_IW_XF].cnt); 4492 set_64bit_val(buf, IRDMA_BYTE_64, (u64)obj_info[IRDMA_HMC_IW_XFFL].cnt); 4493 set_64bit_val(buf, IRDMA_BYTE_72, (u64)obj_info[IRDMA_HMC_IW_Q1].cnt); 4494 set_64bit_val(buf, IRDMA_BYTE_80, (u64)obj_info[IRDMA_HMC_IW_Q1FL].cnt); 4495 set_64bit_val(buf, IRDMA_BYTE_88, 4496 (u64)obj_info[IRDMA_HMC_IW_TIMER].cnt); 4497 set_64bit_val(buf, IRDMA_BYTE_96, 4498 (u64)obj_info[IRDMA_HMC_IW_FSIMC].cnt); 4499 set_64bit_val(buf, IRDMA_BYTE_104, 4500 (u64)obj_info[IRDMA_HMC_IW_FSIAV].cnt); 4501 set_64bit_val(buf, IRDMA_BYTE_112, 4502 (u64)obj_info[IRDMA_HMC_IW_PBLE].cnt); 4503 set_64bit_val(buf, IRDMA_BYTE_120, (u64)0); /* RSVD */ 4504 set_64bit_val(buf, IRDMA_BYTE_128, (u64)obj_info[IRDMA_HMC_IW_RRF].cnt); 4505 set_64bit_val(buf, IRDMA_BYTE_136, 4506 (u64)obj_info[IRDMA_HMC_IW_RRFFL].cnt); 4507 set_64bit_val(buf, IRDMA_BYTE_144, (u64)obj_info[IRDMA_HMC_IW_HDR].cnt); 4508 set_64bit_val(buf, IRDMA_BYTE_152, (u64)obj_info[IRDMA_HMC_IW_MD].cnt); 4509 set_64bit_val(buf, IRDMA_BYTE_160, 4510 (u64)obj_info[IRDMA_HMC_IW_OOISC].cnt); 4511 set_64bit_val(buf, IRDMA_BYTE_168, 4512 (u64)obj_info[IRDMA_HMC_IW_OOISCFFL].cnt); 4513 commit_fpm_mem.pa = dev->fpm_commit_buf_pa; 4514 commit_fpm_mem.va = dev->fpm_commit_buf; 4515 4516 wait_type = (u8)IRDMA_CQP_WAIT_POLL_REGS; 4517 irdma_debug_buf(dev, IRDMA_DEBUG_HMC, "COMMIT FPM BUFFER", 4518 commit_fpm_mem.va, IRDMA_COMMIT_FPM_BUF_SIZE); 4519 ret_code = irdma_sc_commit_fpm_val(dev->cqp, 0, hmc_info->hmc_fn_id, 4520 &commit_fpm_mem, true, wait_type); 4521 if (!ret_code) 4522 irdma_sc_parse_fpm_commit_buf(dev, dev->fpm_commit_buf, 4523 hmc_info->hmc_obj, 4524 &hmc_info->sd_table.sd_cnt); 4525 irdma_debug_buf(dev, IRDMA_DEBUG_HMC, "COMMIT FPM BUFFER", 4526 commit_fpm_mem.va, IRDMA_COMMIT_FPM_BUF_SIZE); 4527 4528 return ret_code; 4529 } 4530 4531 /** 4532 * cqp_sds_wqe_fill - fill cqp wqe doe sd 4533 * @cqp: struct for cqp hw 4534 * @info: sd info for wqe 4535 * @scratch: u64 saved to be used during cqp completion 4536 */ 4537 static int 4538 cqp_sds_wqe_fill(struct irdma_sc_cqp *cqp, 4539 struct irdma_update_sds_info *info, u64 scratch) 4540 { 4541 u64 data; 4542 u64 hdr; 4543 __le64 *wqe; 4544 int mem_entries, wqe_entries; 4545 struct irdma_dma_mem *sdbuf = &cqp->sdbuf; 4546 u64 offset = 0; 4547 u32 wqe_idx; 4548 4549 wqe = irdma_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx); 4550 if (!wqe) 4551 return -ENOSPC; 4552 4553 wqe_entries = (info->cnt > 3) ? 3 : info->cnt; 4554 mem_entries = info->cnt - wqe_entries; 4555 4556 if (mem_entries) { 4557 offset = wqe_idx * IRDMA_UPDATE_SD_BUFF_SIZE; 4558 irdma_memcpy(((char *)sdbuf->va + offset), &info->entry[3], mem_entries << 4); 4559 4560 data = (u64)sdbuf->pa + offset; 4561 } else { 4562 data = 0; 4563 } 4564 data |= FLD_LS_64(cqp->dev, info->hmc_fn_id, IRDMA_CQPSQ_UPESD_HMCFNID); 4565 set_64bit_val(wqe, IRDMA_BYTE_16, data); 4566 4567 switch (wqe_entries) { 4568 case 3: 4569 set_64bit_val(wqe, IRDMA_BYTE_48, 4570 (FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info->entry[2].cmd) | 4571 FIELD_PREP(IRDMA_CQPSQ_UPESD_ENTRY_VALID, 1))); 4572 4573 set_64bit_val(wqe, IRDMA_BYTE_56, info->entry[2].data); 4574 /* fallthrough */ 4575 case 2: 4576 set_64bit_val(wqe, IRDMA_BYTE_32, 4577 (FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info->entry[1].cmd) | 4578 FIELD_PREP(IRDMA_CQPSQ_UPESD_ENTRY_VALID, 1))); 4579 4580 set_64bit_val(wqe, IRDMA_BYTE_40, info->entry[1].data); 4581 /* fallthrough */ 4582 case 1: 4583 set_64bit_val(wqe, IRDMA_BYTE_0, 4584 FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info->entry[0].cmd)); 4585 4586 set_64bit_val(wqe, IRDMA_BYTE_8, info->entry[0].data); 4587 break; 4588 default: 4589 break; 4590 } 4591 4592 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_UPDATE_PE_SDS) | 4593 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) | 4594 FIELD_PREP(IRDMA_CQPSQ_UPESD_ENTRY_COUNT, mem_entries); 4595 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 4596 4597 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 4598 4599 if (mem_entries) 4600 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "UPDATE_PE_SDS WQE Buffer", 4601 (char *)sdbuf->va + offset, mem_entries << 4); 4602 4603 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "UPDATE_PE_SDS WQE", wqe, 4604 IRDMA_CQP_WQE_SIZE * 8); 4605 4606 return 0; 4607 } 4608 4609 /** 4610 * irdma_update_pe_sds - cqp wqe for sd 4611 * @dev: ptr to irdma_dev struct 4612 * @info: sd info for sd's 4613 * @scratch: u64 saved to be used during cqp completion 4614 */ 4615 static int 4616 irdma_update_pe_sds(struct irdma_sc_dev *dev, 4617 struct irdma_update_sds_info *info, u64 scratch) 4618 { 4619 struct irdma_sc_cqp *cqp = dev->cqp; 4620 int ret_code; 4621 4622 ret_code = cqp_sds_wqe_fill(cqp, info, scratch); 4623 if (!ret_code) 4624 irdma_sc_cqp_post_sq(cqp); 4625 4626 return ret_code; 4627 } 4628 4629 /** 4630 * irdma_update_sds_noccq - update sd before ccq created 4631 * @dev: sc device struct 4632 * @info: sd info for sd's 4633 */ 4634 int 4635 irdma_update_sds_noccq(struct irdma_sc_dev *dev, 4636 struct irdma_update_sds_info *info) 4637 { 4638 u32 error, val, tail; 4639 struct irdma_sc_cqp *cqp = dev->cqp; 4640 int ret_code; 4641 4642 ret_code = cqp_sds_wqe_fill(cqp, info, 0); 4643 if (ret_code) 4644 return ret_code; 4645 4646 irdma_get_cqp_reg_info(cqp, &val, &tail, &error); 4647 4648 irdma_sc_cqp_post_sq(cqp); 4649 return irdma_cqp_poll_registers(cqp, tail, 4650 cqp->dev->hw_attrs.max_done_count); 4651 } 4652 4653 /** 4654 * irdma_sc_static_hmc_pages_allocated - cqp wqe to allocate hmc pages 4655 * @cqp: struct for cqp hw 4656 * @scratch: u64 saved to be used during cqp completion 4657 * @hmc_fn_id: hmc function id 4658 * @post_sq: flag for cqp db to ring 4659 * @poll_registers: flag to poll register for cqp completion 4660 */ 4661 int 4662 irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch, 4663 u16 hmc_fn_id, bool post_sq, 4664 bool poll_registers) 4665 { 4666 u64 hdr; 4667 __le64 *wqe; 4668 u32 tail, val, error; 4669 4670 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 4671 if (!wqe) 4672 return -ENOSPC; 4673 4674 set_64bit_val(wqe, IRDMA_BYTE_16, 4675 FIELD_PREP(IRDMA_SHMC_PAGE_ALLOCATED_HMC_FN_ID, hmc_fn_id)); 4676 4677 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, 4678 IRDMA_CQP_OP_SHMC_PAGES_ALLOCATED) | 4679 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); 4680 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 4681 4682 set_64bit_val(wqe, IRDMA_BYTE_24, hdr); 4683 4684 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "SHMC_PAGES_ALLOCATED WQE", 4685 wqe, IRDMA_CQP_WQE_SIZE * 8); 4686 irdma_get_cqp_reg_info(cqp, &val, &tail, &error); 4687 4688 if (post_sq) { 4689 irdma_sc_cqp_post_sq(cqp); 4690 if (poll_registers) 4691 /* check for cqp sq tail update */ 4692 return irdma_cqp_poll_registers(cqp, tail, 4693 cqp->dev->hw_attrs.max_done_count); 4694 else 4695 return irdma_sc_poll_for_cqp_op_done(cqp, 4696 IRDMA_CQP_OP_SHMC_PAGES_ALLOCATED, 4697 NULL); 4698 } 4699 4700 return 0; 4701 } 4702 4703 /** 4704 * irdma_cqp_ring_full - check if cqp ring is full 4705 * @cqp: struct for cqp hw 4706 */ 4707 static bool 4708 irdma_cqp_ring_full(struct irdma_sc_cqp *cqp) 4709 { 4710 return IRDMA_RING_FULL_ERR(cqp->sq_ring); 4711 } 4712 4713 /** 4714 * irdma_est_sd - returns approximate number of SDs for HMC 4715 * @dev: sc device struct 4716 * @hmc_info: hmc structure, size and count for HMC objects 4717 */ 4718 static u32 irdma_est_sd(struct irdma_sc_dev *dev, 4719 struct irdma_hmc_info *hmc_info){ 4720 struct irdma_hmc_obj_info *pble_info; 4721 u64 size = 0; 4722 u64 sd; 4723 int i; 4724 4725 for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++) { 4726 if (i != IRDMA_HMC_IW_PBLE) 4727 size += round_up(hmc_info->hmc_obj[i].cnt * 4728 hmc_info->hmc_obj[i].size, 512); 4729 } 4730 4731 pble_info = &hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE]; 4732 size += round_up(pble_info->cnt * pble_info->size, 512); 4733 if (size & 0x1FFFFF) 4734 sd = (size >> 21) + 1; /* add 1 for remainder */ 4735 else 4736 sd = size >> 21; 4737 if (sd > 0xFFFFFFFF) { 4738 irdma_debug(dev, IRDMA_DEBUG_HMC, "sd overflow[%ld]\n", sd); 4739 sd = 0xFFFFFFFE; 4740 } 4741 4742 return (u32)sd; 4743 } 4744 4745 /** 4746 * irdma_sc_query_rdma_features - query RDMA features and FW ver 4747 * @cqp: struct for cqp hw 4748 * @buf: buffer to hold query info 4749 * @scratch: u64 saved to be used during cqp completion 4750 */ 4751 static int 4752 irdma_sc_query_rdma_features(struct irdma_sc_cqp *cqp, 4753 struct irdma_dma_mem *buf, u64 scratch) 4754 { 4755 __le64 *wqe; 4756 u64 temp; 4757 u32 tail, val, error; 4758 int status; 4759 4760 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); 4761 if (!wqe) 4762 return -ENOSPC; 4763 4764 temp = buf->pa; 4765 set_64bit_val(wqe, IRDMA_BYTE_32, temp); 4766 4767 temp = FIELD_PREP(IRDMA_CQPSQ_QUERY_RDMA_FEATURES_WQEVALID, 4768 cqp->polarity) | 4769 FIELD_PREP(IRDMA_CQPSQ_QUERY_RDMA_FEATURES_BUF_LEN, buf->size) | 4770 FIELD_PREP(IRDMA_CQPSQ_UP_OP, IRDMA_CQP_OP_QUERY_RDMA_FEATURES); 4771 irdma_wmb(); /* make sure WQE is written before valid bit is set */ 4772 4773 set_64bit_val(wqe, IRDMA_BYTE_24, temp); 4774 4775 irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "QUERY RDMA FEATURES", wqe, 4776 IRDMA_CQP_WQE_SIZE * 8); 4777 irdma_get_cqp_reg_info(cqp, &val, &tail, &error); 4778 4779 irdma_sc_cqp_post_sq(cqp); 4780 status = irdma_cqp_poll_registers(cqp, tail, 4781 cqp->dev->hw_attrs.max_done_count); 4782 if (error || status) 4783 status = -EIO; 4784 4785 return status; 4786 } 4787 4788 /** 4789 * irdma_get_rdma_features - get RDMA features 4790 * @dev: sc device struct 4791 */ 4792 int 4793 irdma_get_rdma_features(struct irdma_sc_dev *dev) 4794 { 4795 int ret_code, byte_idx, feat_type, feat_cnt, feat_idx; 4796 struct irdma_dma_mem feat_buf; 4797 u64 temp; 4798 4799 feat_buf.size = IRDMA_FEATURE_BUF_SIZE; 4800 feat_buf.va = irdma_allocate_dma_mem(dev->hw, &feat_buf, feat_buf.size, 4801 IRDMA_FEATURE_BUF_ALIGNMENT); 4802 if (!feat_buf.va) 4803 return -ENOMEM; 4804 4805 ret_code = irdma_sc_query_rdma_features(dev->cqp, &feat_buf, 0); 4806 if (ret_code) 4807 goto exit; 4808 4809 get_64bit_val(feat_buf.va, IRDMA_BYTE_0, &temp); 4810 feat_cnt = (u16)FIELD_GET(IRDMA_FEATURE_CNT, temp); 4811 if (feat_cnt < IRDMA_MIN_FEATURES) { 4812 ret_code = -EINVAL; 4813 goto exit; 4814 } else if (feat_cnt > IRDMA_MAX_FEATURES) { 4815 irdma_debug(dev, IRDMA_DEBUG_DEV, 4816 "feature buf size insufficient, retrying with larger buffer\n"); 4817 irdma_free_dma_mem(dev->hw, &feat_buf); 4818 feat_buf.size = 8 * feat_cnt; 4819 feat_buf.va = irdma_allocate_dma_mem(dev->hw, &feat_buf, 4820 feat_buf.size, 4821 IRDMA_FEATURE_BUF_ALIGNMENT); 4822 if (!feat_buf.va) 4823 return -ENOMEM; 4824 4825 ret_code = irdma_sc_query_rdma_features(dev->cqp, &feat_buf, 0); 4826 if (ret_code) 4827 goto exit; 4828 4829 get_64bit_val(feat_buf.va, IRDMA_BYTE_0, &temp); 4830 feat_cnt = (u16)FIELD_GET(IRDMA_FEATURE_CNT, temp); 4831 if (feat_cnt < IRDMA_MIN_FEATURES) { 4832 ret_code = -EINVAL; 4833 goto exit; 4834 } 4835 } 4836 4837 irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "QUERY RDMA FEATURES", feat_buf.va, 4838 feat_cnt * 8); 4839 4840 for (byte_idx = 0, feat_idx = 0; feat_idx < min(feat_cnt, IRDMA_MAX_FEATURES); 4841 feat_idx++, byte_idx += 8) { 4842 get_64bit_val(feat_buf.va, byte_idx, &temp); 4843 feat_type = FIELD_GET(IRDMA_FEATURE_TYPE, temp); 4844 dev->feature_info[feat_type] = temp; 4845 } 4846 exit: 4847 irdma_free_dma_mem(dev->hw, &feat_buf); 4848 return ret_code; 4849 } 4850 4851 static u32 irdma_q1_cnt(struct irdma_sc_dev *dev, 4852 struct irdma_hmc_info *hmc_info, u32 qpwanted){ 4853 u32 q1_cnt; 4854 4855 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) { 4856 q1_cnt = roundup_pow_of_two(dev->hw_attrs.max_hw_ird * 2 * qpwanted); 4857 } else { 4858 if (dev->cqp->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY) 4859 q1_cnt = roundup_pow_of_two(dev->hw_attrs.max_hw_ird * 2 * qpwanted + 512); 4860 else 4861 q1_cnt = dev->hw_attrs.max_hw_ird * 2 * qpwanted; 4862 } 4863 4864 return q1_cnt; 4865 } 4866 4867 static void 4868 cfg_fpm_value_gen_1(struct irdma_sc_dev *dev, 4869 struct irdma_hmc_info *hmc_info, u32 qpwanted) 4870 { 4871 hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt = roundup_pow_of_two(qpwanted * dev->hw_attrs.max_hw_wqes); 4872 } 4873 4874 static void 4875 cfg_fpm_value_gen_2(struct irdma_sc_dev *dev, 4876 struct irdma_hmc_info *hmc_info, u32 qpwanted) 4877 { 4878 struct irdma_hmc_fpm_misc *hmc_fpm_misc = &dev->hmc_fpm_misc; 4879 4880 hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt = 4881 4 * hmc_fpm_misc->xf_block_size * qpwanted; 4882 4883 hmc_info->hmc_obj[IRDMA_HMC_IW_HDR].cnt = qpwanted; 4884 4885 if (hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].max_cnt) 4886 hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].cnt = 32 * qpwanted; 4887 if (hmc_info->hmc_obj[IRDMA_HMC_IW_RRFFL].max_cnt) 4888 hmc_info->hmc_obj[IRDMA_HMC_IW_RRFFL].cnt = 4889 hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].cnt / 4890 hmc_fpm_misc->rrf_block_size; 4891 if (dev->cqp->protocol_used == IRDMA_IWARP_PROTOCOL_ONLY) { 4892 if (hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].max_cnt) 4893 hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].cnt = 32 * qpwanted; 4894 if (hmc_info->hmc_obj[IRDMA_HMC_IW_OOISCFFL].max_cnt) 4895 hmc_info->hmc_obj[IRDMA_HMC_IW_OOISCFFL].cnt = 4896 hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].cnt / 4897 hmc_fpm_misc->ooiscf_block_size; 4898 } 4899 } 4900 4901 /** 4902 * irdma_cfg_fpm_val - configure HMC objects 4903 * @dev: sc device struct 4904 * @qp_count: desired qp count 4905 */ 4906 int 4907 irdma_cfg_fpm_val(struct irdma_sc_dev *dev, u32 qp_count) 4908 { 4909 struct irdma_virt_mem virt_mem; 4910 u32 i, mem_size; 4911 u32 qpwanted, mrwanted, pblewanted; 4912 u32 powerof2, hte; 4913 u32 sd_needed; 4914 u32 sd_diff; 4915 u32 loop_count = 0; 4916 struct irdma_hmc_info *hmc_info; 4917 struct irdma_hmc_fpm_misc *hmc_fpm_misc; 4918 int ret_code = 0; 4919 u32 max_sds; 4920 4921 hmc_info = dev->hmc_info; 4922 hmc_fpm_misc = &dev->hmc_fpm_misc; 4923 ret_code = irdma_sc_init_iw_hmc(dev, dev->hmc_fn_id); 4924 if (ret_code) { 4925 irdma_debug(dev, IRDMA_DEBUG_HMC, 4926 "irdma_sc_init_iw_hmc returned error_code = %d\n", 4927 ret_code); 4928 return ret_code; 4929 } 4930 4931 max_sds = hmc_fpm_misc->max_sds; 4932 4933 for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++) 4934 hmc_info->hmc_obj[i].cnt = hmc_info->hmc_obj[i].max_cnt; 4935 4936 sd_needed = irdma_est_sd(dev, hmc_info); 4937 irdma_debug(dev, IRDMA_DEBUG_HMC, "sd count %d where max sd is %d\n", 4938 hmc_info->sd_table.sd_cnt, max_sds); 4939 4940 qpwanted = min(qp_count, hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt); 4941 4942 powerof2 = 1; 4943 while (powerof2 <= qpwanted) 4944 powerof2 *= 2; 4945 powerof2 /= 2; 4946 qpwanted = powerof2; 4947 4948 mrwanted = hmc_info->hmc_obj[IRDMA_HMC_IW_MR].max_cnt; 4949 pblewanted = hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].max_cnt; 4950 4951 irdma_debug(dev, IRDMA_DEBUG_HMC, 4952 "req_qp=%d max_sd=%d, max_qp = %d, max_cq=%d, max_mr=%d, max_pble=%d, mc=%d, av=%d\n", 4953 qp_count, max_sds, 4954 hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt, 4955 hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].max_cnt, 4956 hmc_info->hmc_obj[IRDMA_HMC_IW_MR].max_cnt, 4957 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].max_cnt, 4958 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].max_cnt, 4959 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].max_cnt); 4960 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt = 4961 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].max_cnt; 4962 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt = 4963 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].max_cnt; 4964 hmc_info->hmc_obj[IRDMA_HMC_IW_ARP].cnt = 4965 hmc_info->hmc_obj[IRDMA_HMC_IW_ARP].max_cnt; 4966 if (dev->hw_attrs.uk_attrs.hw_rev <= IRDMA_GEN_2) 4967 hmc_info->hmc_obj[IRDMA_HMC_IW_APBVT_ENTRY].cnt = 1; 4968 4969 while (irdma_q1_cnt(dev, hmc_info, qpwanted) > hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].max_cnt) 4970 qpwanted /= 2; 4971 4972 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) { 4973 cfg_fpm_value_gen_1(dev, hmc_info, qpwanted); 4974 while (hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt > hmc_info->hmc_obj[IRDMA_HMC_IW_XF].max_cnt) { 4975 qpwanted /= 2; 4976 cfg_fpm_value_gen_1(dev, hmc_info, qpwanted); 4977 } 4978 } 4979 4980 do { 4981 ++loop_count; 4982 hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt = qpwanted; 4983 hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt = 4984 min(2 * qpwanted, hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt); 4985 hmc_info->hmc_obj[IRDMA_HMC_IW_RESERVED].cnt = 0; /* Reserved */ 4986 hmc_info->hmc_obj[IRDMA_HMC_IW_MR].cnt = mrwanted; 4987 4988 hte = round_up(qpwanted + hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt, 512); 4989 powerof2 = 1; 4990 while (powerof2 < hte) 4991 powerof2 *= 2; 4992 hmc_info->hmc_obj[IRDMA_HMC_IW_HTE].cnt = 4993 powerof2 * hmc_fpm_misc->ht_multiplier; 4994 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 4995 cfg_fpm_value_gen_1(dev, hmc_info, qpwanted); 4996 else 4997 cfg_fpm_value_gen_2(dev, hmc_info, qpwanted); 4998 4999 hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].cnt = irdma_q1_cnt(dev, hmc_info, qpwanted); 5000 hmc_info->hmc_obj[IRDMA_HMC_IW_XFFL].cnt = 5001 hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt / hmc_fpm_misc->xf_block_size; 5002 hmc_info->hmc_obj[IRDMA_HMC_IW_Q1FL].cnt = 5003 hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].cnt / hmc_fpm_misc->q1_block_size; 5004 hmc_info->hmc_obj[IRDMA_HMC_IW_TIMER].cnt = 5005 (round_up(qpwanted, 512) / 512 + 1) * hmc_fpm_misc->timer_bucket; 5006 5007 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt = pblewanted; 5008 sd_needed = irdma_est_sd(dev, hmc_info); 5009 irdma_debug(dev, IRDMA_DEBUG_HMC, 5010 "sd_needed = %d, max_sds=%d, mrwanted=%d, pblewanted=%d qpwanted=%d\n", 5011 sd_needed, max_sds, mrwanted, pblewanted, qpwanted); 5012 5013 /* Do not reduce resources further. All objects fit with max SDs */ 5014 if (sd_needed <= max_sds) 5015 break; 5016 5017 sd_diff = sd_needed - max_sds; 5018 if (sd_diff > 128) { 5019 if (!(loop_count % 2) && qpwanted > 128) { 5020 qpwanted /= 2; 5021 } else { 5022 mrwanted /= 2; 5023 pblewanted /= 2; 5024 } 5025 continue; 5026 } 5027 if (dev->cqp->hmc_profile != IRDMA_HMC_PROFILE_FAVOR_VF && 5028 pblewanted > (512 * FPM_MULTIPLIER * sd_diff)) { 5029 pblewanted -= 256 * FPM_MULTIPLIER * sd_diff; 5030 continue; 5031 } else if (pblewanted > 100 * FPM_MULTIPLIER) { 5032 pblewanted -= 10 * FPM_MULTIPLIER; 5033 } else if (pblewanted > 16 * FPM_MULTIPLIER) { 5034 pblewanted -= FPM_MULTIPLIER; 5035 } else if (qpwanted <= 128) { 5036 if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt > 256) 5037 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt /= 2; 5038 if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt > 256) 5039 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt /= 2; 5040 } 5041 if (mrwanted > FPM_MULTIPLIER) 5042 mrwanted -= FPM_MULTIPLIER; 5043 if (!(loop_count % 10) && qpwanted > 128) { 5044 qpwanted /= 2; 5045 if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt > 256) 5046 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt /= 2; 5047 } 5048 } while (loop_count < 2000); 5049 5050 if (sd_needed > max_sds) { 5051 irdma_debug(dev, IRDMA_DEBUG_HMC, 5052 "cfg_fpm failed loop_cnt=%d, sd_needed=%d, max sd count %d\n", 5053 loop_count, sd_needed, hmc_info->sd_table.sd_cnt); 5054 return -EINVAL; 5055 } 5056 5057 if (loop_count > 1 && sd_needed < max_sds) { 5058 pblewanted += (max_sds - sd_needed) * 256 * FPM_MULTIPLIER; 5059 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt = pblewanted; 5060 sd_needed = irdma_est_sd(dev, hmc_info); 5061 } 5062 5063 irdma_debug(dev, IRDMA_DEBUG_HMC, 5064 "loop_cnt=%d, sd_needed=%d, qpcnt = %d, cqcnt=%d, mrcnt=%d, pblecnt=%d, mc=%d, ah=%d, max sd count %d, first sd index %d\n", 5065 loop_count, sd_needed, 5066 hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt, 5067 hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt, 5068 hmc_info->hmc_obj[IRDMA_HMC_IW_MR].cnt, 5069 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt, 5070 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt, 5071 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt, 5072 hmc_info->sd_table.sd_cnt, hmc_info->first_sd_index); 5073 5074 ret_code = irdma_sc_cfg_iw_fpm(dev, dev->hmc_fn_id); 5075 if (ret_code) { 5076 irdma_debug(dev, IRDMA_DEBUG_HMC, 5077 "cfg_iw_fpm returned error_code[x%08X]\n", 5078 readl(dev->hw_regs[IRDMA_CQPERRCODES])); 5079 return ret_code; 5080 } 5081 5082 mem_size = sizeof(struct irdma_hmc_sd_entry) * 5083 (hmc_info->sd_table.sd_cnt + hmc_info->first_sd_index + 1); 5084 virt_mem.size = mem_size; 5085 virt_mem.va = kzalloc(virt_mem.size, GFP_KERNEL); 5086 if (!virt_mem.va) { 5087 irdma_debug(dev, IRDMA_DEBUG_HMC, 5088 "failed to allocate memory for sd_entry buffer\n"); 5089 return -ENOMEM; 5090 } 5091 hmc_info->sd_table.sd_entry = virt_mem.va; 5092 5093 return ret_code; 5094 } 5095 5096 /** 5097 * irdma_exec_cqp_cmd - execute cqp cmd when wqe are available 5098 * @dev: rdma device 5099 * @pcmdinfo: cqp command info 5100 */ 5101 static int 5102 irdma_exec_cqp_cmd(struct irdma_sc_dev *dev, 5103 struct cqp_cmds_info *pcmdinfo) 5104 { 5105 int status; 5106 struct irdma_dma_mem val_mem; 5107 bool alloc = false; 5108 5109 dev->cqp_cmd_stats[pcmdinfo->cqp_cmd]++; 5110 switch (pcmdinfo->cqp_cmd) { 5111 case IRDMA_OP_CEQ_DESTROY: 5112 status = irdma_sc_ceq_destroy(pcmdinfo->in.u.ceq_destroy.ceq, 5113 pcmdinfo->in.u.ceq_destroy.scratch, 5114 pcmdinfo->post_sq); 5115 break; 5116 case IRDMA_OP_AEQ_DESTROY: 5117 status = irdma_sc_aeq_destroy(pcmdinfo->in.u.aeq_destroy.aeq, 5118 pcmdinfo->in.u.aeq_destroy.scratch, 5119 pcmdinfo->post_sq); 5120 break; 5121 case IRDMA_OP_CEQ_CREATE: 5122 status = irdma_sc_ceq_create(pcmdinfo->in.u.ceq_create.ceq, 5123 pcmdinfo->in.u.ceq_create.scratch, 5124 pcmdinfo->post_sq); 5125 break; 5126 case IRDMA_OP_AEQ_CREATE: 5127 status = irdma_sc_aeq_create(pcmdinfo->in.u.aeq_create.aeq, 5128 pcmdinfo->in.u.aeq_create.scratch, 5129 pcmdinfo->post_sq); 5130 break; 5131 case IRDMA_OP_QP_UPLOAD_CONTEXT: 5132 status = irdma_sc_qp_upload_context(pcmdinfo->in.u.qp_upload_context.dev, 5133 &pcmdinfo->in.u.qp_upload_context.info, 5134 pcmdinfo->in.u.qp_upload_context.scratch, 5135 pcmdinfo->post_sq); 5136 break; 5137 case IRDMA_OP_CQ_CREATE: 5138 status = irdma_sc_cq_create(pcmdinfo->in.u.cq_create.cq, 5139 pcmdinfo->in.u.cq_create.scratch, 5140 pcmdinfo->in.u.cq_create.check_overflow, 5141 pcmdinfo->post_sq); 5142 break; 5143 case IRDMA_OP_CQ_MODIFY: 5144 status = irdma_sc_cq_modify(pcmdinfo->in.u.cq_modify.cq, 5145 &pcmdinfo->in.u.cq_modify.info, 5146 pcmdinfo->in.u.cq_modify.scratch, 5147 pcmdinfo->post_sq); 5148 break; 5149 case IRDMA_OP_CQ_DESTROY: 5150 status = irdma_sc_cq_destroy(pcmdinfo->in.u.cq_destroy.cq, 5151 pcmdinfo->in.u.cq_destroy.scratch, 5152 pcmdinfo->post_sq); 5153 break; 5154 case IRDMA_OP_QP_FLUSH_WQES: 5155 status = irdma_sc_qp_flush_wqes(pcmdinfo->in.u.qp_flush_wqes.qp, 5156 &pcmdinfo->in.u.qp_flush_wqes.info, 5157 pcmdinfo->in.u.qp_flush_wqes.scratch, 5158 pcmdinfo->post_sq); 5159 break; 5160 case IRDMA_OP_GEN_AE: 5161 status = irdma_sc_gen_ae(pcmdinfo->in.u.gen_ae.qp, 5162 &pcmdinfo->in.u.gen_ae.info, 5163 pcmdinfo->in.u.gen_ae.scratch, 5164 pcmdinfo->post_sq); 5165 break; 5166 case IRDMA_OP_MANAGE_PUSH_PAGE: 5167 status = irdma_sc_manage_push_page(pcmdinfo->in.u.manage_push_page.cqp, 5168 &pcmdinfo->in.u.manage_push_page.info, 5169 pcmdinfo->in.u.manage_push_page.scratch, 5170 pcmdinfo->post_sq); 5171 break; 5172 case IRDMA_OP_UPDATE_PE_SDS: 5173 status = irdma_update_pe_sds(pcmdinfo->in.u.update_pe_sds.dev, 5174 &pcmdinfo->in.u.update_pe_sds.info, 5175 pcmdinfo->in.u.update_pe_sds.scratch); 5176 break; 5177 case IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE: 5178 /* switch to calling through the call table */ 5179 status = 5180 irdma_sc_manage_hmc_pm_func_table(pcmdinfo->in.u.manage_hmc_pm.dev->cqp, 5181 &pcmdinfo->in.u.manage_hmc_pm.info, 5182 pcmdinfo->in.u.manage_hmc_pm.scratch, 5183 true); 5184 break; 5185 case IRDMA_OP_SUSPEND: 5186 status = irdma_sc_suspend_qp(pcmdinfo->in.u.suspend_resume.cqp, 5187 pcmdinfo->in.u.suspend_resume.qp, 5188 pcmdinfo->in.u.suspend_resume.scratch); 5189 break; 5190 case IRDMA_OP_RESUME: 5191 status = irdma_sc_resume_qp(pcmdinfo->in.u.suspend_resume.cqp, 5192 pcmdinfo->in.u.suspend_resume.qp, 5193 pcmdinfo->in.u.suspend_resume.scratch); 5194 break; 5195 case IRDMA_OP_QUERY_FPM_VAL: 5196 val_mem.pa = pcmdinfo->in.u.query_fpm_val.fpm_val_pa; 5197 val_mem.va = pcmdinfo->in.u.query_fpm_val.fpm_val_va; 5198 status = irdma_sc_query_fpm_val(pcmdinfo->in.u.query_fpm_val.cqp, 5199 pcmdinfo->in.u.query_fpm_val.scratch, 5200 pcmdinfo->in.u.query_fpm_val.hmc_fn_id, 5201 &val_mem, true, IRDMA_CQP_WAIT_EVENT); 5202 break; 5203 case IRDMA_OP_COMMIT_FPM_VAL: 5204 val_mem.pa = pcmdinfo->in.u.commit_fpm_val.fpm_val_pa; 5205 val_mem.va = pcmdinfo->in.u.commit_fpm_val.fpm_val_va; 5206 status = irdma_sc_commit_fpm_val(pcmdinfo->in.u.commit_fpm_val.cqp, 5207 pcmdinfo->in.u.commit_fpm_val.scratch, 5208 pcmdinfo->in.u.commit_fpm_val.hmc_fn_id, 5209 &val_mem, 5210 true, 5211 IRDMA_CQP_WAIT_EVENT); 5212 break; 5213 case IRDMA_OP_STATS_ALLOCATE: 5214 alloc = true; 5215 /* fallthrough */ 5216 case IRDMA_OP_STATS_FREE: 5217 status = irdma_sc_manage_stats_inst(pcmdinfo->in.u.stats_manage.cqp, 5218 &pcmdinfo->in.u.stats_manage.info, 5219 alloc, 5220 pcmdinfo->in.u.stats_manage.scratch); 5221 break; 5222 case IRDMA_OP_STATS_GATHER: 5223 status = irdma_sc_gather_stats(pcmdinfo->in.u.stats_gather.cqp, 5224 &pcmdinfo->in.u.stats_gather.info, 5225 pcmdinfo->in.u.stats_gather.scratch); 5226 break; 5227 case IRDMA_OP_WS_MODIFY_NODE: 5228 status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp, 5229 &pcmdinfo->in.u.ws_node.info, 5230 IRDMA_MODIFY_NODE, 5231 pcmdinfo->in.u.ws_node.scratch); 5232 break; 5233 case IRDMA_OP_WS_DELETE_NODE: 5234 status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp, 5235 &pcmdinfo->in.u.ws_node.info, 5236 IRDMA_DEL_NODE, 5237 pcmdinfo->in.u.ws_node.scratch); 5238 break; 5239 case IRDMA_OP_WS_ADD_NODE: 5240 status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp, 5241 &pcmdinfo->in.u.ws_node.info, 5242 IRDMA_ADD_NODE, 5243 pcmdinfo->in.u.ws_node.scratch); 5244 break; 5245 case IRDMA_OP_SET_UP_MAP: 5246 status = irdma_sc_set_up_map(pcmdinfo->in.u.up_map.cqp, 5247 &pcmdinfo->in.u.up_map.info, 5248 pcmdinfo->in.u.up_map.scratch); 5249 break; 5250 case IRDMA_OP_QUERY_RDMA_FEATURES: 5251 status = irdma_sc_query_rdma_features(pcmdinfo->in.u.query_rdma.cqp, 5252 &pcmdinfo->in.u.query_rdma.query_buff_mem, 5253 pcmdinfo->in.u.query_rdma.scratch); 5254 break; 5255 case IRDMA_OP_DELETE_ARP_CACHE_ENTRY: 5256 status = irdma_sc_del_arp_cache_entry(pcmdinfo->in.u.del_arp_cache_entry.cqp, 5257 pcmdinfo->in.u.del_arp_cache_entry.scratch, 5258 pcmdinfo->in.u.del_arp_cache_entry.arp_index, 5259 pcmdinfo->post_sq); 5260 break; 5261 case IRDMA_OP_MANAGE_APBVT_ENTRY: 5262 status = irdma_sc_manage_apbvt_entry(pcmdinfo->in.u.manage_apbvt_entry.cqp, 5263 &pcmdinfo->in.u.manage_apbvt_entry.info, 5264 pcmdinfo->in.u.manage_apbvt_entry.scratch, 5265 pcmdinfo->post_sq); 5266 break; 5267 case IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY: 5268 status = irdma_sc_manage_qhash_table_entry(pcmdinfo->in.u.manage_qhash_table_entry.cqp, 5269 &pcmdinfo->in.u.manage_qhash_table_entry.info, 5270 pcmdinfo->in.u.manage_qhash_table_entry.scratch, 5271 pcmdinfo->post_sq); 5272 break; 5273 case IRDMA_OP_QP_MODIFY: 5274 status = irdma_sc_qp_modify(pcmdinfo->in.u.qp_modify.qp, 5275 &pcmdinfo->in.u.qp_modify.info, 5276 pcmdinfo->in.u.qp_modify.scratch, 5277 pcmdinfo->post_sq); 5278 break; 5279 case IRDMA_OP_QP_CREATE: 5280 status = irdma_sc_qp_create(pcmdinfo->in.u.qp_create.qp, 5281 &pcmdinfo->in.u.qp_create.info, 5282 pcmdinfo->in.u.qp_create.scratch, 5283 pcmdinfo->post_sq); 5284 break; 5285 case IRDMA_OP_QP_DESTROY: 5286 status = irdma_sc_qp_destroy(pcmdinfo->in.u.qp_destroy.qp, 5287 pcmdinfo->in.u.qp_destroy.scratch, 5288 pcmdinfo->in.u.qp_destroy.remove_hash_idx, 5289 pcmdinfo->in.u.qp_destroy.ignore_mw_bnd, 5290 pcmdinfo->post_sq); 5291 break; 5292 case IRDMA_OP_ALLOC_STAG: 5293 status = irdma_sc_alloc_stag(pcmdinfo->in.u.alloc_stag.dev, 5294 &pcmdinfo->in.u.alloc_stag.info, 5295 pcmdinfo->in.u.alloc_stag.scratch, 5296 pcmdinfo->post_sq); 5297 break; 5298 case IRDMA_OP_MR_REG_NON_SHARED: 5299 status = irdma_sc_mr_reg_non_shared(pcmdinfo->in.u.mr_reg_non_shared.dev, 5300 &pcmdinfo->in.u.mr_reg_non_shared.info, 5301 pcmdinfo->in.u.mr_reg_non_shared.scratch, 5302 pcmdinfo->post_sq); 5303 break; 5304 case IRDMA_OP_DEALLOC_STAG: 5305 status = irdma_sc_dealloc_stag(pcmdinfo->in.u.dealloc_stag.dev, 5306 &pcmdinfo->in.u.dealloc_stag.info, 5307 pcmdinfo->in.u.dealloc_stag.scratch, 5308 pcmdinfo->post_sq); 5309 break; 5310 case IRDMA_OP_MW_ALLOC: 5311 status = irdma_sc_mw_alloc(pcmdinfo->in.u.mw_alloc.dev, 5312 &pcmdinfo->in.u.mw_alloc.info, 5313 pcmdinfo->in.u.mw_alloc.scratch, 5314 pcmdinfo->post_sq); 5315 break; 5316 case IRDMA_OP_ADD_ARP_CACHE_ENTRY: 5317 status = irdma_sc_add_arp_cache_entry(pcmdinfo->in.u.add_arp_cache_entry.cqp, 5318 &pcmdinfo->in.u.add_arp_cache_entry.info, 5319 pcmdinfo->in.u.add_arp_cache_entry.scratch, 5320 pcmdinfo->post_sq); 5321 break; 5322 case IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY: 5323 status = irdma_sc_alloc_local_mac_entry(pcmdinfo->in.u.alloc_local_mac_entry.cqp, 5324 pcmdinfo->in.u.alloc_local_mac_entry.scratch, 5325 pcmdinfo->post_sq); 5326 break; 5327 case IRDMA_OP_ADD_LOCAL_MAC_ENTRY: 5328 status = irdma_sc_add_local_mac_entry(pcmdinfo->in.u.add_local_mac_entry.cqp, 5329 &pcmdinfo->in.u.add_local_mac_entry.info, 5330 pcmdinfo->in.u.add_local_mac_entry.scratch, 5331 pcmdinfo->post_sq); 5332 break; 5333 case IRDMA_OP_DELETE_LOCAL_MAC_ENTRY: 5334 status = irdma_sc_del_local_mac_entry(pcmdinfo->in.u.del_local_mac_entry.cqp, 5335 pcmdinfo->in.u.del_local_mac_entry.scratch, 5336 pcmdinfo->in.u.del_local_mac_entry.entry_idx, 5337 pcmdinfo->in.u.del_local_mac_entry.ignore_ref_count, 5338 pcmdinfo->post_sq); 5339 break; 5340 case IRDMA_OP_AH_CREATE: 5341 status = irdma_sc_create_ah(pcmdinfo->in.u.ah_create.cqp, 5342 &pcmdinfo->in.u.ah_create.info, 5343 pcmdinfo->in.u.ah_create.scratch); 5344 break; 5345 case IRDMA_OP_AH_DESTROY: 5346 status = irdma_sc_destroy_ah(pcmdinfo->in.u.ah_destroy.cqp, 5347 &pcmdinfo->in.u.ah_destroy.info, 5348 pcmdinfo->in.u.ah_destroy.scratch); 5349 break; 5350 case IRDMA_OP_MC_CREATE: 5351 status = irdma_sc_create_mcast_grp(pcmdinfo->in.u.mc_create.cqp, 5352 &pcmdinfo->in.u.mc_create.info, 5353 pcmdinfo->in.u.mc_create.scratch); 5354 break; 5355 case IRDMA_OP_MC_DESTROY: 5356 status = irdma_sc_destroy_mcast_grp(pcmdinfo->in.u.mc_destroy.cqp, 5357 &pcmdinfo->in.u.mc_destroy.info, 5358 pcmdinfo->in.u.mc_destroy.scratch); 5359 break; 5360 case IRDMA_OP_MC_MODIFY: 5361 status = irdma_sc_modify_mcast_grp(pcmdinfo->in.u.mc_modify.cqp, 5362 &pcmdinfo->in.u.mc_modify.info, 5363 pcmdinfo->in.u.mc_modify.scratch); 5364 break; 5365 default: 5366 status = -EOPNOTSUPP; 5367 break; 5368 } 5369 5370 return status; 5371 } 5372 5373 /** 5374 * irdma_process_cqp_cmd - process all cqp commands 5375 * @dev: sc device struct 5376 * @pcmdinfo: cqp command info 5377 */ 5378 int 5379 irdma_process_cqp_cmd(struct irdma_sc_dev *dev, 5380 struct cqp_cmds_info *pcmdinfo) 5381 { 5382 int status = 0; 5383 unsigned long flags; 5384 5385 spin_lock_irqsave(&dev->cqp_lock, flags); 5386 if (list_empty(&dev->cqp_cmd_head) && !irdma_cqp_ring_full(dev->cqp)) 5387 status = irdma_exec_cqp_cmd(dev, pcmdinfo); 5388 else 5389 list_add_tail(&pcmdinfo->cqp_cmd_entry, &dev->cqp_cmd_head); 5390 spin_unlock_irqrestore(&dev->cqp_lock, flags); 5391 return status; 5392 } 5393 5394 /** 5395 * irdma_process_bh - called from tasklet for cqp list 5396 * @dev: sc device struct 5397 */ 5398 int 5399 irdma_process_bh(struct irdma_sc_dev *dev) 5400 { 5401 int status = 0; 5402 struct cqp_cmds_info *pcmdinfo; 5403 unsigned long flags; 5404 5405 spin_lock_irqsave(&dev->cqp_lock, flags); 5406 while (!list_empty(&dev->cqp_cmd_head) && 5407 !irdma_cqp_ring_full(dev->cqp)) { 5408 pcmdinfo = (struct cqp_cmds_info *)irdma_remove_cqp_head(dev); 5409 status = irdma_exec_cqp_cmd(dev, pcmdinfo); 5410 if (status) 5411 break; 5412 } 5413 spin_unlock_irqrestore(&dev->cqp_lock, flags); 5414 return status; 5415 } 5416 5417 /** 5418 * irdma_cfg_aeq- Configure AEQ interrupt 5419 * @dev: pointer to the device structure 5420 * @idx: vector index 5421 * @enable: True to enable, False disables 5422 */ 5423 void 5424 irdma_cfg_aeq(struct irdma_sc_dev *dev, u32 idx, bool enable) 5425 { 5426 u32 reg_val; 5427 5428 reg_val = FIELD_PREP(IRDMA_PFINT_AEQCTL_CAUSE_ENA, enable) | 5429 FIELD_PREP(IRDMA_PFINT_AEQCTL_MSIX_INDX, idx) | 5430 FIELD_PREP(IRDMA_PFINT_AEQCTL_ITR_INDX, IRDMA_IDX_NOITR); 5431 5432 writel(reg_val, dev->hw_regs[IRDMA_PFINT_AEQCTL]); 5433 } 5434 5435 /** 5436 * sc_vsi_update_stats - Update statistics 5437 * @vsi: sc_vsi instance to update 5438 */ 5439 void 5440 sc_vsi_update_stats(struct irdma_sc_vsi *vsi) 5441 { 5442 struct irdma_dev_hw_stats *hw_stats = &vsi->pestat->hw_stats; 5443 struct irdma_gather_stats *gather_stats = 5444 vsi->pestat->gather_info.gather_stats_va; 5445 struct irdma_gather_stats *last_gather_stats = 5446 vsi->pestat->gather_info.last_gather_stats_va; 5447 const struct irdma_hw_stat_map *map = vsi->dev->hw_stats_map; 5448 u16 max_stat_idx = vsi->dev->hw_attrs.max_stat_idx; 5449 5450 irdma_update_stats(hw_stats, gather_stats, last_gather_stats, 5451 map, max_stat_idx); 5452 } 5453 5454 /** 5455 * irdma_wait_pe_ready - Check if firmware is ready 5456 * @dev: provides access to registers 5457 */ 5458 static int 5459 irdma_wait_pe_ready(struct irdma_sc_dev *dev) 5460 { 5461 u32 statuscpu0; 5462 u32 statuscpu1; 5463 u32 statuscpu2; 5464 u32 retrycount = 0; 5465 5466 do { 5467 statuscpu0 = readl(dev->hw_regs[IRDMA_GLPE_CPUSTATUS0]); 5468 statuscpu1 = readl(dev->hw_regs[IRDMA_GLPE_CPUSTATUS1]); 5469 statuscpu2 = readl(dev->hw_regs[IRDMA_GLPE_CPUSTATUS2]); 5470 if (statuscpu0 == 0x80 && statuscpu1 == 0x80 && 5471 statuscpu2 == 0x80) 5472 return 0; 5473 mdelay(1000); 5474 } while (retrycount++ < dev->hw_attrs.max_pe_ready_count); 5475 return -1; 5476 } 5477 5478 static inline void 5479 irdma_sc_init_hw(struct irdma_sc_dev *dev) 5480 { 5481 switch (dev->hw_attrs.uk_attrs.hw_rev) { 5482 case IRDMA_GEN_2: 5483 icrdma_init_hw(dev); 5484 break; 5485 } 5486 } 5487 5488 /** 5489 * irdma_sc_dev_init - Initialize control part of device 5490 * @dev: Device pointer 5491 * @info: Device init info 5492 */ 5493 int 5494 irdma_sc_dev_init(struct irdma_sc_dev *dev, struct irdma_device_init_info *info) 5495 { 5496 u32 val; 5497 int ret_code = 0; 5498 u8 db_size; 5499 5500 INIT_LIST_HEAD(&dev->cqp_cmd_head); /* for CQP command backlog */ 5501 mutex_init(&dev->ws_mutex); 5502 dev->debug_mask = info->debug_mask; 5503 dev->hmc_fn_id = info->hmc_fn_id; 5504 dev->fpm_query_buf_pa = info->fpm_query_buf_pa; 5505 dev->fpm_query_buf = info->fpm_query_buf; 5506 dev->fpm_commit_buf_pa = info->fpm_commit_buf_pa; 5507 dev->fpm_commit_buf = info->fpm_commit_buf; 5508 dev->hw = info->hw; 5509 dev->hw->hw_addr = info->bar0; 5510 /* Setup the hardware limits, hmc may limit further */ 5511 dev->hw_attrs.min_hw_qp_id = IRDMA_MIN_IW_QP_ID; 5512 dev->hw_attrs.min_hw_aeq_size = IRDMA_MIN_AEQ_ENTRIES; 5513 dev->hw_attrs.max_hw_aeq_size = IRDMA_MAX_AEQ_ENTRIES; 5514 dev->hw_attrs.min_hw_ceq_size = IRDMA_MIN_CEQ_ENTRIES; 5515 dev->hw_attrs.max_hw_ceq_size = IRDMA_MAX_CEQ_ENTRIES; 5516 dev->hw_attrs.uk_attrs.min_hw_cq_size = IRDMA_MIN_CQ_SIZE; 5517 dev->hw_attrs.uk_attrs.max_hw_cq_size = IRDMA_MAX_CQ_SIZE; 5518 dev->hw_attrs.max_hw_outbound_msg_size = IRDMA_MAX_OUTBOUND_MSG_SIZE; 5519 dev->hw_attrs.max_mr_size = IRDMA_MAX_MR_SIZE; 5520 dev->hw_attrs.max_hw_inbound_msg_size = IRDMA_MAX_INBOUND_MSG_SIZE; 5521 dev->hw_attrs.uk_attrs.max_hw_inline = IRDMA_MAX_INLINE_DATA_SIZE; 5522 dev->hw_attrs.max_hw_wqes = IRDMA_MAX_WQ_ENTRIES; 5523 dev->hw_attrs.max_qp_wr = IRDMA_MAX_QP_WRS(IRDMA_MAX_QUANTA_PER_WR); 5524 5525 dev->hw_attrs.uk_attrs.max_hw_rq_quanta = IRDMA_QP_SW_MAX_RQ_QUANTA; 5526 dev->hw_attrs.uk_attrs.max_hw_wq_quanta = IRDMA_QP_SW_MAX_WQ_QUANTA; 5527 dev->hw_attrs.max_hw_pds = IRDMA_MAX_PDS; 5528 dev->hw_attrs.max_hw_ena_vf_count = IRDMA_MAX_PE_ENA_VF_COUNT; 5529 5530 dev->hw_attrs.max_pe_ready_count = 14; 5531 dev->hw_attrs.max_done_count = IRDMA_DONE_COUNT; 5532 dev->hw_attrs.max_sleep_count = IRDMA_SLEEP_COUNT; 5533 dev->hw_attrs.max_cqp_compl_wait_time_ms = CQP_COMPL_WAIT_TIME_MS; 5534 5535 irdma_sc_init_hw(dev); 5536 5537 if (irdma_wait_pe_ready(dev)) 5538 return -ETIMEDOUT; 5539 5540 val = readl(dev->hw_regs[IRDMA_GLPCI_LBARCTRL]); 5541 db_size = (u8)FIELD_GET(IRDMA_GLPCI_LBARCTRL_PE_DB_SIZE, val); 5542 if (db_size != IRDMA_PE_DB_SIZE_4M && db_size != IRDMA_PE_DB_SIZE_8M) { 5543 irdma_debug(dev, IRDMA_DEBUG_DEV, 5544 "RDMA PE doorbell is not enabled in CSR val 0x%x db_size=%d\n", 5545 val, db_size); 5546 return -ENODEV; 5547 } 5548 5549 return ret_code; 5550 } 5551 5552 /** 5553 * irdma_stat_val - Extract HW counter value from statistics buffer 5554 * @stats_val: pointer to statistics buffer 5555 * @byteoff: byte offset of counter value in the buffer (8B-aligned) 5556 * @bitoff: bit offset of counter value within 8B entry 5557 * @bitmask: maximum counter value (e.g. 0xffffff for 24-bit counter) 5558 */ 5559 static inline u64 irdma_stat_val(const u64 *stats_val, u16 byteoff, 5560 u8 bitoff, u64 bitmask){ 5561 u16 idx = byteoff / sizeof(*stats_val); 5562 5563 return (stats_val[idx] >> bitoff) & bitmask; 5564 } 5565 5566 /** 5567 * irdma_stat_delta - Calculate counter delta 5568 * @new_val: updated counter value 5569 * @old_val: last counter value 5570 * @max_val: maximum counter value (e.g. 0xffffff for 24-bit counter) 5571 */ 5572 static inline u64 irdma_stat_delta(u64 new_val, u64 old_val, u64 max_val) { 5573 if (new_val >= old_val) 5574 return new_val - old_val; 5575 else 5576 /* roll-over case */ 5577 return max_val - old_val + new_val + 1; 5578 } 5579 5580 /** 5581 * irdma_update_stats - Update statistics 5582 * @hw_stats: hw_stats instance to update 5583 * @gather_stats: updated stat counters 5584 * @last_gather_stats: last stat counters 5585 * @map: HW stat map (hw_stats => gather_stats) 5586 * @max_stat_idx: number of HW stats 5587 */ 5588 void 5589 irdma_update_stats(struct irdma_dev_hw_stats *hw_stats, 5590 struct irdma_gather_stats *gather_stats, 5591 struct irdma_gather_stats *last_gather_stats, 5592 const struct irdma_hw_stat_map *map, 5593 u16 max_stat_idx) 5594 { 5595 u64 *stats_val = hw_stats->stats_val; 5596 u16 i; 5597 5598 for (i = 0; i < max_stat_idx; i++) { 5599 u64 new_val = irdma_stat_val(gather_stats->val, 5600 map[i].byteoff, map[i].bitoff, 5601 map[i].bitmask); 5602 u64 last_val = irdma_stat_val(last_gather_stats->val, 5603 map[i].byteoff, map[i].bitoff, 5604 map[i].bitmask); 5605 5606 stats_val[i] += irdma_stat_delta(new_val, last_val, 5607 map[i].bitmask); 5608 } 5609 5610 irdma_memcpy(last_gather_stats, gather_stats, 5611 sizeof(*last_gather_stats)); 5612 } 5613