1 /* QLogic qed NIC Driver 2 * Copyright (c) 2015-2017 QLogic Corporation 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and /or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <linux/types.h> 33 #include <asm/byteorder.h> 34 #include <linux/bitops.h> 35 #include <linux/delay.h> 36 #include <linux/dma-mapping.h> 37 #include <linux/errno.h> 38 #include <linux/etherdevice.h> 39 #include <linux/if_ether.h> 40 #include <linux/if_vlan.h> 41 #include <linux/io.h> 42 #include <linux/ip.h> 43 #include <linux/ipv6.h> 44 #include <linux/kernel.h> 45 #include <linux/list.h> 46 #include <linux/module.h> 47 #include <linux/mutex.h> 48 #include <linux/pci.h> 49 #include <linux/slab.h> 50 #include <linux/spinlock.h> 51 #include <linux/string.h> 52 #include <linux/tcp.h> 53 #include <linux/bitops.h> 54 #include <linux/qed/qed_roce_if.h> 55 #include <linux/qed/qed_roce_if.h> 56 #include "qed.h" 57 #include "qed_cxt.h" 58 #include "qed_hsi.h" 59 #include "qed_hw.h" 60 #include "qed_init_ops.h" 61 #include "qed_int.h" 62 #include "qed_ll2.h" 63 #include "qed_mcp.h" 64 #include "qed_reg_addr.h" 65 #include "qed_sp.h" 66 #include "qed_roce.h" 67 #include "qed_ll2.h" 68 69 void qed_async_roce_event(struct qed_hwfn *p_hwfn, 70 struct event_ring_entry *p_eqe) 71 { 72 struct qed_rdma_info *p_rdma_info = p_hwfn->p_rdma_info; 73 74 p_rdma_info->events.affiliated_event(p_rdma_info->events.context, 75 p_eqe->opcode, &p_eqe->data); 76 } 77 78 static int qed_rdma_bmap_alloc(struct qed_hwfn *p_hwfn, 79 struct qed_bmap *bmap, u32 max_count) 80 { 81 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "max_count = %08x\n", max_count); 82 83 bmap->max_count = max_count; 84 85 bmap->bitmap = kzalloc(BITS_TO_LONGS(max_count) * sizeof(long), 86 GFP_KERNEL); 87 if (!bmap->bitmap) { 88 DP_NOTICE(p_hwfn, 89 "qed bmap alloc failed: cannot allocate memory (bitmap)\n"); 90 return -ENOMEM; 91 } 92 93 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Allocated bitmap %p\n", 94 bmap->bitmap); 95 return 0; 96 } 97 98 static int qed_rdma_bmap_alloc_id(struct qed_hwfn *p_hwfn, 99 struct qed_bmap *bmap, u32 *id_num) 100 { 101 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "bmap = %p\n", bmap); 102 103 *id_num = find_first_zero_bit(bmap->bitmap, bmap->max_count); 104 105 if (*id_num >= bmap->max_count) { 106 DP_NOTICE(p_hwfn, "no id available max_count=%d\n", 107 bmap->max_count); 108 return -EINVAL; 109 } 110 111 __set_bit(*id_num, bmap->bitmap); 112 113 return 0; 114 } 115 116 static void qed_bmap_release_id(struct qed_hwfn *p_hwfn, 117 struct qed_bmap *bmap, u32 id_num) 118 { 119 bool b_acquired; 120 121 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "id_num = %08x", id_num); 122 if (id_num >= bmap->max_count) 123 return; 124 125 b_acquired = test_and_clear_bit(id_num, bmap->bitmap); 126 if (!b_acquired) { 127 DP_NOTICE(p_hwfn, "ID %d already released\n", id_num); 128 return; 129 } 130 } 131 132 static u32 qed_rdma_get_sb_id(void *p_hwfn, u32 rel_sb_id) 133 { 134 /* First sb id for RoCE is after all the l2 sb */ 135 return FEAT_NUM((struct qed_hwfn *)p_hwfn, QED_PF_L2_QUE) + rel_sb_id; 136 } 137 138 static int qed_rdma_alloc(struct qed_hwfn *p_hwfn, 139 struct qed_ptt *p_ptt, 140 struct qed_rdma_start_in_params *params) 141 { 142 struct qed_rdma_info *p_rdma_info; 143 u32 num_cons, num_tasks; 144 int rc = -ENOMEM; 145 146 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Allocating RDMA\n"); 147 148 /* Allocate a struct with current pf rdma info */ 149 p_rdma_info = kzalloc(sizeof(*p_rdma_info), GFP_KERNEL); 150 if (!p_rdma_info) { 151 DP_NOTICE(p_hwfn, 152 "qed rdma alloc failed: cannot allocate memory (rdma info). rc = %d\n", 153 rc); 154 return rc; 155 } 156 157 p_hwfn->p_rdma_info = p_rdma_info; 158 p_rdma_info->proto = PROTOCOLID_ROCE; 159 160 num_cons = qed_cxt_get_proto_cid_count(p_hwfn, p_rdma_info->proto, 161 NULL); 162 163 p_rdma_info->num_qps = num_cons / 2; 164 165 num_tasks = qed_cxt_get_proto_tid_count(p_hwfn, PROTOCOLID_ROCE); 166 167 /* Each MR uses a single task */ 168 p_rdma_info->num_mrs = num_tasks; 169 170 /* Queue zone lines are shared between RoCE and L2 in such a way that 171 * they can be used by each without obstructing the other. 172 */ 173 p_rdma_info->queue_zone_base = (u16)FEAT_NUM(p_hwfn, QED_L2_QUEUE); 174 175 /* Allocate a struct with device params and fill it */ 176 p_rdma_info->dev = kzalloc(sizeof(*p_rdma_info->dev), GFP_KERNEL); 177 if (!p_rdma_info->dev) { 178 DP_NOTICE(p_hwfn, 179 "qed rdma alloc failed: cannot allocate memory (rdma info dev). rc = %d\n", 180 rc); 181 goto free_rdma_info; 182 } 183 184 /* Allocate a struct with port params and fill it */ 185 p_rdma_info->port = kzalloc(sizeof(*p_rdma_info->port), GFP_KERNEL); 186 if (!p_rdma_info->port) { 187 DP_NOTICE(p_hwfn, 188 "qed rdma alloc failed: cannot allocate memory (rdma info port). rc = %d\n", 189 rc); 190 goto free_rdma_dev; 191 } 192 193 /* Allocate bit map for pd's */ 194 rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->pd_map, RDMA_MAX_PDS); 195 if (rc) { 196 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 197 "Failed to allocate pd_map, rc = %d\n", 198 rc); 199 goto free_rdma_port; 200 } 201 202 /* Allocate DPI bitmap */ 203 rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->dpi_map, 204 p_hwfn->dpi_count); 205 if (rc) { 206 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 207 "Failed to allocate DPI bitmap, rc = %d\n", rc); 208 goto free_pd_map; 209 } 210 211 /* Allocate bitmap for cq's. The maximum number of CQs is bounded to 212 * twice the number of QPs. 213 */ 214 rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->cq_map, 215 p_rdma_info->num_qps * 2); 216 if (rc) { 217 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 218 "Failed to allocate cq bitmap, rc = %d\n", rc); 219 goto free_dpi_map; 220 } 221 222 /* Allocate bitmap for toggle bit for cq icids 223 * We toggle the bit every time we create or resize cq for a given icid. 224 * The maximum number of CQs is bounded to twice the number of QPs. 225 */ 226 rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->toggle_bits, 227 p_rdma_info->num_qps * 2); 228 if (rc) { 229 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 230 "Failed to allocate toogle bits, rc = %d\n", rc); 231 goto free_cq_map; 232 } 233 234 /* Allocate bitmap for itids */ 235 rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->tid_map, 236 p_rdma_info->num_mrs); 237 if (rc) { 238 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 239 "Failed to allocate itids bitmaps, rc = %d\n", rc); 240 goto free_toggle_map; 241 } 242 243 /* Allocate bitmap for cids used for qps. */ 244 rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->cid_map, num_cons); 245 if (rc) { 246 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 247 "Failed to allocate cid bitmap, rc = %d\n", rc); 248 goto free_tid_map; 249 } 250 251 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Allocation successful\n"); 252 return 0; 253 254 free_tid_map: 255 kfree(p_rdma_info->tid_map.bitmap); 256 free_toggle_map: 257 kfree(p_rdma_info->toggle_bits.bitmap); 258 free_cq_map: 259 kfree(p_rdma_info->cq_map.bitmap); 260 free_dpi_map: 261 kfree(p_rdma_info->dpi_map.bitmap); 262 free_pd_map: 263 kfree(p_rdma_info->pd_map.bitmap); 264 free_rdma_port: 265 kfree(p_rdma_info->port); 266 free_rdma_dev: 267 kfree(p_rdma_info->dev); 268 free_rdma_info: 269 kfree(p_rdma_info); 270 271 return rc; 272 } 273 274 static void qed_rdma_resc_free(struct qed_hwfn *p_hwfn) 275 { 276 struct qed_rdma_info *p_rdma_info = p_hwfn->p_rdma_info; 277 278 kfree(p_rdma_info->cid_map.bitmap); 279 kfree(p_rdma_info->tid_map.bitmap); 280 kfree(p_rdma_info->toggle_bits.bitmap); 281 kfree(p_rdma_info->cq_map.bitmap); 282 kfree(p_rdma_info->dpi_map.bitmap); 283 kfree(p_rdma_info->pd_map.bitmap); 284 285 kfree(p_rdma_info->port); 286 kfree(p_rdma_info->dev); 287 288 kfree(p_rdma_info); 289 } 290 291 static void qed_rdma_free(struct qed_hwfn *p_hwfn) 292 { 293 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Freeing RDMA\n"); 294 295 qed_rdma_resc_free(p_hwfn); 296 } 297 298 static void qed_rdma_get_guid(struct qed_hwfn *p_hwfn, u8 *guid) 299 { 300 guid[0] = p_hwfn->hw_info.hw_mac_addr[0] ^ 2; 301 guid[1] = p_hwfn->hw_info.hw_mac_addr[1]; 302 guid[2] = p_hwfn->hw_info.hw_mac_addr[2]; 303 guid[3] = 0xff; 304 guid[4] = 0xfe; 305 guid[5] = p_hwfn->hw_info.hw_mac_addr[3]; 306 guid[6] = p_hwfn->hw_info.hw_mac_addr[4]; 307 guid[7] = p_hwfn->hw_info.hw_mac_addr[5]; 308 } 309 310 static void qed_rdma_init_events(struct qed_hwfn *p_hwfn, 311 struct qed_rdma_start_in_params *params) 312 { 313 struct qed_rdma_events *events; 314 315 events = &p_hwfn->p_rdma_info->events; 316 317 events->unaffiliated_event = params->events->unaffiliated_event; 318 events->affiliated_event = params->events->affiliated_event; 319 events->context = params->events->context; 320 } 321 322 static void qed_rdma_init_devinfo(struct qed_hwfn *p_hwfn, 323 struct qed_rdma_start_in_params *params) 324 { 325 struct qed_rdma_device *dev = p_hwfn->p_rdma_info->dev; 326 struct qed_dev *cdev = p_hwfn->cdev; 327 u32 pci_status_control; 328 u32 num_qps; 329 330 /* Vendor specific information */ 331 dev->vendor_id = cdev->vendor_id; 332 dev->vendor_part_id = cdev->device_id; 333 dev->hw_ver = 0; 334 dev->fw_ver = (FW_MAJOR_VERSION << 24) | (FW_MINOR_VERSION << 16) | 335 (FW_REVISION_VERSION << 8) | (FW_ENGINEERING_VERSION); 336 337 qed_rdma_get_guid(p_hwfn, (u8 *)&dev->sys_image_guid); 338 dev->node_guid = dev->sys_image_guid; 339 340 dev->max_sge = min_t(u32, RDMA_MAX_SGE_PER_SQ_WQE, 341 RDMA_MAX_SGE_PER_RQ_WQE); 342 343 if (cdev->rdma_max_sge) 344 dev->max_sge = min_t(u32, cdev->rdma_max_sge, dev->max_sge); 345 346 dev->max_inline = ROCE_REQ_MAX_INLINE_DATA_SIZE; 347 348 dev->max_inline = (cdev->rdma_max_inline) ? 349 min_t(u32, cdev->rdma_max_inline, dev->max_inline) : 350 dev->max_inline; 351 352 dev->max_wqe = QED_RDMA_MAX_WQE; 353 dev->max_cnq = (u8)FEAT_NUM(p_hwfn, QED_RDMA_CNQ); 354 355 /* The number of QPs may be higher than QED_ROCE_MAX_QPS, because 356 * it is up-aligned to 16 and then to ILT page size within qed cxt. 357 * This is OK in terms of ILT but we don't want to configure the FW 358 * above its abilities 359 */ 360 num_qps = ROCE_MAX_QPS; 361 num_qps = min_t(u64, num_qps, p_hwfn->p_rdma_info->num_qps); 362 dev->max_qp = num_qps; 363 364 /* CQs uses the same icids that QPs use hence they are limited by the 365 * number of icids. There are two icids per QP. 366 */ 367 dev->max_cq = num_qps * 2; 368 369 /* The number of mrs is smaller by 1 since the first is reserved */ 370 dev->max_mr = p_hwfn->p_rdma_info->num_mrs - 1; 371 dev->max_mr_size = QED_RDMA_MAX_MR_SIZE; 372 373 /* The maximum CQE capacity per CQ supported. 374 * max number of cqes will be in two layer pbl, 375 * 8 is the pointer size in bytes 376 * 32 is the size of cq element in bytes 377 */ 378 if (params->cq_mode == QED_RDMA_CQ_MODE_32_BITS) 379 dev->max_cqe = QED_RDMA_MAX_CQE_32_BIT; 380 else 381 dev->max_cqe = QED_RDMA_MAX_CQE_16_BIT; 382 383 dev->max_mw = 0; 384 dev->max_fmr = QED_RDMA_MAX_FMR; 385 dev->max_mr_mw_fmr_pbl = (PAGE_SIZE / 8) * (PAGE_SIZE / 8); 386 dev->max_mr_mw_fmr_size = dev->max_mr_mw_fmr_pbl * PAGE_SIZE; 387 dev->max_pkey = QED_RDMA_MAX_P_KEY; 388 389 dev->max_qp_resp_rd_atomic_resc = RDMA_RING_PAGE_SIZE / 390 (RDMA_RESP_RD_ATOMIC_ELM_SIZE * 2); 391 dev->max_qp_req_rd_atomic_resc = RDMA_RING_PAGE_SIZE / 392 RDMA_REQ_RD_ATOMIC_ELM_SIZE; 393 dev->max_dev_resp_rd_atomic_resc = dev->max_qp_resp_rd_atomic_resc * 394 p_hwfn->p_rdma_info->num_qps; 395 dev->page_size_caps = QED_RDMA_PAGE_SIZE_CAPS; 396 dev->dev_ack_delay = QED_RDMA_ACK_DELAY; 397 dev->max_pd = RDMA_MAX_PDS; 398 dev->max_ah = p_hwfn->p_rdma_info->num_qps; 399 dev->max_stats_queues = (u8)RESC_NUM(p_hwfn, QED_RDMA_STATS_QUEUE); 400 401 /* Set capablities */ 402 dev->dev_caps = 0; 403 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_RNR_NAK, 1); 404 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_PORT_ACTIVE_EVENT, 1); 405 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_PORT_CHANGE_EVENT, 1); 406 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_RESIZE_CQ, 1); 407 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_BASE_MEMORY_EXT, 1); 408 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_BASE_QUEUE_EXT, 1); 409 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_ZBVA, 1); 410 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_LOCAL_INV_FENCE, 1); 411 412 /* Check atomic operations support in PCI configuration space. */ 413 pci_read_config_dword(cdev->pdev, 414 cdev->pdev->pcie_cap + PCI_EXP_DEVCTL2, 415 &pci_status_control); 416 417 if (pci_status_control & PCI_EXP_DEVCTL2_LTR_EN) 418 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_ATOMIC_OP, 1); 419 } 420 421 static void qed_rdma_init_port(struct qed_hwfn *p_hwfn) 422 { 423 struct qed_rdma_port *port = p_hwfn->p_rdma_info->port; 424 struct qed_rdma_device *dev = p_hwfn->p_rdma_info->dev; 425 426 port->port_state = p_hwfn->mcp_info->link_output.link_up ? 427 QED_RDMA_PORT_UP : QED_RDMA_PORT_DOWN; 428 429 port->max_msg_size = min_t(u64, 430 (dev->max_mr_mw_fmr_size * 431 p_hwfn->cdev->rdma_max_sge), 432 BIT(31)); 433 434 port->pkey_bad_counter = 0; 435 } 436 437 static int qed_rdma_init_hw(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 438 { 439 u32 ll2_ethertype_en; 440 441 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Initializing HW\n"); 442 p_hwfn->b_rdma_enabled_in_prs = false; 443 444 qed_wr(p_hwfn, p_ptt, PRS_REG_ROCE_DEST_QP_MAX_PF, 0); 445 446 p_hwfn->rdma_prs_search_reg = PRS_REG_SEARCH_ROCE; 447 448 /* We delay writing to this reg until first cid is allocated. See 449 * qed_cxt_dynamic_ilt_alloc function for more details 450 */ 451 ll2_ethertype_en = qed_rd(p_hwfn, p_ptt, PRS_REG_LIGHT_L2_ETHERTYPE_EN); 452 qed_wr(p_hwfn, p_ptt, PRS_REG_LIGHT_L2_ETHERTYPE_EN, 453 (ll2_ethertype_en | 0x01)); 454 455 if (qed_cxt_get_proto_cid_start(p_hwfn, PROTOCOLID_ROCE) % 2) { 456 DP_NOTICE(p_hwfn, "The first RoCE's cid should be even\n"); 457 return -EINVAL; 458 } 459 460 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Initializing HW - Done\n"); 461 return 0; 462 } 463 464 static int qed_rdma_start_fw(struct qed_hwfn *p_hwfn, 465 struct qed_rdma_start_in_params *params, 466 struct qed_ptt *p_ptt) 467 { 468 struct rdma_init_func_ramrod_data *p_ramrod; 469 struct qed_rdma_cnq_params *p_cnq_pbl_list; 470 struct rdma_init_func_hdr *p_params_header; 471 struct rdma_cnq_params *p_cnq_params; 472 struct qed_sp_init_data init_data; 473 struct qed_spq_entry *p_ent; 474 u32 cnq_id, sb_id; 475 int rc; 476 477 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Starting FW\n"); 478 479 /* Save the number of cnqs for the function close ramrod */ 480 p_hwfn->p_rdma_info->num_cnqs = params->desired_cnq; 481 482 /* Get SPQ entry */ 483 memset(&init_data, 0, sizeof(init_data)); 484 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 485 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 486 487 rc = qed_sp_init_request(p_hwfn, &p_ent, RDMA_RAMROD_FUNC_INIT, 488 p_hwfn->p_rdma_info->proto, &init_data); 489 if (rc) 490 return rc; 491 492 p_ramrod = &p_ent->ramrod.roce_init_func.rdma; 493 494 p_params_header = &p_ramrod->params_header; 495 p_params_header->cnq_start_offset = (u8)RESC_START(p_hwfn, 496 QED_RDMA_CNQ_RAM); 497 p_params_header->num_cnqs = params->desired_cnq; 498 499 if (params->cq_mode == QED_RDMA_CQ_MODE_16_BITS) 500 p_params_header->cq_ring_mode = 1; 501 else 502 p_params_header->cq_ring_mode = 0; 503 504 for (cnq_id = 0; cnq_id < params->desired_cnq; cnq_id++) { 505 sb_id = qed_rdma_get_sb_id(p_hwfn, cnq_id); 506 p_cnq_params = &p_ramrod->cnq_params[cnq_id]; 507 p_cnq_pbl_list = ¶ms->cnq_pbl_list[cnq_id]; 508 p_cnq_params->sb_num = 509 cpu_to_le16(p_hwfn->sbs_info[sb_id]->igu_sb_id); 510 511 p_cnq_params->sb_index = p_hwfn->pf_params.rdma_pf_params.gl_pi; 512 p_cnq_params->num_pbl_pages = p_cnq_pbl_list->num_pbl_pages; 513 514 DMA_REGPAIR_LE(p_cnq_params->pbl_base_addr, 515 p_cnq_pbl_list->pbl_ptr); 516 517 /* we assume here that cnq_id and qz_offset are the same */ 518 p_cnq_params->queue_zone_num = 519 cpu_to_le16(p_hwfn->p_rdma_info->queue_zone_base + 520 cnq_id); 521 } 522 523 return qed_spq_post(p_hwfn, p_ent, NULL); 524 } 525 526 static int qed_rdma_alloc_tid(void *rdma_cxt, u32 *itid) 527 { 528 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 529 int rc; 530 531 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Allocate TID\n"); 532 533 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 534 rc = qed_rdma_bmap_alloc_id(p_hwfn, 535 &p_hwfn->p_rdma_info->tid_map, itid); 536 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 537 if (rc) 538 goto out; 539 540 rc = qed_cxt_dynamic_ilt_alloc(p_hwfn, QED_ELEM_TASK, *itid); 541 out: 542 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Allocate TID - done, rc = %d\n", rc); 543 return rc; 544 } 545 546 static int qed_rdma_reserve_lkey(struct qed_hwfn *p_hwfn) 547 { 548 struct qed_rdma_device *dev = p_hwfn->p_rdma_info->dev; 549 550 /* The first DPI is reserved for the Kernel */ 551 __set_bit(0, p_hwfn->p_rdma_info->dpi_map.bitmap); 552 553 /* Tid 0 will be used as the key for "reserved MR". 554 * The driver should allocate memory for it so it can be loaded but no 555 * ramrod should be passed on it. 556 */ 557 qed_rdma_alloc_tid(p_hwfn, &dev->reserved_lkey); 558 if (dev->reserved_lkey != RDMA_RESERVED_LKEY) { 559 DP_NOTICE(p_hwfn, 560 "Reserved lkey should be equal to RDMA_RESERVED_LKEY\n"); 561 return -EINVAL; 562 } 563 564 return 0; 565 } 566 567 static int qed_rdma_setup(struct qed_hwfn *p_hwfn, 568 struct qed_ptt *p_ptt, 569 struct qed_rdma_start_in_params *params) 570 { 571 int rc; 572 573 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "RDMA setup\n"); 574 575 spin_lock_init(&p_hwfn->p_rdma_info->lock); 576 577 qed_rdma_init_devinfo(p_hwfn, params); 578 qed_rdma_init_port(p_hwfn); 579 qed_rdma_init_events(p_hwfn, params); 580 581 rc = qed_rdma_reserve_lkey(p_hwfn); 582 if (rc) 583 return rc; 584 585 rc = qed_rdma_init_hw(p_hwfn, p_ptt); 586 if (rc) 587 return rc; 588 589 return qed_rdma_start_fw(p_hwfn, params, p_ptt); 590 } 591 592 static int qed_rdma_stop(void *rdma_cxt) 593 { 594 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 595 struct rdma_close_func_ramrod_data *p_ramrod; 596 struct qed_sp_init_data init_data; 597 struct qed_spq_entry *p_ent; 598 struct qed_ptt *p_ptt; 599 u32 ll2_ethertype_en; 600 int rc = -EBUSY; 601 602 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "RDMA stop\n"); 603 604 p_ptt = qed_ptt_acquire(p_hwfn); 605 if (!p_ptt) { 606 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Failed to acquire PTT\n"); 607 return rc; 608 } 609 610 /* Disable RoCE search */ 611 qed_wr(p_hwfn, p_ptt, p_hwfn->rdma_prs_search_reg, 0); 612 p_hwfn->b_rdma_enabled_in_prs = false; 613 614 qed_wr(p_hwfn, p_ptt, PRS_REG_ROCE_DEST_QP_MAX_PF, 0); 615 616 ll2_ethertype_en = qed_rd(p_hwfn, p_ptt, PRS_REG_LIGHT_L2_ETHERTYPE_EN); 617 618 qed_wr(p_hwfn, p_ptt, PRS_REG_LIGHT_L2_ETHERTYPE_EN, 619 (ll2_ethertype_en & 0xFFFE)); 620 621 qed_ptt_release(p_hwfn, p_ptt); 622 623 /* Get SPQ entry */ 624 memset(&init_data, 0, sizeof(init_data)); 625 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 626 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 627 628 /* Stop RoCE */ 629 rc = qed_sp_init_request(p_hwfn, &p_ent, RDMA_RAMROD_FUNC_CLOSE, 630 p_hwfn->p_rdma_info->proto, &init_data); 631 if (rc) 632 goto out; 633 634 p_ramrod = &p_ent->ramrod.rdma_close_func; 635 636 p_ramrod->num_cnqs = p_hwfn->p_rdma_info->num_cnqs; 637 p_ramrod->cnq_start_offset = (u8)RESC_START(p_hwfn, QED_RDMA_CNQ_RAM); 638 639 rc = qed_spq_post(p_hwfn, p_ent, NULL); 640 641 out: 642 qed_rdma_free(p_hwfn); 643 644 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "RDMA stop done, rc = %d\n", rc); 645 return rc; 646 } 647 648 static int qed_rdma_add_user(void *rdma_cxt, 649 struct qed_rdma_add_user_out_params *out_params) 650 { 651 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 652 u32 dpi_start_offset; 653 u32 returned_id = 0; 654 int rc; 655 656 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Adding User\n"); 657 658 /* Allocate DPI */ 659 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 660 rc = qed_rdma_bmap_alloc_id(p_hwfn, &p_hwfn->p_rdma_info->dpi_map, 661 &returned_id); 662 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 663 664 out_params->dpi = (u16)returned_id; 665 666 /* Calculate the corresponding DPI address */ 667 dpi_start_offset = p_hwfn->dpi_start_offset; 668 669 out_params->dpi_addr = (u64)((u8 __iomem *)p_hwfn->doorbells + 670 dpi_start_offset + 671 ((out_params->dpi) * p_hwfn->dpi_size)); 672 673 out_params->dpi_phys_addr = p_hwfn->cdev->db_phys_addr + 674 dpi_start_offset + 675 ((out_params->dpi) * p_hwfn->dpi_size); 676 677 out_params->dpi_size = p_hwfn->dpi_size; 678 679 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Adding user - done, rc = %d\n", rc); 680 return rc; 681 } 682 683 static struct qed_rdma_port *qed_rdma_query_port(void *rdma_cxt) 684 { 685 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 686 struct qed_rdma_port *p_port = p_hwfn->p_rdma_info->port; 687 688 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "RDMA Query port\n"); 689 690 /* Link may have changed */ 691 p_port->port_state = p_hwfn->mcp_info->link_output.link_up ? 692 QED_RDMA_PORT_UP : QED_RDMA_PORT_DOWN; 693 694 p_port->link_speed = p_hwfn->mcp_info->link_output.speed; 695 696 return p_port; 697 } 698 699 static struct qed_rdma_device *qed_rdma_query_device(void *rdma_cxt) 700 { 701 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 702 703 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Query device\n"); 704 705 /* Return struct with device parameters */ 706 return p_hwfn->p_rdma_info->dev; 707 } 708 709 static void qed_rdma_free_tid(void *rdma_cxt, u32 itid) 710 { 711 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 712 713 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "itid = %08x\n", itid); 714 715 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 716 qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->tid_map, itid); 717 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 718 } 719 720 static void qed_rdma_cnq_prod_update(void *rdma_cxt, u8 qz_offset, u16 prod) 721 { 722 struct qed_hwfn *p_hwfn; 723 u16 qz_num; 724 u32 addr; 725 726 p_hwfn = (struct qed_hwfn *)rdma_cxt; 727 qz_num = p_hwfn->p_rdma_info->queue_zone_base + qz_offset; 728 addr = GTT_BAR0_MAP_REG_USDM_RAM + 729 USTORM_COMMON_QUEUE_CONS_OFFSET(qz_num); 730 731 REG_WR16(p_hwfn, addr, prod); 732 733 /* keep prod updates ordered */ 734 wmb(); 735 } 736 737 static int qed_fill_rdma_dev_info(struct qed_dev *cdev, 738 struct qed_dev_rdma_info *info) 739 { 740 memset(info, 0, sizeof(*info)); 741 742 info->rdma_type = QED_RDMA_TYPE_ROCE; 743 744 qed_fill_dev_info(cdev, &info->common); 745 746 return 0; 747 } 748 749 static int qed_rdma_get_sb_start(struct qed_dev *cdev) 750 { 751 int feat_num; 752 753 if (cdev->num_hwfns > 1) 754 feat_num = FEAT_NUM(QED_LEADING_HWFN(cdev), QED_PF_L2_QUE); 755 else 756 feat_num = FEAT_NUM(QED_LEADING_HWFN(cdev), QED_PF_L2_QUE) * 757 cdev->num_hwfns; 758 759 return feat_num; 760 } 761 762 static int qed_rdma_get_min_cnq_msix(struct qed_dev *cdev) 763 { 764 int n_cnq = FEAT_NUM(QED_LEADING_HWFN(cdev), QED_RDMA_CNQ); 765 int n_msix = cdev->int_params.rdma_msix_cnt; 766 767 return min_t(int, n_cnq, n_msix); 768 } 769 770 static int qed_rdma_set_int(struct qed_dev *cdev, u16 cnt) 771 { 772 int limit = 0; 773 774 /* Mark the fastpath as free/used */ 775 cdev->int_params.fp_initialized = cnt ? true : false; 776 777 if (cdev->int_params.out.int_mode != QED_INT_MODE_MSIX) { 778 DP_ERR(cdev, 779 "qed roce supports only MSI-X interrupts (detected %d).\n", 780 cdev->int_params.out.int_mode); 781 return -EINVAL; 782 } else if (cdev->int_params.fp_msix_cnt) { 783 limit = cdev->int_params.rdma_msix_cnt; 784 } 785 786 if (!limit) 787 return -ENOMEM; 788 789 return min_t(int, cnt, limit); 790 } 791 792 static int qed_rdma_get_int(struct qed_dev *cdev, struct qed_int_info *info) 793 { 794 memset(info, 0, sizeof(*info)); 795 796 if (!cdev->int_params.fp_initialized) { 797 DP_INFO(cdev, 798 "Protocol driver requested interrupt information, but its support is not yet configured\n"); 799 return -EINVAL; 800 } 801 802 if (cdev->int_params.out.int_mode == QED_INT_MODE_MSIX) { 803 int msix_base = cdev->int_params.rdma_msix_base; 804 805 info->msix_cnt = cdev->int_params.rdma_msix_cnt; 806 info->msix = &cdev->int_params.msix_table[msix_base]; 807 808 DP_VERBOSE(cdev, QED_MSG_RDMA, "msix_cnt = %d msix_base=%d\n", 809 info->msix_cnt, msix_base); 810 } 811 812 return 0; 813 } 814 815 static int qed_rdma_alloc_pd(void *rdma_cxt, u16 *pd) 816 { 817 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 818 u32 returned_id; 819 int rc; 820 821 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Alloc PD\n"); 822 823 /* Allocates an unused protection domain */ 824 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 825 rc = qed_rdma_bmap_alloc_id(p_hwfn, 826 &p_hwfn->p_rdma_info->pd_map, &returned_id); 827 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 828 829 *pd = (u16)returned_id; 830 831 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Alloc PD - done, rc = %d\n", rc); 832 return rc; 833 } 834 835 static void qed_rdma_free_pd(void *rdma_cxt, u16 pd) 836 { 837 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 838 839 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "pd = %08x\n", pd); 840 841 /* Returns a previously allocated protection domain for reuse */ 842 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 843 qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->pd_map, pd); 844 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 845 } 846 847 static enum qed_rdma_toggle_bit 848 qed_rdma_toggle_bit_create_resize_cq(struct qed_hwfn *p_hwfn, u16 icid) 849 { 850 struct qed_rdma_info *p_info = p_hwfn->p_rdma_info; 851 enum qed_rdma_toggle_bit toggle_bit; 852 u32 bmap_id; 853 854 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", icid); 855 856 /* the function toggle the bit that is related to a given icid 857 * and returns the new toggle bit's value 858 */ 859 bmap_id = icid - qed_cxt_get_proto_cid_start(p_hwfn, p_info->proto); 860 861 spin_lock_bh(&p_info->lock); 862 toggle_bit = !test_and_change_bit(bmap_id, 863 p_info->toggle_bits.bitmap); 864 spin_unlock_bh(&p_info->lock); 865 866 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QED_RDMA_TOGGLE_BIT_= %d\n", 867 toggle_bit); 868 869 return toggle_bit; 870 } 871 872 static int qed_rdma_create_cq(void *rdma_cxt, 873 struct qed_rdma_create_cq_in_params *params, 874 u16 *icid) 875 { 876 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 877 struct qed_rdma_info *p_info = p_hwfn->p_rdma_info; 878 struct rdma_create_cq_ramrod_data *p_ramrod; 879 enum qed_rdma_toggle_bit toggle_bit; 880 struct qed_sp_init_data init_data; 881 struct qed_spq_entry *p_ent; 882 u32 returned_id, start_cid; 883 int rc; 884 885 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "cq_handle = %08x%08x\n", 886 params->cq_handle_hi, params->cq_handle_lo); 887 888 /* Allocate icid */ 889 spin_lock_bh(&p_info->lock); 890 rc = qed_rdma_bmap_alloc_id(p_hwfn, 891 &p_info->cq_map, &returned_id); 892 spin_unlock_bh(&p_info->lock); 893 894 if (rc) { 895 DP_NOTICE(p_hwfn, "Can't create CQ, rc = %d\n", rc); 896 return rc; 897 } 898 899 start_cid = qed_cxt_get_proto_cid_start(p_hwfn, 900 p_info->proto); 901 *icid = returned_id + start_cid; 902 903 /* Check if icid requires a page allocation */ 904 rc = qed_cxt_dynamic_ilt_alloc(p_hwfn, QED_ELEM_CXT, *icid); 905 if (rc) 906 goto err; 907 908 /* Get SPQ entry */ 909 memset(&init_data, 0, sizeof(init_data)); 910 init_data.cid = *icid; 911 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 912 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 913 914 /* Send create CQ ramrod */ 915 rc = qed_sp_init_request(p_hwfn, &p_ent, 916 RDMA_RAMROD_CREATE_CQ, 917 p_info->proto, &init_data); 918 if (rc) 919 goto err; 920 921 p_ramrod = &p_ent->ramrod.rdma_create_cq; 922 923 p_ramrod->cq_handle.hi = cpu_to_le32(params->cq_handle_hi); 924 p_ramrod->cq_handle.lo = cpu_to_le32(params->cq_handle_lo); 925 p_ramrod->dpi = cpu_to_le16(params->dpi); 926 p_ramrod->is_two_level_pbl = params->pbl_two_level; 927 p_ramrod->max_cqes = cpu_to_le32(params->cq_size); 928 DMA_REGPAIR_LE(p_ramrod->pbl_addr, params->pbl_ptr); 929 p_ramrod->pbl_num_pages = cpu_to_le16(params->pbl_num_pages); 930 p_ramrod->cnq_id = (u8)RESC_START(p_hwfn, QED_RDMA_CNQ_RAM) + 931 params->cnq_id; 932 p_ramrod->int_timeout = params->int_timeout; 933 934 /* toggle the bit for every resize or create cq for a given icid */ 935 toggle_bit = qed_rdma_toggle_bit_create_resize_cq(p_hwfn, *icid); 936 937 p_ramrod->toggle_bit = toggle_bit; 938 939 rc = qed_spq_post(p_hwfn, p_ent, NULL); 940 if (rc) { 941 /* restore toggle bit */ 942 qed_rdma_toggle_bit_create_resize_cq(p_hwfn, *icid); 943 goto err; 944 } 945 946 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Created CQ, rc = %d\n", rc); 947 return rc; 948 949 err: 950 /* release allocated icid */ 951 spin_lock_bh(&p_info->lock); 952 qed_bmap_release_id(p_hwfn, &p_info->cq_map, returned_id); 953 spin_unlock_bh(&p_info->lock); 954 DP_NOTICE(p_hwfn, "Create CQ failed, rc = %d\n", rc); 955 956 return rc; 957 } 958 959 static int 960 qed_rdma_destroy_cq(void *rdma_cxt, 961 struct qed_rdma_destroy_cq_in_params *in_params, 962 struct qed_rdma_destroy_cq_out_params *out_params) 963 { 964 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 965 struct rdma_destroy_cq_output_params *p_ramrod_res; 966 struct rdma_destroy_cq_ramrod_data *p_ramrod; 967 struct qed_sp_init_data init_data; 968 struct qed_spq_entry *p_ent; 969 dma_addr_t ramrod_res_phys; 970 int rc = -ENOMEM; 971 972 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", in_params->icid); 973 974 p_ramrod_res = 975 (struct rdma_destroy_cq_output_params *) 976 dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 977 sizeof(struct rdma_destroy_cq_output_params), 978 &ramrod_res_phys, GFP_KERNEL); 979 if (!p_ramrod_res) { 980 DP_NOTICE(p_hwfn, 981 "qed destroy cq failed: cannot allocate memory (ramrod)\n"); 982 return rc; 983 } 984 985 /* Get SPQ entry */ 986 memset(&init_data, 0, sizeof(init_data)); 987 init_data.cid = in_params->icid; 988 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 989 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 990 991 /* Send destroy CQ ramrod */ 992 rc = qed_sp_init_request(p_hwfn, &p_ent, 993 RDMA_RAMROD_DESTROY_CQ, 994 p_hwfn->p_rdma_info->proto, &init_data); 995 if (rc) 996 goto err; 997 998 p_ramrod = &p_ent->ramrod.rdma_destroy_cq; 999 DMA_REGPAIR_LE(p_ramrod->output_params_addr, ramrod_res_phys); 1000 1001 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1002 if (rc) 1003 goto err; 1004 1005 out_params->num_cq_notif = le16_to_cpu(p_ramrod_res->cnq_num); 1006 1007 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 1008 sizeof(struct rdma_destroy_cq_output_params), 1009 p_ramrod_res, ramrod_res_phys); 1010 1011 /* Free icid */ 1012 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 1013 1014 qed_bmap_release_id(p_hwfn, 1015 &p_hwfn->p_rdma_info->cq_map, 1016 (in_params->icid - 1017 qed_cxt_get_proto_cid_start(p_hwfn, 1018 p_hwfn-> 1019 p_rdma_info->proto))); 1020 1021 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 1022 1023 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Destroyed CQ, rc = %d\n", rc); 1024 return rc; 1025 1026 err: dma_free_coherent(&p_hwfn->cdev->pdev->dev, 1027 sizeof(struct rdma_destroy_cq_output_params), 1028 p_ramrod_res, ramrod_res_phys); 1029 1030 return rc; 1031 } 1032 1033 static void qed_rdma_set_fw_mac(u16 *p_fw_mac, u8 *p_qed_mac) 1034 { 1035 p_fw_mac[0] = cpu_to_le16((p_qed_mac[0] << 8) + p_qed_mac[1]); 1036 p_fw_mac[1] = cpu_to_le16((p_qed_mac[2] << 8) + p_qed_mac[3]); 1037 p_fw_mac[2] = cpu_to_le16((p_qed_mac[4] << 8) + p_qed_mac[5]); 1038 } 1039 1040 static void qed_rdma_copy_gids(struct qed_rdma_qp *qp, __le32 *src_gid, 1041 __le32 *dst_gid) 1042 { 1043 u32 i; 1044 1045 if (qp->roce_mode == ROCE_V2_IPV4) { 1046 /* The IPv4 addresses shall be aligned to the highest word. 1047 * The lower words must be zero. 1048 */ 1049 memset(src_gid, 0, sizeof(union qed_gid)); 1050 memset(dst_gid, 0, sizeof(union qed_gid)); 1051 src_gid[3] = cpu_to_le32(qp->sgid.ipv4_addr); 1052 dst_gid[3] = cpu_to_le32(qp->dgid.ipv4_addr); 1053 } else { 1054 /* GIDs and IPv6 addresses coincide in location and size */ 1055 for (i = 0; i < ARRAY_SIZE(qp->sgid.dwords); i++) { 1056 src_gid[i] = cpu_to_le32(qp->sgid.dwords[i]); 1057 dst_gid[i] = cpu_to_le32(qp->dgid.dwords[i]); 1058 } 1059 } 1060 } 1061 1062 static enum roce_flavor qed_roce_mode_to_flavor(enum roce_mode roce_mode) 1063 { 1064 enum roce_flavor flavor; 1065 1066 switch (roce_mode) { 1067 case ROCE_V1: 1068 flavor = PLAIN_ROCE; 1069 break; 1070 case ROCE_V2_IPV4: 1071 flavor = RROCE_IPV4; 1072 break; 1073 case ROCE_V2_IPV6: 1074 flavor = ROCE_V2_IPV6; 1075 break; 1076 default: 1077 flavor = MAX_ROCE_MODE; 1078 break; 1079 } 1080 return flavor; 1081 } 1082 1083 static int qed_roce_alloc_cid(struct qed_hwfn *p_hwfn, u16 *cid) 1084 { 1085 struct qed_rdma_info *p_rdma_info = p_hwfn->p_rdma_info; 1086 u32 responder_icid; 1087 u32 requester_icid; 1088 int rc; 1089 1090 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 1091 rc = qed_rdma_bmap_alloc_id(p_hwfn, &p_rdma_info->cid_map, 1092 &responder_icid); 1093 if (rc) { 1094 spin_unlock_bh(&p_rdma_info->lock); 1095 return rc; 1096 } 1097 1098 rc = qed_rdma_bmap_alloc_id(p_hwfn, &p_rdma_info->cid_map, 1099 &requester_icid); 1100 1101 spin_unlock_bh(&p_rdma_info->lock); 1102 if (rc) 1103 goto err; 1104 1105 /* the two icid's should be adjacent */ 1106 if ((requester_icid - responder_icid) != 1) { 1107 DP_NOTICE(p_hwfn, "Failed to allocate two adjacent qp's'\n"); 1108 rc = -EINVAL; 1109 goto err; 1110 } 1111 1112 responder_icid += qed_cxt_get_proto_cid_start(p_hwfn, 1113 p_rdma_info->proto); 1114 requester_icid += qed_cxt_get_proto_cid_start(p_hwfn, 1115 p_rdma_info->proto); 1116 1117 /* If these icids require a new ILT line allocate DMA-able context for 1118 * an ILT page 1119 */ 1120 rc = qed_cxt_dynamic_ilt_alloc(p_hwfn, QED_ELEM_CXT, responder_icid); 1121 if (rc) 1122 goto err; 1123 1124 rc = qed_cxt_dynamic_ilt_alloc(p_hwfn, QED_ELEM_CXT, requester_icid); 1125 if (rc) 1126 goto err; 1127 1128 *cid = (u16)responder_icid; 1129 return rc; 1130 1131 err: 1132 spin_lock_bh(&p_rdma_info->lock); 1133 qed_bmap_release_id(p_hwfn, &p_rdma_info->cid_map, responder_icid); 1134 qed_bmap_release_id(p_hwfn, &p_rdma_info->cid_map, requester_icid); 1135 1136 spin_unlock_bh(&p_rdma_info->lock); 1137 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 1138 "Allocate CID - failed, rc = %d\n", rc); 1139 return rc; 1140 } 1141 1142 static int qed_roce_sp_create_responder(struct qed_hwfn *p_hwfn, 1143 struct qed_rdma_qp *qp) 1144 { 1145 struct roce_create_qp_resp_ramrod_data *p_ramrod; 1146 struct qed_sp_init_data init_data; 1147 union qed_qm_pq_params qm_params; 1148 enum roce_flavor roce_flavor; 1149 struct qed_spq_entry *p_ent; 1150 u16 physical_queue0 = 0; 1151 int rc; 1152 1153 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); 1154 1155 /* Allocate DMA-able memory for IRQ */ 1156 qp->irq_num_pages = 1; 1157 qp->irq = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 1158 RDMA_RING_PAGE_SIZE, 1159 &qp->irq_phys_addr, GFP_KERNEL); 1160 if (!qp->irq) { 1161 rc = -ENOMEM; 1162 DP_NOTICE(p_hwfn, 1163 "qed create responder failed: cannot allocate memory (irq). rc = %d\n", 1164 rc); 1165 return rc; 1166 } 1167 1168 /* Get SPQ entry */ 1169 memset(&init_data, 0, sizeof(init_data)); 1170 init_data.cid = qp->icid; 1171 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 1172 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 1173 1174 rc = qed_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_CREATE_QP, 1175 PROTOCOLID_ROCE, &init_data); 1176 if (rc) 1177 goto err; 1178 1179 p_ramrod = &p_ent->ramrod.roce_create_qp_resp; 1180 1181 p_ramrod->flags = 0; 1182 1183 roce_flavor = qed_roce_mode_to_flavor(qp->roce_mode); 1184 SET_FIELD(p_ramrod->flags, 1185 ROCE_CREATE_QP_RESP_RAMROD_DATA_ROCE_FLAVOR, roce_flavor); 1186 1187 SET_FIELD(p_ramrod->flags, 1188 ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_RD_EN, 1189 qp->incoming_rdma_read_en); 1190 1191 SET_FIELD(p_ramrod->flags, 1192 ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_WR_EN, 1193 qp->incoming_rdma_write_en); 1194 1195 SET_FIELD(p_ramrod->flags, 1196 ROCE_CREATE_QP_RESP_RAMROD_DATA_ATOMIC_EN, 1197 qp->incoming_atomic_en); 1198 1199 SET_FIELD(p_ramrod->flags, 1200 ROCE_CREATE_QP_RESP_RAMROD_DATA_E2E_FLOW_CONTROL_EN, 1201 qp->e2e_flow_control_en); 1202 1203 SET_FIELD(p_ramrod->flags, 1204 ROCE_CREATE_QP_RESP_RAMROD_DATA_SRQ_FLG, qp->use_srq); 1205 1206 SET_FIELD(p_ramrod->flags, 1207 ROCE_CREATE_QP_RESP_RAMROD_DATA_RESERVED_KEY_EN, 1208 qp->fmr_and_reserved_lkey); 1209 1210 SET_FIELD(p_ramrod->flags, 1211 ROCE_CREATE_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER, 1212 qp->min_rnr_nak_timer); 1213 1214 p_ramrod->max_ird = qp->max_rd_atomic_resp; 1215 p_ramrod->traffic_class = qp->traffic_class_tos; 1216 p_ramrod->hop_limit = qp->hop_limit_ttl; 1217 p_ramrod->irq_num_pages = qp->irq_num_pages; 1218 p_ramrod->p_key = cpu_to_le16(qp->pkey); 1219 p_ramrod->flow_label = cpu_to_le32(qp->flow_label); 1220 p_ramrod->dst_qp_id = cpu_to_le32(qp->dest_qp); 1221 p_ramrod->mtu = cpu_to_le16(qp->mtu); 1222 p_ramrod->initial_psn = cpu_to_le32(qp->rq_psn); 1223 p_ramrod->pd = cpu_to_le16(qp->pd); 1224 p_ramrod->rq_num_pages = cpu_to_le16(qp->rq_num_pages); 1225 DMA_REGPAIR_LE(p_ramrod->rq_pbl_addr, qp->rq_pbl_ptr); 1226 DMA_REGPAIR_LE(p_ramrod->irq_pbl_addr, qp->irq_phys_addr); 1227 qed_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid); 1228 p_ramrod->qp_handle_for_async.hi = cpu_to_le32(qp->qp_handle_async.hi); 1229 p_ramrod->qp_handle_for_async.lo = cpu_to_le32(qp->qp_handle_async.lo); 1230 p_ramrod->qp_handle_for_cqe.hi = cpu_to_le32(qp->qp_handle.hi); 1231 p_ramrod->qp_handle_for_cqe.lo = cpu_to_le32(qp->qp_handle.lo); 1232 p_ramrod->stats_counter_id = p_hwfn->rel_pf_id; 1233 p_ramrod->cq_cid = cpu_to_le32((p_hwfn->hw_info.opaque_fid << 16) | 1234 qp->rq_cq_id); 1235 1236 memset(&qm_params, 0, sizeof(qm_params)); 1237 qm_params.roce.qpid = qp->icid >> 1; 1238 physical_queue0 = qed_get_qm_pq(p_hwfn, PROTOCOLID_ROCE, &qm_params); 1239 1240 p_ramrod->physical_queue0 = cpu_to_le16(physical_queue0); 1241 p_ramrod->dpi = cpu_to_le16(qp->dpi); 1242 1243 qed_rdma_set_fw_mac(p_ramrod->remote_mac_addr, qp->remote_mac_addr); 1244 qed_rdma_set_fw_mac(p_ramrod->local_mac_addr, qp->local_mac_addr); 1245 1246 p_ramrod->udp_src_port = qp->udp_src_port; 1247 p_ramrod->vlan_id = cpu_to_le16(qp->vlan_id); 1248 p_ramrod->srq_id.srq_idx = cpu_to_le16(qp->srq_id); 1249 p_ramrod->srq_id.opaque_fid = cpu_to_le16(p_hwfn->hw_info.opaque_fid); 1250 1251 p_ramrod->stats_counter_id = RESC_START(p_hwfn, QED_RDMA_STATS_QUEUE) + 1252 qp->stats_queue; 1253 1254 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1255 1256 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d physical_queue0 = 0x%x\n", 1257 rc, physical_queue0); 1258 1259 if (rc) 1260 goto err; 1261 1262 qp->resp_offloaded = true; 1263 1264 return rc; 1265 1266 err: 1267 DP_NOTICE(p_hwfn, "create responder - failed, rc = %d\n", rc); 1268 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 1269 qp->irq_num_pages * RDMA_RING_PAGE_SIZE, 1270 qp->irq, qp->irq_phys_addr); 1271 1272 return rc; 1273 } 1274 1275 static int qed_roce_sp_create_requester(struct qed_hwfn *p_hwfn, 1276 struct qed_rdma_qp *qp) 1277 { 1278 struct roce_create_qp_req_ramrod_data *p_ramrod; 1279 struct qed_sp_init_data init_data; 1280 union qed_qm_pq_params qm_params; 1281 enum roce_flavor roce_flavor; 1282 struct qed_spq_entry *p_ent; 1283 u16 physical_queue0 = 0; 1284 int rc; 1285 1286 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); 1287 1288 /* Allocate DMA-able memory for ORQ */ 1289 qp->orq_num_pages = 1; 1290 qp->orq = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 1291 RDMA_RING_PAGE_SIZE, 1292 &qp->orq_phys_addr, GFP_KERNEL); 1293 if (!qp->orq) { 1294 rc = -ENOMEM; 1295 DP_NOTICE(p_hwfn, 1296 "qed create requester failed: cannot allocate memory (orq). rc = %d\n", 1297 rc); 1298 return rc; 1299 } 1300 1301 /* Get SPQ entry */ 1302 memset(&init_data, 0, sizeof(init_data)); 1303 init_data.cid = qp->icid + 1; 1304 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 1305 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 1306 1307 rc = qed_sp_init_request(p_hwfn, &p_ent, 1308 ROCE_RAMROD_CREATE_QP, 1309 PROTOCOLID_ROCE, &init_data); 1310 if (rc) 1311 goto err; 1312 1313 p_ramrod = &p_ent->ramrod.roce_create_qp_req; 1314 1315 p_ramrod->flags = 0; 1316 1317 roce_flavor = qed_roce_mode_to_flavor(qp->roce_mode); 1318 SET_FIELD(p_ramrod->flags, 1319 ROCE_CREATE_QP_REQ_RAMROD_DATA_ROCE_FLAVOR, roce_flavor); 1320 1321 SET_FIELD(p_ramrod->flags, 1322 ROCE_CREATE_QP_REQ_RAMROD_DATA_FMR_AND_RESERVED_EN, 1323 qp->fmr_and_reserved_lkey); 1324 1325 SET_FIELD(p_ramrod->flags, 1326 ROCE_CREATE_QP_REQ_RAMROD_DATA_SIGNALED_COMP, qp->signal_all); 1327 1328 SET_FIELD(p_ramrod->flags, 1329 ROCE_CREATE_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT, qp->retry_cnt); 1330 1331 SET_FIELD(p_ramrod->flags, 1332 ROCE_CREATE_QP_REQ_RAMROD_DATA_RNR_NAK_CNT, 1333 qp->rnr_retry_cnt); 1334 1335 p_ramrod->max_ord = qp->max_rd_atomic_req; 1336 p_ramrod->traffic_class = qp->traffic_class_tos; 1337 p_ramrod->hop_limit = qp->hop_limit_ttl; 1338 p_ramrod->orq_num_pages = qp->orq_num_pages; 1339 p_ramrod->p_key = cpu_to_le16(qp->pkey); 1340 p_ramrod->flow_label = cpu_to_le32(qp->flow_label); 1341 p_ramrod->dst_qp_id = cpu_to_le32(qp->dest_qp); 1342 p_ramrod->ack_timeout_val = cpu_to_le32(qp->ack_timeout); 1343 p_ramrod->mtu = cpu_to_le16(qp->mtu); 1344 p_ramrod->initial_psn = cpu_to_le32(qp->sq_psn); 1345 p_ramrod->pd = cpu_to_le16(qp->pd); 1346 p_ramrod->sq_num_pages = cpu_to_le16(qp->sq_num_pages); 1347 DMA_REGPAIR_LE(p_ramrod->sq_pbl_addr, qp->sq_pbl_ptr); 1348 DMA_REGPAIR_LE(p_ramrod->orq_pbl_addr, qp->orq_phys_addr); 1349 qed_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid); 1350 p_ramrod->qp_handle_for_async.hi = cpu_to_le32(qp->qp_handle_async.hi); 1351 p_ramrod->qp_handle_for_async.lo = cpu_to_le32(qp->qp_handle_async.lo); 1352 p_ramrod->qp_handle_for_cqe.hi = cpu_to_le32(qp->qp_handle.hi); 1353 p_ramrod->qp_handle_for_cqe.lo = cpu_to_le32(qp->qp_handle.lo); 1354 p_ramrod->stats_counter_id = p_hwfn->rel_pf_id; 1355 p_ramrod->cq_cid = cpu_to_le32((p_hwfn->hw_info.opaque_fid << 16) | 1356 qp->sq_cq_id); 1357 1358 memset(&qm_params, 0, sizeof(qm_params)); 1359 qm_params.roce.qpid = qp->icid >> 1; 1360 physical_queue0 = qed_get_qm_pq(p_hwfn, PROTOCOLID_ROCE, &qm_params); 1361 1362 p_ramrod->physical_queue0 = cpu_to_le16(physical_queue0); 1363 p_ramrod->dpi = cpu_to_le16(qp->dpi); 1364 1365 qed_rdma_set_fw_mac(p_ramrod->remote_mac_addr, qp->remote_mac_addr); 1366 qed_rdma_set_fw_mac(p_ramrod->local_mac_addr, qp->local_mac_addr); 1367 1368 p_ramrod->udp_src_port = qp->udp_src_port; 1369 p_ramrod->vlan_id = cpu_to_le16(qp->vlan_id); 1370 p_ramrod->stats_counter_id = RESC_START(p_hwfn, QED_RDMA_STATS_QUEUE) + 1371 qp->stats_queue; 1372 1373 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1374 1375 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc); 1376 1377 if (rc) 1378 goto err; 1379 1380 qp->req_offloaded = true; 1381 1382 return rc; 1383 1384 err: 1385 DP_NOTICE(p_hwfn, "Create requested - failed, rc = %d\n", rc); 1386 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 1387 qp->orq_num_pages * RDMA_RING_PAGE_SIZE, 1388 qp->orq, qp->orq_phys_addr); 1389 return rc; 1390 } 1391 1392 static int qed_roce_sp_modify_responder(struct qed_hwfn *p_hwfn, 1393 struct qed_rdma_qp *qp, 1394 bool move_to_err, u32 modify_flags) 1395 { 1396 struct roce_modify_qp_resp_ramrod_data *p_ramrod; 1397 struct qed_sp_init_data init_data; 1398 struct qed_spq_entry *p_ent; 1399 int rc; 1400 1401 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); 1402 1403 if (move_to_err && !qp->resp_offloaded) 1404 return 0; 1405 1406 /* Get SPQ entry */ 1407 memset(&init_data, 0, sizeof(init_data)); 1408 init_data.cid = qp->icid; 1409 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 1410 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 1411 1412 rc = qed_sp_init_request(p_hwfn, &p_ent, 1413 ROCE_EVENT_MODIFY_QP, 1414 PROTOCOLID_ROCE, &init_data); 1415 if (rc) { 1416 DP_NOTICE(p_hwfn, "rc = %d\n", rc); 1417 return rc; 1418 } 1419 1420 p_ramrod = &p_ent->ramrod.roce_modify_qp_resp; 1421 1422 p_ramrod->flags = 0; 1423 1424 SET_FIELD(p_ramrod->flags, 1425 ROCE_MODIFY_QP_RESP_RAMROD_DATA_MOVE_TO_ERR_FLG, move_to_err); 1426 1427 SET_FIELD(p_ramrod->flags, 1428 ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_RD_EN, 1429 qp->incoming_rdma_read_en); 1430 1431 SET_FIELD(p_ramrod->flags, 1432 ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_WR_EN, 1433 qp->incoming_rdma_write_en); 1434 1435 SET_FIELD(p_ramrod->flags, 1436 ROCE_MODIFY_QP_RESP_RAMROD_DATA_ATOMIC_EN, 1437 qp->incoming_atomic_en); 1438 1439 SET_FIELD(p_ramrod->flags, 1440 ROCE_CREATE_QP_RESP_RAMROD_DATA_E2E_FLOW_CONTROL_EN, 1441 qp->e2e_flow_control_en); 1442 1443 SET_FIELD(p_ramrod->flags, 1444 ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_OPS_EN_FLG, 1445 GET_FIELD(modify_flags, 1446 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN)); 1447 1448 SET_FIELD(p_ramrod->flags, 1449 ROCE_MODIFY_QP_RESP_RAMROD_DATA_P_KEY_FLG, 1450 GET_FIELD(modify_flags, QED_ROCE_MODIFY_QP_VALID_PKEY)); 1451 1452 SET_FIELD(p_ramrod->flags, 1453 ROCE_MODIFY_QP_RESP_RAMROD_DATA_ADDRESS_VECTOR_FLG, 1454 GET_FIELD(modify_flags, 1455 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR)); 1456 1457 SET_FIELD(p_ramrod->flags, 1458 ROCE_MODIFY_QP_RESP_RAMROD_DATA_MAX_IRD_FLG, 1459 GET_FIELD(modify_flags, 1460 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP)); 1461 1462 SET_FIELD(p_ramrod->flags, 1463 ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER_FLG, 1464 GET_FIELD(modify_flags, 1465 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER)); 1466 1467 p_ramrod->fields = 0; 1468 SET_FIELD(p_ramrod->fields, 1469 ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER, 1470 qp->min_rnr_nak_timer); 1471 1472 p_ramrod->max_ird = qp->max_rd_atomic_resp; 1473 p_ramrod->traffic_class = qp->traffic_class_tos; 1474 p_ramrod->hop_limit = qp->hop_limit_ttl; 1475 p_ramrod->p_key = cpu_to_le16(qp->pkey); 1476 p_ramrod->flow_label = cpu_to_le32(qp->flow_label); 1477 p_ramrod->mtu = cpu_to_le16(qp->mtu); 1478 qed_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid); 1479 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1480 1481 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Modify responder, rc = %d\n", rc); 1482 return rc; 1483 } 1484 1485 static int qed_roce_sp_modify_requester(struct qed_hwfn *p_hwfn, 1486 struct qed_rdma_qp *qp, 1487 bool move_to_sqd, 1488 bool move_to_err, u32 modify_flags) 1489 { 1490 struct roce_modify_qp_req_ramrod_data *p_ramrod; 1491 struct qed_sp_init_data init_data; 1492 struct qed_spq_entry *p_ent; 1493 int rc; 1494 1495 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); 1496 1497 if (move_to_err && !(qp->req_offloaded)) 1498 return 0; 1499 1500 /* Get SPQ entry */ 1501 memset(&init_data, 0, sizeof(init_data)); 1502 init_data.cid = qp->icid + 1; 1503 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 1504 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 1505 1506 rc = qed_sp_init_request(p_hwfn, &p_ent, 1507 ROCE_EVENT_MODIFY_QP, 1508 PROTOCOLID_ROCE, &init_data); 1509 if (rc) { 1510 DP_NOTICE(p_hwfn, "rc = %d\n", rc); 1511 return rc; 1512 } 1513 1514 p_ramrod = &p_ent->ramrod.roce_modify_qp_req; 1515 1516 p_ramrod->flags = 0; 1517 1518 SET_FIELD(p_ramrod->flags, 1519 ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_ERR_FLG, move_to_err); 1520 1521 SET_FIELD(p_ramrod->flags, 1522 ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_SQD_FLG, move_to_sqd); 1523 1524 SET_FIELD(p_ramrod->flags, 1525 ROCE_MODIFY_QP_REQ_RAMROD_DATA_EN_SQD_ASYNC_NOTIFY, 1526 qp->sqd_async); 1527 1528 SET_FIELD(p_ramrod->flags, 1529 ROCE_MODIFY_QP_REQ_RAMROD_DATA_P_KEY_FLG, 1530 GET_FIELD(modify_flags, QED_ROCE_MODIFY_QP_VALID_PKEY)); 1531 1532 SET_FIELD(p_ramrod->flags, 1533 ROCE_MODIFY_QP_REQ_RAMROD_DATA_ADDRESS_VECTOR_FLG, 1534 GET_FIELD(modify_flags, 1535 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR)); 1536 1537 SET_FIELD(p_ramrod->flags, 1538 ROCE_MODIFY_QP_REQ_RAMROD_DATA_MAX_ORD_FLG, 1539 GET_FIELD(modify_flags, 1540 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ)); 1541 1542 SET_FIELD(p_ramrod->flags, 1543 ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT_FLG, 1544 GET_FIELD(modify_flags, 1545 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT)); 1546 1547 SET_FIELD(p_ramrod->flags, 1548 ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT_FLG, 1549 GET_FIELD(modify_flags, QED_ROCE_MODIFY_QP_VALID_RETRY_CNT)); 1550 1551 SET_FIELD(p_ramrod->flags, 1552 ROCE_MODIFY_QP_REQ_RAMROD_DATA_ACK_TIMEOUT_FLG, 1553 GET_FIELD(modify_flags, 1554 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT)); 1555 1556 p_ramrod->fields = 0; 1557 SET_FIELD(p_ramrod->fields, 1558 ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT, qp->retry_cnt); 1559 1560 SET_FIELD(p_ramrod->fields, 1561 ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT, 1562 qp->rnr_retry_cnt); 1563 1564 p_ramrod->max_ord = qp->max_rd_atomic_req; 1565 p_ramrod->traffic_class = qp->traffic_class_tos; 1566 p_ramrod->hop_limit = qp->hop_limit_ttl; 1567 p_ramrod->p_key = cpu_to_le16(qp->pkey); 1568 p_ramrod->flow_label = cpu_to_le32(qp->flow_label); 1569 p_ramrod->ack_timeout_val = cpu_to_le32(qp->ack_timeout); 1570 p_ramrod->mtu = cpu_to_le16(qp->mtu); 1571 qed_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid); 1572 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1573 1574 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Modify requester, rc = %d\n", rc); 1575 return rc; 1576 } 1577 1578 static int qed_roce_sp_destroy_qp_responder(struct qed_hwfn *p_hwfn, 1579 struct qed_rdma_qp *qp, 1580 u32 *num_invalidated_mw) 1581 { 1582 struct roce_destroy_qp_resp_output_params *p_ramrod_res; 1583 struct roce_destroy_qp_resp_ramrod_data *p_ramrod; 1584 struct qed_sp_init_data init_data; 1585 struct qed_spq_entry *p_ent; 1586 dma_addr_t ramrod_res_phys; 1587 int rc; 1588 1589 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); 1590 1591 if (!qp->resp_offloaded) 1592 return 0; 1593 1594 /* Get SPQ entry */ 1595 memset(&init_data, 0, sizeof(init_data)); 1596 init_data.cid = qp->icid; 1597 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 1598 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 1599 1600 rc = qed_sp_init_request(p_hwfn, &p_ent, 1601 ROCE_RAMROD_DESTROY_QP, 1602 PROTOCOLID_ROCE, &init_data); 1603 if (rc) 1604 return rc; 1605 1606 p_ramrod = &p_ent->ramrod.roce_destroy_qp_resp; 1607 1608 p_ramrod_res = (struct roce_destroy_qp_resp_output_params *) 1609 dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_ramrod_res), 1610 &ramrod_res_phys, GFP_KERNEL); 1611 1612 if (!p_ramrod_res) { 1613 rc = -ENOMEM; 1614 DP_NOTICE(p_hwfn, 1615 "qed destroy responder failed: cannot allocate memory (ramrod). rc = %d\n", 1616 rc); 1617 return rc; 1618 } 1619 1620 DMA_REGPAIR_LE(p_ramrod->output_params_addr, ramrod_res_phys); 1621 1622 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1623 if (rc) 1624 goto err; 1625 1626 *num_invalidated_mw = le32_to_cpu(p_ramrod_res->num_invalidated_mw); 1627 1628 /* Free IRQ - only if ramrod succeeded, in case FW is still using it */ 1629 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 1630 qp->irq_num_pages * RDMA_RING_PAGE_SIZE, 1631 qp->irq, qp->irq_phys_addr); 1632 1633 qp->resp_offloaded = false; 1634 1635 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Destroy responder, rc = %d\n", rc); 1636 1637 err: 1638 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 1639 sizeof(struct roce_destroy_qp_resp_output_params), 1640 p_ramrod_res, ramrod_res_phys); 1641 1642 return rc; 1643 } 1644 1645 static int qed_roce_sp_destroy_qp_requester(struct qed_hwfn *p_hwfn, 1646 struct qed_rdma_qp *qp, 1647 u32 *num_bound_mw) 1648 { 1649 struct roce_destroy_qp_req_output_params *p_ramrod_res; 1650 struct roce_destroy_qp_req_ramrod_data *p_ramrod; 1651 struct qed_sp_init_data init_data; 1652 struct qed_spq_entry *p_ent; 1653 dma_addr_t ramrod_res_phys; 1654 int rc = -ENOMEM; 1655 1656 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); 1657 1658 if (!qp->req_offloaded) 1659 return 0; 1660 1661 p_ramrod_res = (struct roce_destroy_qp_req_output_params *) 1662 dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 1663 sizeof(*p_ramrod_res), 1664 &ramrod_res_phys, GFP_KERNEL); 1665 if (!p_ramrod_res) { 1666 DP_NOTICE(p_hwfn, 1667 "qed destroy requester failed: cannot allocate memory (ramrod)\n"); 1668 return rc; 1669 } 1670 1671 /* Get SPQ entry */ 1672 memset(&init_data, 0, sizeof(init_data)); 1673 init_data.cid = qp->icid + 1; 1674 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 1675 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 1676 1677 rc = qed_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_DESTROY_QP, 1678 PROTOCOLID_ROCE, &init_data); 1679 if (rc) 1680 goto err; 1681 1682 p_ramrod = &p_ent->ramrod.roce_destroy_qp_req; 1683 DMA_REGPAIR_LE(p_ramrod->output_params_addr, ramrod_res_phys); 1684 1685 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1686 if (rc) 1687 goto err; 1688 1689 *num_bound_mw = le32_to_cpu(p_ramrod_res->num_bound_mw); 1690 1691 /* Free ORQ - only if ramrod succeeded, in case FW is still using it */ 1692 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 1693 qp->orq_num_pages * RDMA_RING_PAGE_SIZE, 1694 qp->orq, qp->orq_phys_addr); 1695 1696 qp->req_offloaded = false; 1697 1698 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Destroy requester, rc = %d\n", rc); 1699 1700 err: 1701 dma_free_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_ramrod_res), 1702 p_ramrod_res, ramrod_res_phys); 1703 1704 return rc; 1705 } 1706 1707 static int qed_roce_query_qp(struct qed_hwfn *p_hwfn, 1708 struct qed_rdma_qp *qp, 1709 struct qed_rdma_query_qp_out_params *out_params) 1710 { 1711 struct roce_query_qp_resp_output_params *p_resp_ramrod_res; 1712 struct roce_query_qp_req_output_params *p_req_ramrod_res; 1713 struct roce_query_qp_resp_ramrod_data *p_resp_ramrod; 1714 struct roce_query_qp_req_ramrod_data *p_req_ramrod; 1715 struct qed_sp_init_data init_data; 1716 dma_addr_t resp_ramrod_res_phys; 1717 dma_addr_t req_ramrod_res_phys; 1718 struct qed_spq_entry *p_ent; 1719 bool rq_err_state; 1720 bool sq_err_state; 1721 bool sq_draining; 1722 int rc = -ENOMEM; 1723 1724 if ((!(qp->resp_offloaded)) && (!(qp->req_offloaded))) { 1725 /* We can't send ramrod to the fw since this qp wasn't offloaded 1726 * to the fw yet 1727 */ 1728 out_params->draining = false; 1729 out_params->rq_psn = qp->rq_psn; 1730 out_params->sq_psn = qp->sq_psn; 1731 out_params->state = qp->cur_state; 1732 1733 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "No QPs as no offload\n"); 1734 return 0; 1735 } 1736 1737 if (!(qp->resp_offloaded)) { 1738 DP_NOTICE(p_hwfn, 1739 "The responder's qp should be offloded before requester's\n"); 1740 return -EINVAL; 1741 } 1742 1743 /* Send a query responder ramrod to FW to get RQ-PSN and state */ 1744 p_resp_ramrod_res = (struct roce_query_qp_resp_output_params *) 1745 dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 1746 sizeof(*p_resp_ramrod_res), 1747 &resp_ramrod_res_phys, GFP_KERNEL); 1748 if (!p_resp_ramrod_res) { 1749 DP_NOTICE(p_hwfn, 1750 "qed query qp failed: cannot allocate memory (ramrod)\n"); 1751 return rc; 1752 } 1753 1754 /* Get SPQ entry */ 1755 memset(&init_data, 0, sizeof(init_data)); 1756 init_data.cid = qp->icid; 1757 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 1758 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 1759 rc = qed_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_QUERY_QP, 1760 PROTOCOLID_ROCE, &init_data); 1761 if (rc) 1762 goto err_resp; 1763 1764 p_resp_ramrod = &p_ent->ramrod.roce_query_qp_resp; 1765 DMA_REGPAIR_LE(p_resp_ramrod->output_params_addr, resp_ramrod_res_phys); 1766 1767 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1768 if (rc) 1769 goto err_resp; 1770 1771 out_params->rq_psn = le32_to_cpu(p_resp_ramrod_res->psn); 1772 rq_err_state = GET_FIELD(le32_to_cpu(p_resp_ramrod_res->err_flag), 1773 ROCE_QUERY_QP_RESP_OUTPUT_PARAMS_ERROR_FLG); 1774 1775 dma_free_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_resp_ramrod_res), 1776 p_resp_ramrod_res, resp_ramrod_res_phys); 1777 1778 if (!(qp->req_offloaded)) { 1779 /* Don't send query qp for the requester */ 1780 out_params->sq_psn = qp->sq_psn; 1781 out_params->draining = false; 1782 1783 if (rq_err_state) 1784 qp->cur_state = QED_ROCE_QP_STATE_ERR; 1785 1786 out_params->state = qp->cur_state; 1787 1788 return 0; 1789 } 1790 1791 /* Send a query requester ramrod to FW to get SQ-PSN and state */ 1792 p_req_ramrod_res = (struct roce_query_qp_req_output_params *) 1793 dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 1794 sizeof(*p_req_ramrod_res), 1795 &req_ramrod_res_phys, 1796 GFP_KERNEL); 1797 if (!p_req_ramrod_res) { 1798 rc = -ENOMEM; 1799 DP_NOTICE(p_hwfn, 1800 "qed query qp failed: cannot allocate memory (ramrod)\n"); 1801 return rc; 1802 } 1803 1804 /* Get SPQ entry */ 1805 init_data.cid = qp->icid + 1; 1806 rc = qed_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_QUERY_QP, 1807 PROTOCOLID_ROCE, &init_data); 1808 if (rc) 1809 goto err_req; 1810 1811 p_req_ramrod = &p_ent->ramrod.roce_query_qp_req; 1812 DMA_REGPAIR_LE(p_req_ramrod->output_params_addr, req_ramrod_res_phys); 1813 1814 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1815 if (rc) 1816 goto err_req; 1817 1818 out_params->sq_psn = le32_to_cpu(p_req_ramrod_res->psn); 1819 sq_err_state = GET_FIELD(le32_to_cpu(p_req_ramrod_res->flags), 1820 ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_ERR_FLG); 1821 sq_draining = 1822 GET_FIELD(le32_to_cpu(p_req_ramrod_res->flags), 1823 ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_SQ_DRAINING_FLG); 1824 1825 dma_free_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_req_ramrod_res), 1826 p_req_ramrod_res, req_ramrod_res_phys); 1827 1828 out_params->draining = false; 1829 1830 if (rq_err_state) 1831 qp->cur_state = QED_ROCE_QP_STATE_ERR; 1832 else if (sq_err_state) 1833 qp->cur_state = QED_ROCE_QP_STATE_SQE; 1834 else if (sq_draining) 1835 out_params->draining = true; 1836 out_params->state = qp->cur_state; 1837 1838 return 0; 1839 1840 err_req: 1841 dma_free_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_req_ramrod_res), 1842 p_req_ramrod_res, req_ramrod_res_phys); 1843 return rc; 1844 err_resp: 1845 dma_free_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_resp_ramrod_res), 1846 p_resp_ramrod_res, resp_ramrod_res_phys); 1847 return rc; 1848 } 1849 1850 static int qed_roce_destroy_qp(struct qed_hwfn *p_hwfn, struct qed_rdma_qp *qp) 1851 { 1852 struct qed_rdma_info *p_rdma_info = p_hwfn->p_rdma_info; 1853 u32 num_invalidated_mw = 0; 1854 u32 num_bound_mw = 0; 1855 u32 start_cid; 1856 int rc; 1857 1858 /* Destroys the specified QP */ 1859 if ((qp->cur_state != QED_ROCE_QP_STATE_RESET) && 1860 (qp->cur_state != QED_ROCE_QP_STATE_ERR) && 1861 (qp->cur_state != QED_ROCE_QP_STATE_INIT)) { 1862 DP_NOTICE(p_hwfn, 1863 "QP must be in error, reset or init state before destroying it\n"); 1864 return -EINVAL; 1865 } 1866 1867 if (qp->cur_state != QED_ROCE_QP_STATE_RESET) { 1868 rc = qed_roce_sp_destroy_qp_responder(p_hwfn, qp, 1869 &num_invalidated_mw); 1870 if (rc) 1871 return rc; 1872 1873 /* Send destroy requester ramrod */ 1874 rc = qed_roce_sp_destroy_qp_requester(p_hwfn, qp, 1875 &num_bound_mw); 1876 if (rc) 1877 return rc; 1878 1879 if (num_invalidated_mw != num_bound_mw) { 1880 DP_NOTICE(p_hwfn, 1881 "number of invalidate memory windows is different from bounded ones\n"); 1882 return -EINVAL; 1883 } 1884 1885 spin_lock_bh(&p_rdma_info->lock); 1886 1887 start_cid = qed_cxt_get_proto_cid_start(p_hwfn, 1888 p_rdma_info->proto); 1889 1890 /* Release responder's icid */ 1891 qed_bmap_release_id(p_hwfn, &p_rdma_info->cid_map, 1892 qp->icid - start_cid); 1893 1894 /* Release requester's icid */ 1895 qed_bmap_release_id(p_hwfn, &p_rdma_info->cid_map, 1896 qp->icid + 1 - start_cid); 1897 1898 spin_unlock_bh(&p_rdma_info->lock); 1899 } 1900 1901 return 0; 1902 } 1903 1904 static int qed_rdma_query_qp(void *rdma_cxt, 1905 struct qed_rdma_qp *qp, 1906 struct qed_rdma_query_qp_out_params *out_params) 1907 { 1908 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 1909 int rc; 1910 1911 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); 1912 1913 /* The following fields are filled in from qp and not FW as they can't 1914 * be modified by FW 1915 */ 1916 out_params->mtu = qp->mtu; 1917 out_params->dest_qp = qp->dest_qp; 1918 out_params->incoming_atomic_en = qp->incoming_atomic_en; 1919 out_params->e2e_flow_control_en = qp->e2e_flow_control_en; 1920 out_params->incoming_rdma_read_en = qp->incoming_rdma_read_en; 1921 out_params->incoming_rdma_write_en = qp->incoming_rdma_write_en; 1922 out_params->dgid = qp->dgid; 1923 out_params->flow_label = qp->flow_label; 1924 out_params->hop_limit_ttl = qp->hop_limit_ttl; 1925 out_params->traffic_class_tos = qp->traffic_class_tos; 1926 out_params->timeout = qp->ack_timeout; 1927 out_params->rnr_retry = qp->rnr_retry_cnt; 1928 out_params->retry_cnt = qp->retry_cnt; 1929 out_params->min_rnr_nak_timer = qp->min_rnr_nak_timer; 1930 out_params->pkey_index = 0; 1931 out_params->max_rd_atomic = qp->max_rd_atomic_req; 1932 out_params->max_dest_rd_atomic = qp->max_rd_atomic_resp; 1933 out_params->sqd_async = qp->sqd_async; 1934 1935 rc = qed_roce_query_qp(p_hwfn, qp, out_params); 1936 1937 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Query QP, rc = %d\n", rc); 1938 return rc; 1939 } 1940 1941 static int qed_rdma_destroy_qp(void *rdma_cxt, struct qed_rdma_qp *qp) 1942 { 1943 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 1944 int rc = 0; 1945 1946 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); 1947 1948 rc = qed_roce_destroy_qp(p_hwfn, qp); 1949 1950 /* free qp params struct */ 1951 kfree(qp); 1952 1953 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QP destroyed\n"); 1954 return rc; 1955 } 1956 1957 static struct qed_rdma_qp * 1958 qed_rdma_create_qp(void *rdma_cxt, 1959 struct qed_rdma_create_qp_in_params *in_params, 1960 struct qed_rdma_create_qp_out_params *out_params) 1961 { 1962 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 1963 struct qed_rdma_qp *qp; 1964 u8 max_stats_queues; 1965 int rc; 1966 1967 if (!rdma_cxt || !in_params || !out_params || !p_hwfn->p_rdma_info) { 1968 DP_ERR(p_hwfn->cdev, 1969 "qed roce create qp failed due to NULL entry (rdma_cxt=%p, in=%p, out=%p, roce_info=?\n", 1970 rdma_cxt, in_params, out_params); 1971 return NULL; 1972 } 1973 1974 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 1975 "qed rdma create qp called with qp_handle = %08x%08x\n", 1976 in_params->qp_handle_hi, in_params->qp_handle_lo); 1977 1978 /* Some sanity checks... */ 1979 max_stats_queues = p_hwfn->p_rdma_info->dev->max_stats_queues; 1980 if (in_params->stats_queue >= max_stats_queues) { 1981 DP_ERR(p_hwfn->cdev, 1982 "qed rdma create qp failed due to invalid statistics queue %d. maximum is %d\n", 1983 in_params->stats_queue, max_stats_queues); 1984 return NULL; 1985 } 1986 1987 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 1988 if (!qp) { 1989 DP_NOTICE(p_hwfn, "Failed to allocate qed_rdma_qp\n"); 1990 return NULL; 1991 } 1992 1993 rc = qed_roce_alloc_cid(p_hwfn, &qp->icid); 1994 qp->qpid = ((0xFF << 16) | qp->icid); 1995 1996 DP_INFO(p_hwfn, "ROCE qpid=%x\n", qp->qpid); 1997 1998 if (rc) { 1999 kfree(qp); 2000 return NULL; 2001 } 2002 2003 qp->cur_state = QED_ROCE_QP_STATE_RESET; 2004 qp->qp_handle.hi = cpu_to_le32(in_params->qp_handle_hi); 2005 qp->qp_handle.lo = cpu_to_le32(in_params->qp_handle_lo); 2006 qp->qp_handle_async.hi = cpu_to_le32(in_params->qp_handle_async_hi); 2007 qp->qp_handle_async.lo = cpu_to_le32(in_params->qp_handle_async_lo); 2008 qp->use_srq = in_params->use_srq; 2009 qp->signal_all = in_params->signal_all; 2010 qp->fmr_and_reserved_lkey = in_params->fmr_and_reserved_lkey; 2011 qp->pd = in_params->pd; 2012 qp->dpi = in_params->dpi; 2013 qp->sq_cq_id = in_params->sq_cq_id; 2014 qp->sq_num_pages = in_params->sq_num_pages; 2015 qp->sq_pbl_ptr = in_params->sq_pbl_ptr; 2016 qp->rq_cq_id = in_params->rq_cq_id; 2017 qp->rq_num_pages = in_params->rq_num_pages; 2018 qp->rq_pbl_ptr = in_params->rq_pbl_ptr; 2019 qp->srq_id = in_params->srq_id; 2020 qp->req_offloaded = false; 2021 qp->resp_offloaded = false; 2022 qp->e2e_flow_control_en = qp->use_srq ? false : true; 2023 qp->stats_queue = in_params->stats_queue; 2024 2025 out_params->icid = qp->icid; 2026 out_params->qp_id = qp->qpid; 2027 2028 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Create QP, rc = %d\n", rc); 2029 return qp; 2030 } 2031 2032 static int qed_roce_modify_qp(struct qed_hwfn *p_hwfn, 2033 struct qed_rdma_qp *qp, 2034 enum qed_roce_qp_state prev_state, 2035 struct qed_rdma_modify_qp_in_params *params) 2036 { 2037 u32 num_invalidated_mw = 0, num_bound_mw = 0; 2038 int rc = 0; 2039 2040 /* Perform additional operations according to the current state and the 2041 * next state 2042 */ 2043 if (((prev_state == QED_ROCE_QP_STATE_INIT) || 2044 (prev_state == QED_ROCE_QP_STATE_RESET)) && 2045 (qp->cur_state == QED_ROCE_QP_STATE_RTR)) { 2046 /* Init->RTR or Reset->RTR */ 2047 rc = qed_roce_sp_create_responder(p_hwfn, qp); 2048 return rc; 2049 } else if ((prev_state == QED_ROCE_QP_STATE_RTR) && 2050 (qp->cur_state == QED_ROCE_QP_STATE_RTS)) { 2051 /* RTR-> RTS */ 2052 rc = qed_roce_sp_create_requester(p_hwfn, qp); 2053 if (rc) 2054 return rc; 2055 2056 /* Send modify responder ramrod */ 2057 rc = qed_roce_sp_modify_responder(p_hwfn, qp, false, 2058 params->modify_flags); 2059 return rc; 2060 } else if ((prev_state == QED_ROCE_QP_STATE_RTS) && 2061 (qp->cur_state == QED_ROCE_QP_STATE_RTS)) { 2062 /* RTS->RTS */ 2063 rc = qed_roce_sp_modify_responder(p_hwfn, qp, false, 2064 params->modify_flags); 2065 if (rc) 2066 return rc; 2067 2068 rc = qed_roce_sp_modify_requester(p_hwfn, qp, false, false, 2069 params->modify_flags); 2070 return rc; 2071 } else if ((prev_state == QED_ROCE_QP_STATE_RTS) && 2072 (qp->cur_state == QED_ROCE_QP_STATE_SQD)) { 2073 /* RTS->SQD */ 2074 rc = qed_roce_sp_modify_requester(p_hwfn, qp, true, false, 2075 params->modify_flags); 2076 return rc; 2077 } else if ((prev_state == QED_ROCE_QP_STATE_SQD) && 2078 (qp->cur_state == QED_ROCE_QP_STATE_SQD)) { 2079 /* SQD->SQD */ 2080 rc = qed_roce_sp_modify_responder(p_hwfn, qp, false, 2081 params->modify_flags); 2082 if (rc) 2083 return rc; 2084 2085 rc = qed_roce_sp_modify_requester(p_hwfn, qp, false, false, 2086 params->modify_flags); 2087 return rc; 2088 } else if ((prev_state == QED_ROCE_QP_STATE_SQD) && 2089 (qp->cur_state == QED_ROCE_QP_STATE_RTS)) { 2090 /* SQD->RTS */ 2091 rc = qed_roce_sp_modify_responder(p_hwfn, qp, false, 2092 params->modify_flags); 2093 if (rc) 2094 return rc; 2095 2096 rc = qed_roce_sp_modify_requester(p_hwfn, qp, false, false, 2097 params->modify_flags); 2098 2099 return rc; 2100 } else if (qp->cur_state == QED_ROCE_QP_STATE_ERR || 2101 qp->cur_state == QED_ROCE_QP_STATE_SQE) { 2102 /* ->ERR */ 2103 rc = qed_roce_sp_modify_responder(p_hwfn, qp, true, 2104 params->modify_flags); 2105 if (rc) 2106 return rc; 2107 2108 rc = qed_roce_sp_modify_requester(p_hwfn, qp, false, true, 2109 params->modify_flags); 2110 return rc; 2111 } else if (qp->cur_state == QED_ROCE_QP_STATE_RESET) { 2112 /* Any state -> RESET */ 2113 2114 rc = qed_roce_sp_destroy_qp_responder(p_hwfn, qp, 2115 &num_invalidated_mw); 2116 if (rc) 2117 return rc; 2118 2119 rc = qed_roce_sp_destroy_qp_requester(p_hwfn, qp, 2120 &num_bound_mw); 2121 2122 if (num_invalidated_mw != num_bound_mw) { 2123 DP_NOTICE(p_hwfn, 2124 "number of invalidate memory windows is different from bounded ones\n"); 2125 return -EINVAL; 2126 } 2127 } else { 2128 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "0\n"); 2129 } 2130 2131 return rc; 2132 } 2133 2134 static int qed_rdma_modify_qp(void *rdma_cxt, 2135 struct qed_rdma_qp *qp, 2136 struct qed_rdma_modify_qp_in_params *params) 2137 { 2138 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 2139 enum qed_roce_qp_state prev_state; 2140 int rc = 0; 2141 2142 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x params->new_state=%d\n", 2143 qp->icid, params->new_state); 2144 2145 if (rc) { 2146 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc); 2147 return rc; 2148 } 2149 2150 if (GET_FIELD(params->modify_flags, 2151 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN)) { 2152 qp->incoming_rdma_read_en = params->incoming_rdma_read_en; 2153 qp->incoming_rdma_write_en = params->incoming_rdma_write_en; 2154 qp->incoming_atomic_en = params->incoming_atomic_en; 2155 } 2156 2157 /* Update QP structure with the updated values */ 2158 if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_ROCE_MODE)) 2159 qp->roce_mode = params->roce_mode; 2160 if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_PKEY)) 2161 qp->pkey = params->pkey; 2162 if (GET_FIELD(params->modify_flags, 2163 QED_ROCE_MODIFY_QP_VALID_E2E_FLOW_CONTROL_EN)) 2164 qp->e2e_flow_control_en = params->e2e_flow_control_en; 2165 if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_DEST_QP)) 2166 qp->dest_qp = params->dest_qp; 2167 if (GET_FIELD(params->modify_flags, 2168 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR)) { 2169 /* Indicates that the following parameters have changed: 2170 * Traffic class, flow label, hop limit, source GID, 2171 * destination GID, loopback indicator 2172 */ 2173 qp->traffic_class_tos = params->traffic_class_tos; 2174 qp->flow_label = params->flow_label; 2175 qp->hop_limit_ttl = params->hop_limit_ttl; 2176 2177 qp->sgid = params->sgid; 2178 qp->dgid = params->dgid; 2179 qp->udp_src_port = 0; 2180 qp->vlan_id = params->vlan_id; 2181 qp->mtu = params->mtu; 2182 qp->lb_indication = params->lb_indication; 2183 memcpy((u8 *)&qp->remote_mac_addr[0], 2184 (u8 *)¶ms->remote_mac_addr[0], ETH_ALEN); 2185 if (params->use_local_mac) { 2186 memcpy((u8 *)&qp->local_mac_addr[0], 2187 (u8 *)¶ms->local_mac_addr[0], ETH_ALEN); 2188 } else { 2189 memcpy((u8 *)&qp->local_mac_addr[0], 2190 (u8 *)&p_hwfn->hw_info.hw_mac_addr, ETH_ALEN); 2191 } 2192 } 2193 if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_RQ_PSN)) 2194 qp->rq_psn = params->rq_psn; 2195 if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_SQ_PSN)) 2196 qp->sq_psn = params->sq_psn; 2197 if (GET_FIELD(params->modify_flags, 2198 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ)) 2199 qp->max_rd_atomic_req = params->max_rd_atomic_req; 2200 if (GET_FIELD(params->modify_flags, 2201 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP)) 2202 qp->max_rd_atomic_resp = params->max_rd_atomic_resp; 2203 if (GET_FIELD(params->modify_flags, 2204 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT)) 2205 qp->ack_timeout = params->ack_timeout; 2206 if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_RETRY_CNT)) 2207 qp->retry_cnt = params->retry_cnt; 2208 if (GET_FIELD(params->modify_flags, 2209 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT)) 2210 qp->rnr_retry_cnt = params->rnr_retry_cnt; 2211 if (GET_FIELD(params->modify_flags, 2212 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER)) 2213 qp->min_rnr_nak_timer = params->min_rnr_nak_timer; 2214 2215 qp->sqd_async = params->sqd_async; 2216 2217 prev_state = qp->cur_state; 2218 if (GET_FIELD(params->modify_flags, 2219 QED_RDMA_MODIFY_QP_VALID_NEW_STATE)) { 2220 qp->cur_state = params->new_state; 2221 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "qp->cur_state=%d\n", 2222 qp->cur_state); 2223 } 2224 2225 rc = qed_roce_modify_qp(p_hwfn, qp, prev_state, params); 2226 2227 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Modify QP, rc = %d\n", rc); 2228 return rc; 2229 } 2230 2231 static int 2232 qed_rdma_register_tid(void *rdma_cxt, 2233 struct qed_rdma_register_tid_in_params *params) 2234 { 2235 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 2236 struct rdma_register_tid_ramrod_data *p_ramrod; 2237 struct qed_sp_init_data init_data; 2238 struct qed_spq_entry *p_ent; 2239 enum rdma_tid_type tid_type; 2240 u8 fw_return_code; 2241 int rc; 2242 2243 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "itid = %08x\n", params->itid); 2244 2245 /* Get SPQ entry */ 2246 memset(&init_data, 0, sizeof(init_data)); 2247 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 2248 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 2249 2250 rc = qed_sp_init_request(p_hwfn, &p_ent, RDMA_RAMROD_REGISTER_MR, 2251 p_hwfn->p_rdma_info->proto, &init_data); 2252 if (rc) { 2253 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc); 2254 return rc; 2255 } 2256 2257 if (p_hwfn->p_rdma_info->last_tid < params->itid) 2258 p_hwfn->p_rdma_info->last_tid = params->itid; 2259 2260 p_ramrod = &p_ent->ramrod.rdma_register_tid; 2261 2262 p_ramrod->flags = 0; 2263 SET_FIELD(p_ramrod->flags, 2264 RDMA_REGISTER_TID_RAMROD_DATA_TWO_LEVEL_PBL, 2265 params->pbl_two_level); 2266 2267 SET_FIELD(p_ramrod->flags, 2268 RDMA_REGISTER_TID_RAMROD_DATA_ZERO_BASED, params->zbva); 2269 2270 SET_FIELD(p_ramrod->flags, 2271 RDMA_REGISTER_TID_RAMROD_DATA_PHY_MR, params->phy_mr); 2272 2273 /* Don't initialize D/C field, as it may override other bits. */ 2274 if (!(params->tid_type == QED_RDMA_TID_FMR) && !(params->dma_mr)) 2275 SET_FIELD(p_ramrod->flags, 2276 RDMA_REGISTER_TID_RAMROD_DATA_PAGE_SIZE_LOG, 2277 params->page_size_log - 12); 2278 2279 SET_FIELD(p_ramrod->flags, 2280 RDMA_REGISTER_TID_RAMROD_DATA_MAX_ID, 2281 p_hwfn->p_rdma_info->last_tid); 2282 2283 SET_FIELD(p_ramrod->flags, 2284 RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_READ, 2285 params->remote_read); 2286 2287 SET_FIELD(p_ramrod->flags, 2288 RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_WRITE, 2289 params->remote_write); 2290 2291 SET_FIELD(p_ramrod->flags, 2292 RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_ATOMIC, 2293 params->remote_atomic); 2294 2295 SET_FIELD(p_ramrod->flags, 2296 RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_WRITE, 2297 params->local_write); 2298 2299 SET_FIELD(p_ramrod->flags, 2300 RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_READ, params->local_read); 2301 2302 SET_FIELD(p_ramrod->flags, 2303 RDMA_REGISTER_TID_RAMROD_DATA_ENABLE_MW_BIND, 2304 params->mw_bind); 2305 2306 SET_FIELD(p_ramrod->flags1, 2307 RDMA_REGISTER_TID_RAMROD_DATA_PBL_PAGE_SIZE_LOG, 2308 params->pbl_page_size_log - 12); 2309 2310 SET_FIELD(p_ramrod->flags2, 2311 RDMA_REGISTER_TID_RAMROD_DATA_DMA_MR, params->dma_mr); 2312 2313 switch (params->tid_type) { 2314 case QED_RDMA_TID_REGISTERED_MR: 2315 tid_type = RDMA_TID_REGISTERED_MR; 2316 break; 2317 case QED_RDMA_TID_FMR: 2318 tid_type = RDMA_TID_FMR; 2319 break; 2320 case QED_RDMA_TID_MW_TYPE1: 2321 tid_type = RDMA_TID_MW_TYPE1; 2322 break; 2323 case QED_RDMA_TID_MW_TYPE2A: 2324 tid_type = RDMA_TID_MW_TYPE2A; 2325 break; 2326 default: 2327 rc = -EINVAL; 2328 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc); 2329 return rc; 2330 } 2331 SET_FIELD(p_ramrod->flags1, 2332 RDMA_REGISTER_TID_RAMROD_DATA_TID_TYPE, tid_type); 2333 2334 p_ramrod->itid = cpu_to_le32(params->itid); 2335 p_ramrod->key = params->key; 2336 p_ramrod->pd = cpu_to_le16(params->pd); 2337 p_ramrod->length_hi = (u8)(params->length >> 32); 2338 p_ramrod->length_lo = DMA_LO_LE(params->length); 2339 if (params->zbva) { 2340 /* Lower 32 bits of the registered MR address. 2341 * In case of zero based MR, will hold FBO 2342 */ 2343 p_ramrod->va.hi = 0; 2344 p_ramrod->va.lo = cpu_to_le32(params->fbo); 2345 } else { 2346 DMA_REGPAIR_LE(p_ramrod->va, params->vaddr); 2347 } 2348 DMA_REGPAIR_LE(p_ramrod->pbl_base, params->pbl_ptr); 2349 2350 /* DIF */ 2351 if (params->dif_enabled) { 2352 SET_FIELD(p_ramrod->flags2, 2353 RDMA_REGISTER_TID_RAMROD_DATA_DIF_ON_HOST_FLG, 1); 2354 DMA_REGPAIR_LE(p_ramrod->dif_error_addr, 2355 params->dif_error_addr); 2356 DMA_REGPAIR_LE(p_ramrod->dif_runt_addr, params->dif_runt_addr); 2357 } 2358 2359 rc = qed_spq_post(p_hwfn, p_ent, &fw_return_code); 2360 2361 if (fw_return_code != RDMA_RETURN_OK) { 2362 DP_NOTICE(p_hwfn, "fw_return_code = %d\n", fw_return_code); 2363 return -EINVAL; 2364 } 2365 2366 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Register TID, rc = %d\n", rc); 2367 return rc; 2368 } 2369 2370 static int qed_rdma_deregister_tid(void *rdma_cxt, u32 itid) 2371 { 2372 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 2373 struct rdma_deregister_tid_ramrod_data *p_ramrod; 2374 struct qed_sp_init_data init_data; 2375 struct qed_spq_entry *p_ent; 2376 struct qed_ptt *p_ptt; 2377 u8 fw_return_code; 2378 int rc; 2379 2380 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "itid = %08x\n", itid); 2381 2382 /* Get SPQ entry */ 2383 memset(&init_data, 0, sizeof(init_data)); 2384 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 2385 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 2386 2387 rc = qed_sp_init_request(p_hwfn, &p_ent, RDMA_RAMROD_DEREGISTER_MR, 2388 p_hwfn->p_rdma_info->proto, &init_data); 2389 if (rc) { 2390 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc); 2391 return rc; 2392 } 2393 2394 p_ramrod = &p_ent->ramrod.rdma_deregister_tid; 2395 p_ramrod->itid = cpu_to_le32(itid); 2396 2397 rc = qed_spq_post(p_hwfn, p_ent, &fw_return_code); 2398 if (rc) { 2399 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc); 2400 return rc; 2401 } 2402 2403 if (fw_return_code == RDMA_RETURN_DEREGISTER_MR_BAD_STATE_ERR) { 2404 DP_NOTICE(p_hwfn, "fw_return_code = %d\n", fw_return_code); 2405 return -EINVAL; 2406 } else if (fw_return_code == RDMA_RETURN_NIG_DRAIN_REQ) { 2407 /* Bit indicating that the TID is in use and a nig drain is 2408 * required before sending the ramrod again 2409 */ 2410 p_ptt = qed_ptt_acquire(p_hwfn); 2411 if (!p_ptt) { 2412 rc = -EBUSY; 2413 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 2414 "Failed to acquire PTT\n"); 2415 return rc; 2416 } 2417 2418 rc = qed_mcp_drain(p_hwfn, p_ptt); 2419 if (rc) { 2420 qed_ptt_release(p_hwfn, p_ptt); 2421 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 2422 "Drain failed\n"); 2423 return rc; 2424 } 2425 2426 qed_ptt_release(p_hwfn, p_ptt); 2427 2428 /* Resend the ramrod */ 2429 rc = qed_sp_init_request(p_hwfn, &p_ent, 2430 RDMA_RAMROD_DEREGISTER_MR, 2431 p_hwfn->p_rdma_info->proto, 2432 &init_data); 2433 if (rc) { 2434 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 2435 "Failed to init sp-element\n"); 2436 return rc; 2437 } 2438 2439 rc = qed_spq_post(p_hwfn, p_ent, &fw_return_code); 2440 if (rc) { 2441 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 2442 "Ramrod failed\n"); 2443 return rc; 2444 } 2445 2446 if (fw_return_code != RDMA_RETURN_OK) { 2447 DP_NOTICE(p_hwfn, "fw_return_code = %d\n", 2448 fw_return_code); 2449 return rc; 2450 } 2451 } 2452 2453 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "De-registered TID, rc = %d\n", rc); 2454 return rc; 2455 } 2456 2457 static void *qed_rdma_get_rdma_ctx(struct qed_dev *cdev) 2458 { 2459 return QED_LEADING_HWFN(cdev); 2460 } 2461 2462 static void qed_rdma_dpm_conf(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 2463 { 2464 u32 val; 2465 2466 val = (p_hwfn->dcbx_no_edpm || p_hwfn->db_bar_no_edpm) ? 0 : 1; 2467 2468 qed_wr(p_hwfn, p_ptt, DORQ_REG_PF_DPM_ENABLE, val); 2469 DP_VERBOSE(p_hwfn, (QED_MSG_DCB | QED_MSG_RDMA), 2470 "Changing DPM_EN state to %d (DCBX=%d, DB_BAR=%d)\n", 2471 val, p_hwfn->dcbx_no_edpm, p_hwfn->db_bar_no_edpm); 2472 } 2473 2474 void qed_rdma_dpm_bar(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 2475 { 2476 p_hwfn->db_bar_no_edpm = true; 2477 2478 qed_rdma_dpm_conf(p_hwfn, p_ptt); 2479 } 2480 2481 static int qed_rdma_start(void *rdma_cxt, 2482 struct qed_rdma_start_in_params *params) 2483 { 2484 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 2485 struct qed_ptt *p_ptt; 2486 int rc = -EBUSY; 2487 2488 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 2489 "desired_cnq = %08x\n", params->desired_cnq); 2490 2491 p_ptt = qed_ptt_acquire(p_hwfn); 2492 if (!p_ptt) 2493 goto err; 2494 2495 rc = qed_rdma_alloc(p_hwfn, p_ptt, params); 2496 if (rc) 2497 goto err1; 2498 2499 rc = qed_rdma_setup(p_hwfn, p_ptt, params); 2500 if (rc) 2501 goto err2; 2502 2503 qed_ptt_release(p_hwfn, p_ptt); 2504 2505 return rc; 2506 2507 err2: 2508 qed_rdma_free(p_hwfn); 2509 err1: 2510 qed_ptt_release(p_hwfn, p_ptt); 2511 err: 2512 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "RDMA start - error, rc = %d\n", rc); 2513 return rc; 2514 } 2515 2516 static int qed_rdma_init(struct qed_dev *cdev, 2517 struct qed_rdma_start_in_params *params) 2518 { 2519 return qed_rdma_start(QED_LEADING_HWFN(cdev), params); 2520 } 2521 2522 static void qed_rdma_remove_user(void *rdma_cxt, u16 dpi) 2523 { 2524 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 2525 2526 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "dpi = %08x\n", dpi); 2527 2528 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 2529 qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->dpi_map, dpi); 2530 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 2531 } 2532 2533 void qed_ll2b_complete_tx_gsi_packet(struct qed_hwfn *p_hwfn, 2534 u8 connection_handle, 2535 void *cookie, 2536 dma_addr_t first_frag_addr, 2537 bool b_last_fragment, bool b_last_packet) 2538 { 2539 struct qed_roce_ll2_packet *packet = cookie; 2540 struct qed_roce_ll2_info *roce_ll2 = p_hwfn->ll2; 2541 2542 roce_ll2->cbs.tx_cb(roce_ll2->cb_cookie, packet); 2543 } 2544 2545 void qed_ll2b_release_tx_gsi_packet(struct qed_hwfn *p_hwfn, 2546 u8 connection_handle, 2547 void *cookie, 2548 dma_addr_t first_frag_addr, 2549 bool b_last_fragment, bool b_last_packet) 2550 { 2551 qed_ll2b_complete_tx_gsi_packet(p_hwfn, connection_handle, 2552 cookie, first_frag_addr, 2553 b_last_fragment, b_last_packet); 2554 } 2555 2556 void qed_ll2b_complete_rx_gsi_packet(struct qed_hwfn *p_hwfn, 2557 u8 connection_handle, 2558 void *cookie, 2559 dma_addr_t rx_buf_addr, 2560 u16 data_length, 2561 u8 data_length_error, 2562 u16 parse_flags, 2563 u16 vlan, 2564 u32 src_mac_addr_hi, 2565 u16 src_mac_addr_lo, bool b_last_packet) 2566 { 2567 struct qed_roce_ll2_info *roce_ll2 = p_hwfn->ll2; 2568 struct qed_roce_ll2_rx_params params; 2569 struct qed_dev *cdev = p_hwfn->cdev; 2570 struct qed_roce_ll2_packet pkt; 2571 2572 DP_VERBOSE(cdev, 2573 QED_MSG_LL2, 2574 "roce ll2 rx complete: bus_addr=%p, len=%d, data_len_err=%d\n", 2575 (void *)(uintptr_t)rx_buf_addr, 2576 data_length, data_length_error); 2577 2578 memset(&pkt, 0, sizeof(pkt)); 2579 pkt.n_seg = 1; 2580 pkt.payload[0].baddr = rx_buf_addr; 2581 pkt.payload[0].len = data_length; 2582 2583 memset(¶ms, 0, sizeof(params)); 2584 params.vlan_id = vlan; 2585 *((u32 *)¶ms.smac[0]) = ntohl(src_mac_addr_hi); 2586 *((u16 *)¶ms.smac[4]) = ntohs(src_mac_addr_lo); 2587 2588 if (data_length_error) { 2589 DP_ERR(cdev, 2590 "roce ll2 rx complete: data length error %d, length=%d\n", 2591 data_length_error, data_length); 2592 params.rc = -EINVAL; 2593 } 2594 2595 roce_ll2->cbs.rx_cb(roce_ll2->cb_cookie, &pkt, ¶ms); 2596 } 2597 2598 static int qed_roce_ll2_set_mac_filter(struct qed_dev *cdev, 2599 u8 *old_mac_address, 2600 u8 *new_mac_address) 2601 { 2602 struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev); 2603 struct qed_ptt *p_ptt; 2604 int rc = 0; 2605 2606 if (!hwfn->ll2 || hwfn->ll2->handle == QED_LL2_UNUSED_HANDLE) { 2607 DP_ERR(cdev, 2608 "qed roce mac filter failed - roce_info/ll2 NULL\n"); 2609 return -EINVAL; 2610 } 2611 2612 p_ptt = qed_ptt_acquire(QED_LEADING_HWFN(cdev)); 2613 if (!p_ptt) { 2614 DP_ERR(cdev, 2615 "qed roce ll2 mac filter set: failed to acquire PTT\n"); 2616 return -EINVAL; 2617 } 2618 2619 mutex_lock(&hwfn->ll2->lock); 2620 if (old_mac_address) 2621 qed_llh_remove_mac_filter(QED_LEADING_HWFN(cdev), p_ptt, 2622 old_mac_address); 2623 if (new_mac_address) 2624 rc = qed_llh_add_mac_filter(QED_LEADING_HWFN(cdev), p_ptt, 2625 new_mac_address); 2626 mutex_unlock(&hwfn->ll2->lock); 2627 2628 qed_ptt_release(QED_LEADING_HWFN(cdev), p_ptt); 2629 2630 if (rc) 2631 DP_ERR(cdev, 2632 "qed roce ll2 mac filter set: failed to add mac filter\n"); 2633 2634 return rc; 2635 } 2636 2637 static int qed_roce_ll2_start(struct qed_dev *cdev, 2638 struct qed_roce_ll2_params *params) 2639 { 2640 struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev); 2641 struct qed_roce_ll2_info *roce_ll2; 2642 struct qed_ll2_conn ll2_params; 2643 int rc; 2644 2645 if (!params) { 2646 DP_ERR(cdev, "qed roce ll2 start: failed due to NULL params\n"); 2647 return -EINVAL; 2648 } 2649 if (!params->cbs.tx_cb || !params->cbs.rx_cb) { 2650 DP_ERR(cdev, 2651 "qed roce ll2 start: failed due to NULL tx/rx. tx_cb=%p, rx_cb=%p\n", 2652 params->cbs.tx_cb, params->cbs.rx_cb); 2653 return -EINVAL; 2654 } 2655 if (!is_valid_ether_addr(params->mac_address)) { 2656 DP_ERR(cdev, 2657 "qed roce ll2 start: failed due to invalid Ethernet address %pM\n", 2658 params->mac_address); 2659 return -EINVAL; 2660 } 2661 2662 /* Initialize */ 2663 roce_ll2 = kzalloc(sizeof(*roce_ll2), GFP_ATOMIC); 2664 if (!roce_ll2) { 2665 DP_ERR(cdev, "qed roce ll2 start: failed memory allocation\n"); 2666 return -ENOMEM; 2667 } 2668 roce_ll2->handle = QED_LL2_UNUSED_HANDLE; 2669 roce_ll2->cbs = params->cbs; 2670 roce_ll2->cb_cookie = params->cb_cookie; 2671 mutex_init(&roce_ll2->lock); 2672 2673 memset(&ll2_params, 0, sizeof(ll2_params)); 2674 ll2_params.conn_type = QED_LL2_TYPE_ROCE; 2675 ll2_params.mtu = params->mtu; 2676 ll2_params.rx_drop_ttl0_flg = true; 2677 ll2_params.rx_vlan_removal_en = false; 2678 ll2_params.tx_dest = CORE_TX_DEST_NW; 2679 ll2_params.ai_err_packet_too_big = LL2_DROP_PACKET; 2680 ll2_params.ai_err_no_buf = LL2_DROP_PACKET; 2681 ll2_params.gsi_enable = true; 2682 2683 rc = qed_ll2_acquire_connection(QED_LEADING_HWFN(cdev), &ll2_params, 2684 params->max_rx_buffers, 2685 params->max_tx_buffers, 2686 &roce_ll2->handle); 2687 if (rc) { 2688 DP_ERR(cdev, 2689 "qed roce ll2 start: failed to acquire LL2 connection (rc=%d)\n", 2690 rc); 2691 goto err; 2692 } 2693 2694 rc = qed_ll2_establish_connection(QED_LEADING_HWFN(cdev), 2695 roce_ll2->handle); 2696 if (rc) { 2697 DP_ERR(cdev, 2698 "qed roce ll2 start: failed to establish LL2 connection (rc=%d)\n", 2699 rc); 2700 goto err1; 2701 } 2702 2703 hwfn->ll2 = roce_ll2; 2704 2705 rc = qed_roce_ll2_set_mac_filter(cdev, NULL, params->mac_address); 2706 if (rc) { 2707 hwfn->ll2 = NULL; 2708 goto err2; 2709 } 2710 ether_addr_copy(roce_ll2->mac_address, params->mac_address); 2711 2712 return 0; 2713 2714 err2: 2715 qed_ll2_terminate_connection(QED_LEADING_HWFN(cdev), roce_ll2->handle); 2716 err1: 2717 qed_ll2_release_connection(QED_LEADING_HWFN(cdev), roce_ll2->handle); 2718 err: 2719 kfree(roce_ll2); 2720 return rc; 2721 } 2722 2723 static int qed_roce_ll2_stop(struct qed_dev *cdev) 2724 { 2725 struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev); 2726 struct qed_roce_ll2_info *roce_ll2 = hwfn->ll2; 2727 int rc; 2728 2729 if (roce_ll2->handle == QED_LL2_UNUSED_HANDLE) { 2730 DP_ERR(cdev, "qed roce ll2 stop: cannot stop an unused LL2\n"); 2731 return -EINVAL; 2732 } 2733 2734 /* remove LL2 MAC address filter */ 2735 rc = qed_roce_ll2_set_mac_filter(cdev, roce_ll2->mac_address, NULL); 2736 eth_zero_addr(roce_ll2->mac_address); 2737 2738 rc = qed_ll2_terminate_connection(QED_LEADING_HWFN(cdev), 2739 roce_ll2->handle); 2740 if (rc) 2741 DP_ERR(cdev, 2742 "qed roce ll2 stop: failed to terminate LL2 connection (rc=%d)\n", 2743 rc); 2744 2745 qed_ll2_release_connection(QED_LEADING_HWFN(cdev), roce_ll2->handle); 2746 2747 roce_ll2->handle = QED_LL2_UNUSED_HANDLE; 2748 2749 kfree(roce_ll2); 2750 2751 return rc; 2752 } 2753 2754 static int qed_roce_ll2_tx(struct qed_dev *cdev, 2755 struct qed_roce_ll2_packet *pkt, 2756 struct qed_roce_ll2_tx_params *params) 2757 { 2758 struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev); 2759 struct qed_roce_ll2_info *roce_ll2 = hwfn->ll2; 2760 enum qed_ll2_roce_flavor_type qed_roce_flavor; 2761 u8 flags = 0; 2762 int rc; 2763 int i; 2764 2765 if (!pkt || !params) { 2766 DP_ERR(cdev, 2767 "roce ll2 tx: failed tx because one of the following is NULL - drv=%p, pkt=%p, params=%p\n", 2768 cdev, pkt, params); 2769 return -EINVAL; 2770 } 2771 2772 qed_roce_flavor = (pkt->roce_mode == ROCE_V1) ? QED_LL2_ROCE 2773 : QED_LL2_RROCE; 2774 2775 if (pkt->roce_mode == ROCE_V2_IPV4) 2776 flags |= BIT(CORE_TX_BD_FLAGS_IP_CSUM_SHIFT); 2777 2778 /* Tx header */ 2779 rc = qed_ll2_prepare_tx_packet(QED_LEADING_HWFN(cdev), roce_ll2->handle, 2780 1 + pkt->n_seg, 0, flags, 0, 2781 QED_LL2_TX_DEST_NW, 2782 qed_roce_flavor, pkt->header.baddr, 2783 pkt->header.len, pkt, 1); 2784 if (rc) { 2785 DP_ERR(cdev, "roce ll2 tx: header failed (rc=%d)\n", rc); 2786 return QED_ROCE_TX_HEAD_FAILURE; 2787 } 2788 2789 /* Tx payload */ 2790 for (i = 0; i < pkt->n_seg; i++) { 2791 rc = qed_ll2_set_fragment_of_tx_packet(QED_LEADING_HWFN(cdev), 2792 roce_ll2->handle, 2793 pkt->payload[i].baddr, 2794 pkt->payload[i].len); 2795 if (rc) { 2796 /* If failed not much to do here, partial packet has 2797 * been posted * we can't free memory, will need to wait 2798 * for completion 2799 */ 2800 DP_ERR(cdev, 2801 "roce ll2 tx: payload failed (rc=%d)\n", rc); 2802 return QED_ROCE_TX_FRAG_FAILURE; 2803 } 2804 } 2805 2806 return 0; 2807 } 2808 2809 static int qed_roce_ll2_post_rx_buffer(struct qed_dev *cdev, 2810 struct qed_roce_ll2_buffer *buf, 2811 u64 cookie, u8 notify_fw) 2812 { 2813 return qed_ll2_post_rx_buffer(QED_LEADING_HWFN(cdev), 2814 QED_LEADING_HWFN(cdev)->ll2->handle, 2815 buf->baddr, buf->len, 2816 (void *)(uintptr_t)cookie, notify_fw); 2817 } 2818 2819 static int qed_roce_ll2_stats(struct qed_dev *cdev, struct qed_ll2_stats *stats) 2820 { 2821 struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev); 2822 struct qed_roce_ll2_info *roce_ll2 = hwfn->ll2; 2823 2824 return qed_ll2_get_stats(QED_LEADING_HWFN(cdev), 2825 roce_ll2->handle, stats); 2826 } 2827 2828 static const struct qed_rdma_ops qed_rdma_ops_pass = { 2829 .common = &qed_common_ops_pass, 2830 .fill_dev_info = &qed_fill_rdma_dev_info, 2831 .rdma_get_rdma_ctx = &qed_rdma_get_rdma_ctx, 2832 .rdma_init = &qed_rdma_init, 2833 .rdma_add_user = &qed_rdma_add_user, 2834 .rdma_remove_user = &qed_rdma_remove_user, 2835 .rdma_stop = &qed_rdma_stop, 2836 .rdma_query_port = &qed_rdma_query_port, 2837 .rdma_query_device = &qed_rdma_query_device, 2838 .rdma_get_start_sb = &qed_rdma_get_sb_start, 2839 .rdma_get_rdma_int = &qed_rdma_get_int, 2840 .rdma_set_rdma_int = &qed_rdma_set_int, 2841 .rdma_get_min_cnq_msix = &qed_rdma_get_min_cnq_msix, 2842 .rdma_cnq_prod_update = &qed_rdma_cnq_prod_update, 2843 .rdma_alloc_pd = &qed_rdma_alloc_pd, 2844 .rdma_dealloc_pd = &qed_rdma_free_pd, 2845 .rdma_create_cq = &qed_rdma_create_cq, 2846 .rdma_destroy_cq = &qed_rdma_destroy_cq, 2847 .rdma_create_qp = &qed_rdma_create_qp, 2848 .rdma_modify_qp = &qed_rdma_modify_qp, 2849 .rdma_query_qp = &qed_rdma_query_qp, 2850 .rdma_destroy_qp = &qed_rdma_destroy_qp, 2851 .rdma_alloc_tid = &qed_rdma_alloc_tid, 2852 .rdma_free_tid = &qed_rdma_free_tid, 2853 .rdma_register_tid = &qed_rdma_register_tid, 2854 .rdma_deregister_tid = &qed_rdma_deregister_tid, 2855 .roce_ll2_start = &qed_roce_ll2_start, 2856 .roce_ll2_stop = &qed_roce_ll2_stop, 2857 .roce_ll2_tx = &qed_roce_ll2_tx, 2858 .roce_ll2_post_rx_buffer = &qed_roce_ll2_post_rx_buffer, 2859 .roce_ll2_set_mac_filter = &qed_roce_ll2_set_mac_filter, 2860 .roce_ll2_stats = &qed_roce_ll2_stats, 2861 }; 2862 2863 const struct qed_rdma_ops *qed_get_rdma_ops(void) 2864 { 2865 return &qed_rdma_ops_pass; 2866 } 2867 EXPORT_SYMBOL(qed_get_rdma_ops); 2868