1 /* 2 * Copyright (c) 2016-2017 Hisilicon Limited. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/acpi.h> 34 #include <linux/etherdevice.h> 35 #include <linux/interrupt.h> 36 #include <linux/iopoll.h> 37 #include <linux/kernel.h> 38 #include <linux/types.h> 39 #include <net/addrconf.h> 40 #include <rdma/ib_addr.h> 41 #include <rdma/ib_cache.h> 42 #include <rdma/ib_umem.h> 43 #include <rdma/uverbs_ioctl.h> 44 45 #include "hnae3.h" 46 #include "hns_roce_common.h" 47 #include "hns_roce_device.h" 48 #include "hns_roce_cmd.h" 49 #include "hns_roce_hem.h" 50 #include "hns_roce_hw_v2.h" 51 52 enum { 53 CMD_RST_PRC_OTHERS, 54 CMD_RST_PRC_SUCCESS, 55 CMD_RST_PRC_EBUSY, 56 }; 57 58 enum ecc_resource_type { 59 ECC_RESOURCE_QPC, 60 ECC_RESOURCE_CQC, 61 ECC_RESOURCE_MPT, 62 ECC_RESOURCE_SRQC, 63 ECC_RESOURCE_GMV, 64 ECC_RESOURCE_QPC_TIMER, 65 ECC_RESOURCE_CQC_TIMER, 66 ECC_RESOURCE_SCCC, 67 ECC_RESOURCE_COUNT, 68 }; 69 70 static const struct { 71 const char *name; 72 u8 read_bt0_op; 73 u8 write_bt0_op; 74 } fmea_ram_res[] = { 75 { "ECC_RESOURCE_QPC", 76 HNS_ROCE_CMD_READ_QPC_BT0, HNS_ROCE_CMD_WRITE_QPC_BT0 }, 77 { "ECC_RESOURCE_CQC", 78 HNS_ROCE_CMD_READ_CQC_BT0, HNS_ROCE_CMD_WRITE_CQC_BT0 }, 79 { "ECC_RESOURCE_MPT", 80 HNS_ROCE_CMD_READ_MPT_BT0, HNS_ROCE_CMD_WRITE_MPT_BT0 }, 81 { "ECC_RESOURCE_SRQC", 82 HNS_ROCE_CMD_READ_SRQC_BT0, HNS_ROCE_CMD_WRITE_SRQC_BT0 }, 83 /* ECC_RESOURCE_GMV is handled by cmdq, not mailbox */ 84 { "ECC_RESOURCE_GMV", 85 0, 0 }, 86 { "ECC_RESOURCE_QPC_TIMER", 87 HNS_ROCE_CMD_READ_QPC_TIMER_BT0, HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0 }, 88 { "ECC_RESOURCE_CQC_TIMER", 89 HNS_ROCE_CMD_READ_CQC_TIMER_BT0, HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0 }, 90 { "ECC_RESOURCE_SCCC", 91 HNS_ROCE_CMD_READ_SCCC_BT0, HNS_ROCE_CMD_WRITE_SCCC_BT0 }, 92 }; 93 94 static inline void set_data_seg_v2(struct hns_roce_v2_wqe_data_seg *dseg, 95 struct ib_sge *sg) 96 { 97 dseg->lkey = cpu_to_le32(sg->lkey); 98 dseg->addr = cpu_to_le64(sg->addr); 99 dseg->len = cpu_to_le32(sg->length); 100 } 101 102 /* 103 * mapped-value = 1 + real-value 104 * The hns wr opcode real value is start from 0, In order to distinguish between 105 * initialized and uninitialized map values, we plus 1 to the actual value when 106 * defining the mapping, so that the validity can be identified by checking the 107 * mapped value is greater than 0. 108 */ 109 #define HR_OPC_MAP(ib_key, hr_key) \ 110 [IB_WR_ ## ib_key] = 1 + HNS_ROCE_V2_WQE_OP_ ## hr_key 111 112 static const u32 hns_roce_op_code[] = { 113 HR_OPC_MAP(RDMA_WRITE, RDMA_WRITE), 114 HR_OPC_MAP(RDMA_WRITE_WITH_IMM, RDMA_WRITE_WITH_IMM), 115 HR_OPC_MAP(SEND, SEND), 116 HR_OPC_MAP(SEND_WITH_IMM, SEND_WITH_IMM), 117 HR_OPC_MAP(RDMA_READ, RDMA_READ), 118 HR_OPC_MAP(ATOMIC_CMP_AND_SWP, ATOM_CMP_AND_SWAP), 119 HR_OPC_MAP(ATOMIC_FETCH_AND_ADD, ATOM_FETCH_AND_ADD), 120 HR_OPC_MAP(SEND_WITH_INV, SEND_WITH_INV), 121 HR_OPC_MAP(MASKED_ATOMIC_CMP_AND_SWP, ATOM_MSK_CMP_AND_SWAP), 122 HR_OPC_MAP(MASKED_ATOMIC_FETCH_AND_ADD, ATOM_MSK_FETCH_AND_ADD), 123 HR_OPC_MAP(REG_MR, FAST_REG_PMR), 124 }; 125 126 static u32 to_hr_opcode(u32 ib_opcode) 127 { 128 if (ib_opcode >= ARRAY_SIZE(hns_roce_op_code)) 129 return HNS_ROCE_V2_WQE_OP_MASK; 130 131 return hns_roce_op_code[ib_opcode] ? hns_roce_op_code[ib_opcode] - 1 : 132 HNS_ROCE_V2_WQE_OP_MASK; 133 } 134 135 static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, 136 const struct ib_reg_wr *wr) 137 { 138 struct hns_roce_wqe_frmr_seg *fseg = 139 (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe); 140 struct hns_roce_mr *mr = to_hr_mr(wr->mr); 141 u64 pbl_ba; 142 143 /* use ib_access_flags */ 144 hr_reg_write_bool(fseg, FRMR_BIND_EN, wr->access & IB_ACCESS_MW_BIND); 145 hr_reg_write_bool(fseg, FRMR_ATOMIC, 146 wr->access & IB_ACCESS_REMOTE_ATOMIC); 147 hr_reg_write_bool(fseg, FRMR_RR, wr->access & IB_ACCESS_REMOTE_READ); 148 hr_reg_write_bool(fseg, FRMR_RW, wr->access & IB_ACCESS_REMOTE_WRITE); 149 hr_reg_write_bool(fseg, FRMR_LW, wr->access & IB_ACCESS_LOCAL_WRITE); 150 151 /* Data structure reuse may lead to confusion */ 152 pbl_ba = mr->pbl_mtr.hem_cfg.root_ba; 153 rc_sq_wqe->msg_len = cpu_to_le32(lower_32_bits(pbl_ba)); 154 rc_sq_wqe->inv_key = cpu_to_le32(upper_32_bits(pbl_ba)); 155 156 rc_sq_wqe->byte_16 = cpu_to_le32(wr->mr->length & 0xffffffff); 157 rc_sq_wqe->byte_20 = cpu_to_le32(wr->mr->length >> 32); 158 rc_sq_wqe->rkey = cpu_to_le32(wr->key); 159 rc_sq_wqe->va = cpu_to_le64(wr->mr->iova); 160 161 hr_reg_write(fseg, FRMR_PBL_SIZE, mr->npages); 162 hr_reg_write(fseg, FRMR_PBL_BUF_PG_SZ, 163 to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift)); 164 hr_reg_clear(fseg, FRMR_BLK_MODE); 165 } 166 167 static void set_atomic_seg(const struct ib_send_wr *wr, 168 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, 169 unsigned int valid_num_sge) 170 { 171 struct hns_roce_v2_wqe_data_seg *dseg = 172 (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe); 173 struct hns_roce_wqe_atomic_seg *aseg = 174 (void *)dseg + sizeof(struct hns_roce_v2_wqe_data_seg); 175 176 set_data_seg_v2(dseg, wr->sg_list); 177 178 if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) { 179 aseg->fetchadd_swap_data = cpu_to_le64(atomic_wr(wr)->swap); 180 aseg->cmp_data = cpu_to_le64(atomic_wr(wr)->compare_add); 181 } else { 182 aseg->fetchadd_swap_data = 183 cpu_to_le64(atomic_wr(wr)->compare_add); 184 aseg->cmp_data = 0; 185 } 186 187 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_SGE_NUM, valid_num_sge); 188 } 189 190 static int fill_ext_sge_inl_data(struct hns_roce_qp *qp, 191 const struct ib_send_wr *wr, 192 unsigned int *sge_idx, u32 msg_len) 193 { 194 struct ib_device *ibdev = &(to_hr_dev(qp->ibqp.device))->ib_dev; 195 unsigned int left_len_in_pg; 196 unsigned int idx = *sge_idx; 197 unsigned int i = 0; 198 unsigned int len; 199 void *addr; 200 void *dseg; 201 202 if (msg_len > qp->sq.ext_sge_cnt * HNS_ROCE_SGE_SIZE) { 203 ibdev_err(ibdev, 204 "no enough extended sge space for inline data.\n"); 205 return -EINVAL; 206 } 207 208 dseg = hns_roce_get_extend_sge(qp, idx & (qp->sge.sge_cnt - 1)); 209 left_len_in_pg = hr_hw_page_align((uintptr_t)dseg) - (uintptr_t)dseg; 210 len = wr->sg_list[0].length; 211 addr = (void *)(unsigned long)(wr->sg_list[0].addr); 212 213 /* When copying data to extended sge space, the left length in page may 214 * not long enough for current user's sge. So the data should be 215 * splited into several parts, one in the first page, and the others in 216 * the subsequent pages. 217 */ 218 while (1) { 219 if (len <= left_len_in_pg) { 220 memcpy(dseg, addr, len); 221 222 idx += len / HNS_ROCE_SGE_SIZE; 223 224 i++; 225 if (i >= wr->num_sge) 226 break; 227 228 left_len_in_pg -= len; 229 len = wr->sg_list[i].length; 230 addr = (void *)(unsigned long)(wr->sg_list[i].addr); 231 dseg += len; 232 } else { 233 memcpy(dseg, addr, left_len_in_pg); 234 235 len -= left_len_in_pg; 236 addr += left_len_in_pg; 237 idx += left_len_in_pg / HNS_ROCE_SGE_SIZE; 238 dseg = hns_roce_get_extend_sge(qp, 239 idx & (qp->sge.sge_cnt - 1)); 240 left_len_in_pg = 1 << HNS_HW_PAGE_SHIFT; 241 } 242 } 243 244 *sge_idx = idx; 245 246 return 0; 247 } 248 249 static void set_extend_sge(struct hns_roce_qp *qp, struct ib_sge *sge, 250 unsigned int *sge_ind, unsigned int cnt) 251 { 252 struct hns_roce_v2_wqe_data_seg *dseg; 253 unsigned int idx = *sge_ind; 254 255 while (cnt > 0) { 256 dseg = hns_roce_get_extend_sge(qp, idx & (qp->sge.sge_cnt - 1)); 257 if (likely(sge->length)) { 258 set_data_seg_v2(dseg, sge); 259 idx++; 260 cnt--; 261 } 262 sge++; 263 } 264 265 *sge_ind = idx; 266 } 267 268 static bool check_inl_data_len(struct hns_roce_qp *qp, unsigned int len) 269 { 270 struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device); 271 int mtu = ib_mtu_enum_to_int(qp->path_mtu); 272 273 if (mtu < 0 || len > qp->max_inline_data || len > mtu) { 274 ibdev_err(&hr_dev->ib_dev, 275 "invalid length of data, data len = %u, max inline len = %u, path mtu = %d.\n", 276 len, qp->max_inline_data, mtu); 277 return false; 278 } 279 280 return true; 281 } 282 283 static int set_rc_inl(struct hns_roce_qp *qp, const struct ib_send_wr *wr, 284 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, 285 unsigned int *sge_idx) 286 { 287 struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device); 288 u32 msg_len = le32_to_cpu(rc_sq_wqe->msg_len); 289 struct ib_device *ibdev = &hr_dev->ib_dev; 290 unsigned int curr_idx = *sge_idx; 291 void *dseg = rc_sq_wqe; 292 unsigned int i; 293 int ret; 294 295 if (unlikely(wr->opcode == IB_WR_RDMA_READ)) { 296 ibdev_err(ibdev, "invalid inline parameters!\n"); 297 return -EINVAL; 298 } 299 300 if (!check_inl_data_len(qp, msg_len)) 301 return -EINVAL; 302 303 dseg += sizeof(struct hns_roce_v2_rc_send_wqe); 304 305 if (msg_len <= HNS_ROCE_V2_MAX_RC_INL_INN_SZ) { 306 hr_reg_clear(rc_sq_wqe, RC_SEND_WQE_INL_TYPE); 307 308 for (i = 0; i < wr->num_sge; i++) { 309 memcpy(dseg, ((void *)wr->sg_list[i].addr), 310 wr->sg_list[i].length); 311 dseg += wr->sg_list[i].length; 312 } 313 } else { 314 hr_reg_enable(rc_sq_wqe, RC_SEND_WQE_INL_TYPE); 315 316 ret = fill_ext_sge_inl_data(qp, wr, &curr_idx, msg_len); 317 if (ret) 318 return ret; 319 320 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_SGE_NUM, curr_idx - *sge_idx); 321 } 322 323 *sge_idx = curr_idx; 324 325 return 0; 326 } 327 328 static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr, 329 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, 330 unsigned int *sge_ind, 331 unsigned int valid_num_sge) 332 { 333 struct hns_roce_v2_wqe_data_seg *dseg = 334 (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe); 335 struct hns_roce_qp *qp = to_hr_qp(ibqp); 336 int j = 0; 337 int i; 338 339 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_MSG_START_SGE_IDX, 340 (*sge_ind) & (qp->sge.sge_cnt - 1)); 341 342 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_INLINE, 343 !!(wr->send_flags & IB_SEND_INLINE)); 344 if (wr->send_flags & IB_SEND_INLINE) 345 return set_rc_inl(qp, wr, rc_sq_wqe, sge_ind); 346 347 if (valid_num_sge <= HNS_ROCE_SGE_IN_WQE) { 348 for (i = 0; i < wr->num_sge; i++) { 349 if (likely(wr->sg_list[i].length)) { 350 set_data_seg_v2(dseg, wr->sg_list + i); 351 dseg++; 352 } 353 } 354 } else { 355 for (i = 0; i < wr->num_sge && j < HNS_ROCE_SGE_IN_WQE; i++) { 356 if (likely(wr->sg_list[i].length)) { 357 set_data_seg_v2(dseg, wr->sg_list + i); 358 dseg++; 359 j++; 360 } 361 } 362 363 set_extend_sge(qp, wr->sg_list + i, sge_ind, 364 valid_num_sge - HNS_ROCE_SGE_IN_WQE); 365 } 366 367 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_SGE_NUM, valid_num_sge); 368 369 return 0; 370 } 371 372 static int check_send_valid(struct hns_roce_dev *hr_dev, 373 struct hns_roce_qp *hr_qp) 374 { 375 struct ib_device *ibdev = &hr_dev->ib_dev; 376 377 if (unlikely(hr_qp->state == IB_QPS_RESET || 378 hr_qp->state == IB_QPS_INIT || 379 hr_qp->state == IB_QPS_RTR)) { 380 ibdev_err(ibdev, "failed to post WQE, QP state %u!\n", 381 hr_qp->state); 382 return -EINVAL; 383 } else if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN)) { 384 ibdev_err(ibdev, "failed to post WQE, dev state %d!\n", 385 hr_dev->state); 386 return -EIO; 387 } 388 389 return 0; 390 } 391 392 static unsigned int calc_wr_sge_num(const struct ib_send_wr *wr, 393 unsigned int *sge_len) 394 { 395 unsigned int valid_num = 0; 396 unsigned int len = 0; 397 int i; 398 399 for (i = 0; i < wr->num_sge; i++) { 400 if (likely(wr->sg_list[i].length)) { 401 len += wr->sg_list[i].length; 402 valid_num++; 403 } 404 } 405 406 *sge_len = len; 407 return valid_num; 408 } 409 410 static __le32 get_immtdata(const struct ib_send_wr *wr) 411 { 412 switch (wr->opcode) { 413 case IB_WR_SEND_WITH_IMM: 414 case IB_WR_RDMA_WRITE_WITH_IMM: 415 return cpu_to_le32(be32_to_cpu(wr->ex.imm_data)); 416 default: 417 return 0; 418 } 419 } 420 421 static int set_ud_opcode(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe, 422 const struct ib_send_wr *wr) 423 { 424 u32 ib_op = wr->opcode; 425 426 if (ib_op != IB_WR_SEND && ib_op != IB_WR_SEND_WITH_IMM) 427 return -EINVAL; 428 429 ud_sq_wqe->immtdata = get_immtdata(wr); 430 431 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_OPCODE, to_hr_opcode(ib_op)); 432 433 return 0; 434 } 435 436 static int fill_ud_av(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe, 437 struct hns_roce_ah *ah) 438 { 439 struct ib_device *ib_dev = ah->ibah.device; 440 struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev); 441 442 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_UDPSPN, ah->av.udp_sport); 443 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_HOPLIMIT, ah->av.hop_limit); 444 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_TCLASS, ah->av.tclass); 445 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_FLOW_LABEL, ah->av.flowlabel); 446 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SL, ah->av.sl); 447 448 ud_sq_wqe->sgid_index = ah->av.gid_index; 449 450 memcpy(ud_sq_wqe->dmac, ah->av.mac, ETH_ALEN); 451 memcpy(ud_sq_wqe->dgid, ah->av.dgid, GID_LEN_V2); 452 453 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) 454 return 0; 455 456 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_VLAN_EN, ah->av.vlan_en); 457 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_VLAN, ah->av.vlan_id); 458 459 return 0; 460 } 461 462 static inline int set_ud_wqe(struct hns_roce_qp *qp, 463 const struct ib_send_wr *wr, 464 void *wqe, unsigned int *sge_idx, 465 unsigned int owner_bit) 466 { 467 struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah); 468 struct hns_roce_v2_ud_send_wqe *ud_sq_wqe = wqe; 469 unsigned int curr_idx = *sge_idx; 470 unsigned int valid_num_sge; 471 u32 msg_len = 0; 472 int ret; 473 474 valid_num_sge = calc_wr_sge_num(wr, &msg_len); 475 476 ret = set_ud_opcode(ud_sq_wqe, wr); 477 if (WARN_ON(ret)) 478 return ret; 479 480 ud_sq_wqe->msg_len = cpu_to_le32(msg_len); 481 482 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_CQE, 483 !!(wr->send_flags & IB_SEND_SIGNALED)); 484 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SE, 485 !!(wr->send_flags & IB_SEND_SOLICITED)); 486 487 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_PD, to_hr_pd(qp->ibqp.pd)->pdn); 488 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SGE_NUM, valid_num_sge); 489 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_MSG_START_SGE_IDX, 490 curr_idx & (qp->sge.sge_cnt - 1)); 491 492 ud_sq_wqe->qkey = cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ? 493 qp->qkey : ud_wr(wr)->remote_qkey); 494 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_DQPN, ud_wr(wr)->remote_qpn); 495 496 ret = fill_ud_av(ud_sq_wqe, ah); 497 if (ret) 498 return ret; 499 500 qp->sl = to_hr_ah(ud_wr(wr)->ah)->av.sl; 501 502 set_extend_sge(qp, wr->sg_list, &curr_idx, valid_num_sge); 503 504 /* 505 * The pipeline can sequentially post all valid WQEs into WQ buffer, 506 * including new WQEs waiting for the doorbell to update the PI again. 507 * Therefore, the owner bit of WQE MUST be updated after all fields 508 * and extSGEs have been written into DDR instead of cache. 509 */ 510 if (qp->en_flags & HNS_ROCE_QP_CAP_OWNER_DB) 511 dma_wmb(); 512 513 *sge_idx = curr_idx; 514 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_OWNER, owner_bit); 515 516 return 0; 517 } 518 519 static int set_rc_opcode(struct hns_roce_dev *hr_dev, 520 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, 521 const struct ib_send_wr *wr) 522 { 523 u32 ib_op = wr->opcode; 524 int ret = 0; 525 526 rc_sq_wqe->immtdata = get_immtdata(wr); 527 528 switch (ib_op) { 529 case IB_WR_RDMA_READ: 530 case IB_WR_RDMA_WRITE: 531 case IB_WR_RDMA_WRITE_WITH_IMM: 532 rc_sq_wqe->rkey = cpu_to_le32(rdma_wr(wr)->rkey); 533 rc_sq_wqe->va = cpu_to_le64(rdma_wr(wr)->remote_addr); 534 break; 535 case IB_WR_SEND: 536 case IB_WR_SEND_WITH_IMM: 537 break; 538 case IB_WR_ATOMIC_CMP_AND_SWP: 539 case IB_WR_ATOMIC_FETCH_AND_ADD: 540 rc_sq_wqe->rkey = cpu_to_le32(atomic_wr(wr)->rkey); 541 rc_sq_wqe->va = cpu_to_le64(atomic_wr(wr)->remote_addr); 542 break; 543 case IB_WR_REG_MR: 544 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) 545 set_frmr_seg(rc_sq_wqe, reg_wr(wr)); 546 else 547 ret = -EOPNOTSUPP; 548 break; 549 case IB_WR_SEND_WITH_INV: 550 rc_sq_wqe->inv_key = cpu_to_le32(wr->ex.invalidate_rkey); 551 break; 552 default: 553 ret = -EINVAL; 554 } 555 556 if (unlikely(ret)) 557 return ret; 558 559 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_OPCODE, to_hr_opcode(ib_op)); 560 561 return ret; 562 } 563 564 static inline int set_rc_wqe(struct hns_roce_qp *qp, 565 const struct ib_send_wr *wr, 566 void *wqe, unsigned int *sge_idx, 567 unsigned int owner_bit) 568 { 569 struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device); 570 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe = wqe; 571 unsigned int curr_idx = *sge_idx; 572 unsigned int valid_num_sge; 573 u32 msg_len = 0; 574 int ret; 575 576 valid_num_sge = calc_wr_sge_num(wr, &msg_len); 577 578 rc_sq_wqe->msg_len = cpu_to_le32(msg_len); 579 580 ret = set_rc_opcode(hr_dev, rc_sq_wqe, wr); 581 if (WARN_ON(ret)) 582 return ret; 583 584 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_FENCE, 585 (wr->send_flags & IB_SEND_FENCE) ? 1 : 0); 586 587 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_SE, 588 (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0); 589 590 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_CQE, 591 (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0); 592 593 if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP || 594 wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) 595 set_atomic_seg(wr, rc_sq_wqe, valid_num_sge); 596 else if (wr->opcode != IB_WR_REG_MR) 597 ret = set_rwqe_data_seg(&qp->ibqp, wr, rc_sq_wqe, 598 &curr_idx, valid_num_sge); 599 600 /* 601 * The pipeline can sequentially post all valid WQEs into WQ buffer, 602 * including new WQEs waiting for the doorbell to update the PI again. 603 * Therefore, the owner bit of WQE MUST be updated after all fields 604 * and extSGEs have been written into DDR instead of cache. 605 */ 606 if (qp->en_flags & HNS_ROCE_QP_CAP_OWNER_DB) 607 dma_wmb(); 608 609 *sge_idx = curr_idx; 610 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_OWNER, owner_bit); 611 612 return ret; 613 } 614 615 static inline void update_sq_db(struct hns_roce_dev *hr_dev, 616 struct hns_roce_qp *qp) 617 { 618 if (unlikely(qp->state == IB_QPS_ERR)) { 619 flush_cqe(hr_dev, qp); 620 } else { 621 struct hns_roce_v2_db sq_db = {}; 622 623 hr_reg_write(&sq_db, DB_TAG, qp->qpn); 624 hr_reg_write(&sq_db, DB_CMD, HNS_ROCE_V2_SQ_DB); 625 hr_reg_write(&sq_db, DB_PI, qp->sq.head); 626 hr_reg_write(&sq_db, DB_SL, qp->sl); 627 628 hns_roce_write64(hr_dev, (__le32 *)&sq_db, qp->sq.db_reg); 629 } 630 } 631 632 static inline void update_rq_db(struct hns_roce_dev *hr_dev, 633 struct hns_roce_qp *qp) 634 { 635 if (unlikely(qp->state == IB_QPS_ERR)) { 636 flush_cqe(hr_dev, qp); 637 } else { 638 if (likely(qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)) { 639 *qp->rdb.db_record = 640 qp->rq.head & V2_DB_PRODUCER_IDX_M; 641 } else { 642 struct hns_roce_v2_db rq_db = {}; 643 644 hr_reg_write(&rq_db, DB_TAG, qp->qpn); 645 hr_reg_write(&rq_db, DB_CMD, HNS_ROCE_V2_RQ_DB); 646 hr_reg_write(&rq_db, DB_PI, qp->rq.head); 647 648 hns_roce_write64(hr_dev, (__le32 *)&rq_db, 649 qp->rq.db_reg); 650 } 651 } 652 } 653 654 static void hns_roce_write512(struct hns_roce_dev *hr_dev, u64 *val, 655 u64 __iomem *dest) 656 { 657 #define HNS_ROCE_WRITE_TIMES 8 658 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; 659 struct hnae3_handle *handle = priv->handle; 660 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 661 int i; 662 663 if (!hr_dev->dis_db && !ops->get_hw_reset_stat(handle)) 664 for (i = 0; i < HNS_ROCE_WRITE_TIMES; i++) 665 writeq_relaxed(*(val + i), dest + i); 666 } 667 668 static void write_dwqe(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp, 669 void *wqe) 670 { 671 #define HNS_ROCE_SL_SHIFT 2 672 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe = wqe; 673 674 /* All kinds of DirectWQE have the same header field layout */ 675 hr_reg_enable(rc_sq_wqe, RC_SEND_WQE_FLAG); 676 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_DB_SL_L, qp->sl); 677 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_DB_SL_H, 678 qp->sl >> HNS_ROCE_SL_SHIFT); 679 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_WQE_INDEX, qp->sq.head); 680 681 hns_roce_write512(hr_dev, wqe, qp->sq.db_reg); 682 } 683 684 static int hns_roce_v2_post_send(struct ib_qp *ibqp, 685 const struct ib_send_wr *wr, 686 const struct ib_send_wr **bad_wr) 687 { 688 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 689 struct ib_device *ibdev = &hr_dev->ib_dev; 690 struct hns_roce_qp *qp = to_hr_qp(ibqp); 691 unsigned long flags = 0; 692 unsigned int owner_bit; 693 unsigned int sge_idx; 694 unsigned int wqe_idx; 695 void *wqe = NULL; 696 u32 nreq; 697 int ret; 698 699 spin_lock_irqsave(&qp->sq.lock, flags); 700 701 ret = check_send_valid(hr_dev, qp); 702 if (unlikely(ret)) { 703 *bad_wr = wr; 704 nreq = 0; 705 goto out; 706 } 707 708 sge_idx = qp->next_sge; 709 710 for (nreq = 0; wr; ++nreq, wr = wr->next) { 711 if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) { 712 ret = -ENOMEM; 713 *bad_wr = wr; 714 goto out; 715 } 716 717 wqe_idx = (qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1); 718 719 if (unlikely(wr->num_sge > qp->sq.max_gs)) { 720 ibdev_err(ibdev, "num_sge = %d > qp->sq.max_gs = %u.\n", 721 wr->num_sge, qp->sq.max_gs); 722 ret = -EINVAL; 723 *bad_wr = wr; 724 goto out; 725 } 726 727 wqe = hns_roce_get_send_wqe(qp, wqe_idx); 728 qp->sq.wrid[wqe_idx] = wr->wr_id; 729 owner_bit = 730 ~(((qp->sq.head + nreq) >> ilog2(qp->sq.wqe_cnt)) & 0x1); 731 732 /* Corresponding to the QP type, wqe process separately */ 733 if (ibqp->qp_type == IB_QPT_RC) 734 ret = set_rc_wqe(qp, wr, wqe, &sge_idx, owner_bit); 735 else 736 ret = set_ud_wqe(qp, wr, wqe, &sge_idx, owner_bit); 737 738 if (unlikely(ret)) { 739 *bad_wr = wr; 740 goto out; 741 } 742 } 743 744 out: 745 if (likely(nreq)) { 746 qp->sq.head += nreq; 747 qp->next_sge = sge_idx; 748 749 if (nreq == 1 && !ret && 750 (qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE)) 751 write_dwqe(hr_dev, qp, wqe); 752 else 753 update_sq_db(hr_dev, qp); 754 } 755 756 spin_unlock_irqrestore(&qp->sq.lock, flags); 757 758 return ret; 759 } 760 761 static int check_recv_valid(struct hns_roce_dev *hr_dev, 762 struct hns_roce_qp *hr_qp) 763 { 764 if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN)) 765 return -EIO; 766 767 if (hr_qp->state == IB_QPS_RESET) 768 return -EINVAL; 769 770 return 0; 771 } 772 773 static void fill_recv_sge_to_wqe(const struct ib_recv_wr *wr, void *wqe, 774 u32 max_sge, bool rsv) 775 { 776 struct hns_roce_v2_wqe_data_seg *dseg = wqe; 777 u32 i, cnt; 778 779 for (i = 0, cnt = 0; i < wr->num_sge; i++) { 780 /* Skip zero-length sge */ 781 if (!wr->sg_list[i].length) 782 continue; 783 set_data_seg_v2(dseg + cnt, wr->sg_list + i); 784 cnt++; 785 } 786 787 /* Fill a reserved sge to make hw stop reading remaining segments */ 788 if (rsv) { 789 dseg[cnt].lkey = cpu_to_le32(HNS_ROCE_INVALID_LKEY); 790 dseg[cnt].addr = 0; 791 dseg[cnt].len = cpu_to_le32(HNS_ROCE_INVALID_SGE_LENGTH); 792 } else { 793 /* Clear remaining segments to make ROCEE ignore sges */ 794 if (cnt < max_sge) 795 memset(dseg + cnt, 0, 796 (max_sge - cnt) * HNS_ROCE_SGE_SIZE); 797 } 798 } 799 800 static void fill_rq_wqe(struct hns_roce_qp *hr_qp, const struct ib_recv_wr *wr, 801 u32 wqe_idx, u32 max_sge) 802 { 803 void *wqe = NULL; 804 805 wqe = hns_roce_get_recv_wqe(hr_qp, wqe_idx); 806 fill_recv_sge_to_wqe(wr, wqe, max_sge, hr_qp->rq.rsv_sge); 807 } 808 809 static int hns_roce_v2_post_recv(struct ib_qp *ibqp, 810 const struct ib_recv_wr *wr, 811 const struct ib_recv_wr **bad_wr) 812 { 813 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 814 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 815 struct ib_device *ibdev = &hr_dev->ib_dev; 816 u32 wqe_idx, nreq, max_sge; 817 unsigned long flags; 818 int ret; 819 820 spin_lock_irqsave(&hr_qp->rq.lock, flags); 821 822 ret = check_recv_valid(hr_dev, hr_qp); 823 if (unlikely(ret)) { 824 *bad_wr = wr; 825 nreq = 0; 826 goto out; 827 } 828 829 max_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge; 830 for (nreq = 0; wr; ++nreq, wr = wr->next) { 831 if (unlikely(hns_roce_wq_overflow(&hr_qp->rq, nreq, 832 hr_qp->ibqp.recv_cq))) { 833 ret = -ENOMEM; 834 *bad_wr = wr; 835 goto out; 836 } 837 838 if (unlikely(wr->num_sge > max_sge)) { 839 ibdev_err(ibdev, "num_sge = %d >= max_sge = %u.\n", 840 wr->num_sge, max_sge); 841 ret = -EINVAL; 842 *bad_wr = wr; 843 goto out; 844 } 845 846 wqe_idx = (hr_qp->rq.head + nreq) & (hr_qp->rq.wqe_cnt - 1); 847 fill_rq_wqe(hr_qp, wr, wqe_idx, max_sge); 848 hr_qp->rq.wrid[wqe_idx] = wr->wr_id; 849 } 850 851 out: 852 if (likely(nreq)) { 853 hr_qp->rq.head += nreq; 854 855 update_rq_db(hr_dev, hr_qp); 856 } 857 spin_unlock_irqrestore(&hr_qp->rq.lock, flags); 858 859 return ret; 860 } 861 862 static void *get_srq_wqe_buf(struct hns_roce_srq *srq, u32 n) 863 { 864 return hns_roce_buf_offset(srq->buf_mtr.kmem, n << srq->wqe_shift); 865 } 866 867 static void *get_idx_buf(struct hns_roce_idx_que *idx_que, u32 n) 868 { 869 return hns_roce_buf_offset(idx_que->mtr.kmem, 870 n << idx_que->entry_shift); 871 } 872 873 static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, u32 wqe_index) 874 { 875 /* always called with interrupts disabled. */ 876 spin_lock(&srq->lock); 877 878 bitmap_clear(srq->idx_que.bitmap, wqe_index, 1); 879 srq->idx_que.tail++; 880 881 spin_unlock(&srq->lock); 882 } 883 884 static int hns_roce_srqwq_overflow(struct hns_roce_srq *srq) 885 { 886 struct hns_roce_idx_que *idx_que = &srq->idx_que; 887 888 return idx_que->head - idx_que->tail >= srq->wqe_cnt; 889 } 890 891 static int check_post_srq_valid(struct hns_roce_srq *srq, u32 max_sge, 892 const struct ib_recv_wr *wr) 893 { 894 struct ib_device *ib_dev = srq->ibsrq.device; 895 896 if (unlikely(wr->num_sge > max_sge)) { 897 ibdev_err(ib_dev, 898 "failed to check sge, wr->num_sge = %d, max_sge = %u.\n", 899 wr->num_sge, max_sge); 900 return -EINVAL; 901 } 902 903 if (unlikely(hns_roce_srqwq_overflow(srq))) { 904 ibdev_err(ib_dev, 905 "failed to check srqwq status, srqwq is full.\n"); 906 return -ENOMEM; 907 } 908 909 return 0; 910 } 911 912 static int get_srq_wqe_idx(struct hns_roce_srq *srq, u32 *wqe_idx) 913 { 914 struct hns_roce_idx_que *idx_que = &srq->idx_que; 915 u32 pos; 916 917 pos = find_first_zero_bit(idx_que->bitmap, srq->wqe_cnt); 918 if (unlikely(pos == srq->wqe_cnt)) 919 return -ENOSPC; 920 921 bitmap_set(idx_que->bitmap, pos, 1); 922 *wqe_idx = pos; 923 return 0; 924 } 925 926 static void fill_wqe_idx(struct hns_roce_srq *srq, unsigned int wqe_idx) 927 { 928 struct hns_roce_idx_que *idx_que = &srq->idx_que; 929 unsigned int head; 930 __le32 *buf; 931 932 head = idx_que->head & (srq->wqe_cnt - 1); 933 934 buf = get_idx_buf(idx_que, head); 935 *buf = cpu_to_le32(wqe_idx); 936 937 idx_que->head++; 938 } 939 940 static void update_srq_db(struct hns_roce_srq *srq) 941 { 942 struct hns_roce_dev *hr_dev = to_hr_dev(srq->ibsrq.device); 943 struct hns_roce_v2_db db; 944 945 hr_reg_write(&db, DB_TAG, srq->srqn); 946 hr_reg_write(&db, DB_CMD, HNS_ROCE_V2_SRQ_DB); 947 hr_reg_write(&db, DB_PI, srq->idx_que.head); 948 949 hns_roce_write64(hr_dev, (__le32 *)&db, srq->db_reg); 950 } 951 952 static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq, 953 const struct ib_recv_wr *wr, 954 const struct ib_recv_wr **bad_wr) 955 { 956 struct hns_roce_srq *srq = to_hr_srq(ibsrq); 957 unsigned long flags; 958 int ret = 0; 959 u32 max_sge; 960 u32 wqe_idx; 961 void *wqe; 962 u32 nreq; 963 964 spin_lock_irqsave(&srq->lock, flags); 965 966 max_sge = srq->max_gs - srq->rsv_sge; 967 for (nreq = 0; wr; ++nreq, wr = wr->next) { 968 ret = check_post_srq_valid(srq, max_sge, wr); 969 if (ret) { 970 *bad_wr = wr; 971 break; 972 } 973 974 ret = get_srq_wqe_idx(srq, &wqe_idx); 975 if (unlikely(ret)) { 976 *bad_wr = wr; 977 break; 978 } 979 980 wqe = get_srq_wqe_buf(srq, wqe_idx); 981 fill_recv_sge_to_wqe(wr, wqe, max_sge, srq->rsv_sge); 982 fill_wqe_idx(srq, wqe_idx); 983 srq->wrid[wqe_idx] = wr->wr_id; 984 } 985 986 if (likely(nreq)) { 987 if (srq->cap_flags & HNS_ROCE_SRQ_CAP_RECORD_DB) 988 *srq->rdb.db_record = srq->idx_que.head & 989 V2_DB_PRODUCER_IDX_M; 990 else 991 update_srq_db(srq); 992 } 993 994 spin_unlock_irqrestore(&srq->lock, flags); 995 996 return ret; 997 } 998 999 static u32 hns_roce_v2_cmd_hw_reseted(struct hns_roce_dev *hr_dev, 1000 unsigned long instance_stage, 1001 unsigned long reset_stage) 1002 { 1003 /* When hardware reset has been completed once or more, we should stop 1004 * sending mailbox&cmq&doorbell to hardware. If now in .init_instance() 1005 * function, we should exit with error. If now at HNAE3_INIT_CLIENT 1006 * stage of soft reset process, we should exit with error, and then 1007 * HNAE3_INIT_CLIENT related process can rollback the operation like 1008 * notifing hardware to free resources, HNAE3_INIT_CLIENT related 1009 * process will exit with error to notify NIC driver to reschedule soft 1010 * reset process once again. 1011 */ 1012 hr_dev->is_reset = true; 1013 hr_dev->dis_db = true; 1014 1015 if (reset_stage == HNS_ROCE_STATE_RST_INIT || 1016 instance_stage == HNS_ROCE_STATE_INIT) 1017 return CMD_RST_PRC_EBUSY; 1018 1019 return CMD_RST_PRC_SUCCESS; 1020 } 1021 1022 static u32 hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev, 1023 unsigned long instance_stage, 1024 unsigned long reset_stage) 1025 { 1026 #define HW_RESET_TIMEOUT_US 1000000 1027 #define HW_RESET_SLEEP_US 1000 1028 1029 struct hns_roce_v2_priv *priv = hr_dev->priv; 1030 struct hnae3_handle *handle = priv->handle; 1031 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 1032 unsigned long val; 1033 int ret; 1034 1035 /* When hardware reset is detected, we should stop sending mailbox&cmq& 1036 * doorbell to hardware. If now in .init_instance() function, we should 1037 * exit with error. If now at HNAE3_INIT_CLIENT stage of soft reset 1038 * process, we should exit with error, and then HNAE3_INIT_CLIENT 1039 * related process can rollback the operation like notifing hardware to 1040 * free resources, HNAE3_INIT_CLIENT related process will exit with 1041 * error to notify NIC driver to reschedule soft reset process once 1042 * again. 1043 */ 1044 hr_dev->dis_db = true; 1045 1046 ret = read_poll_timeout(ops->ae_dev_reset_cnt, val, 1047 val > hr_dev->reset_cnt, HW_RESET_SLEEP_US, 1048 HW_RESET_TIMEOUT_US, false, handle); 1049 if (!ret) 1050 hr_dev->is_reset = true; 1051 1052 if (!hr_dev->is_reset || reset_stage == HNS_ROCE_STATE_RST_INIT || 1053 instance_stage == HNS_ROCE_STATE_INIT) 1054 return CMD_RST_PRC_EBUSY; 1055 1056 return CMD_RST_PRC_SUCCESS; 1057 } 1058 1059 static u32 hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev *hr_dev) 1060 { 1061 struct hns_roce_v2_priv *priv = hr_dev->priv; 1062 struct hnae3_handle *handle = priv->handle; 1063 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 1064 1065 /* When software reset is detected at .init_instance() function, we 1066 * should stop sending mailbox&cmq&doorbell to hardware, and exit 1067 * with error. 1068 */ 1069 hr_dev->dis_db = true; 1070 if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt) 1071 hr_dev->is_reset = true; 1072 1073 return CMD_RST_PRC_EBUSY; 1074 } 1075 1076 static u32 check_aedev_reset_status(struct hns_roce_dev *hr_dev, 1077 struct hnae3_handle *handle) 1078 { 1079 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 1080 unsigned long instance_stage; /* the current instance stage */ 1081 unsigned long reset_stage; /* the current reset stage */ 1082 unsigned long reset_cnt; 1083 bool sw_resetting; 1084 bool hw_resetting; 1085 1086 /* Get information about reset from NIC driver or RoCE driver itself, 1087 * the meaning of the following variables from NIC driver are described 1088 * as below: 1089 * reset_cnt -- The count value of completed hardware reset. 1090 * hw_resetting -- Whether hardware device is resetting now. 1091 * sw_resetting -- Whether NIC's software reset process is running now. 1092 */ 1093 instance_stage = handle->rinfo.instance_state; 1094 reset_stage = handle->rinfo.reset_state; 1095 reset_cnt = ops->ae_dev_reset_cnt(handle); 1096 if (reset_cnt != hr_dev->reset_cnt) 1097 return hns_roce_v2_cmd_hw_reseted(hr_dev, instance_stage, 1098 reset_stage); 1099 1100 hw_resetting = ops->get_cmdq_stat(handle); 1101 if (hw_resetting) 1102 return hns_roce_v2_cmd_hw_resetting(hr_dev, instance_stage, 1103 reset_stage); 1104 1105 sw_resetting = ops->ae_dev_resetting(handle); 1106 if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT) 1107 return hns_roce_v2_cmd_sw_resetting(hr_dev); 1108 1109 return CMD_RST_PRC_OTHERS; 1110 } 1111 1112 static bool check_device_is_in_reset(struct hns_roce_dev *hr_dev) 1113 { 1114 struct hns_roce_v2_priv *priv = hr_dev->priv; 1115 struct hnae3_handle *handle = priv->handle; 1116 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 1117 1118 if (hr_dev->reset_cnt != ops->ae_dev_reset_cnt(handle)) 1119 return true; 1120 1121 if (ops->get_hw_reset_stat(handle)) 1122 return true; 1123 1124 if (ops->ae_dev_resetting(handle)) 1125 return true; 1126 1127 return false; 1128 } 1129 1130 static bool v2_chk_mbox_is_avail(struct hns_roce_dev *hr_dev, bool *busy) 1131 { 1132 struct hns_roce_v2_priv *priv = hr_dev->priv; 1133 u32 status; 1134 1135 if (hr_dev->is_reset) 1136 status = CMD_RST_PRC_SUCCESS; 1137 else 1138 status = check_aedev_reset_status(hr_dev, priv->handle); 1139 1140 *busy = (status == CMD_RST_PRC_EBUSY); 1141 1142 return status == CMD_RST_PRC_OTHERS; 1143 } 1144 1145 static int hns_roce_alloc_cmq_desc(struct hns_roce_dev *hr_dev, 1146 struct hns_roce_v2_cmq_ring *ring) 1147 { 1148 int size = ring->desc_num * sizeof(struct hns_roce_cmq_desc); 1149 1150 ring->desc = dma_alloc_coherent(hr_dev->dev, size, 1151 &ring->desc_dma_addr, GFP_KERNEL); 1152 if (!ring->desc) 1153 return -ENOMEM; 1154 1155 return 0; 1156 } 1157 1158 static void hns_roce_free_cmq_desc(struct hns_roce_dev *hr_dev, 1159 struct hns_roce_v2_cmq_ring *ring) 1160 { 1161 dma_free_coherent(hr_dev->dev, 1162 ring->desc_num * sizeof(struct hns_roce_cmq_desc), 1163 ring->desc, ring->desc_dma_addr); 1164 1165 ring->desc_dma_addr = 0; 1166 } 1167 1168 static int init_csq(struct hns_roce_dev *hr_dev, 1169 struct hns_roce_v2_cmq_ring *csq) 1170 { 1171 dma_addr_t dma; 1172 int ret; 1173 1174 csq->desc_num = CMD_CSQ_DESC_NUM; 1175 spin_lock_init(&csq->lock); 1176 csq->flag = TYPE_CSQ; 1177 csq->head = 0; 1178 1179 ret = hns_roce_alloc_cmq_desc(hr_dev, csq); 1180 if (ret) 1181 return ret; 1182 1183 dma = csq->desc_dma_addr; 1184 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_L_REG, lower_32_bits(dma)); 1185 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_H_REG, upper_32_bits(dma)); 1186 roce_write(hr_dev, ROCEE_TX_CMQ_DEPTH_REG, 1187 (u32)csq->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S); 1188 1189 /* Make sure to write CI first and then PI */ 1190 roce_write(hr_dev, ROCEE_TX_CMQ_CI_REG, 0); 1191 roce_write(hr_dev, ROCEE_TX_CMQ_PI_REG, 0); 1192 1193 return 0; 1194 } 1195 1196 static int hns_roce_v2_cmq_init(struct hns_roce_dev *hr_dev) 1197 { 1198 struct hns_roce_v2_priv *priv = hr_dev->priv; 1199 int ret; 1200 1201 priv->cmq.tx_timeout = HNS_ROCE_CMQ_TX_TIMEOUT; 1202 1203 ret = init_csq(hr_dev, &priv->cmq.csq); 1204 if (ret) 1205 dev_err(hr_dev->dev, "failed to init CSQ, ret = %d.\n", ret); 1206 1207 return ret; 1208 } 1209 1210 static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev) 1211 { 1212 struct hns_roce_v2_priv *priv = hr_dev->priv; 1213 1214 hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq); 1215 } 1216 1217 static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc, 1218 enum hns_roce_opcode_type opcode, 1219 bool is_read) 1220 { 1221 memset((void *)desc, 0, sizeof(struct hns_roce_cmq_desc)); 1222 desc->opcode = cpu_to_le16(opcode); 1223 desc->flag = cpu_to_le16(HNS_ROCE_CMD_FLAG_IN); 1224 if (is_read) 1225 desc->flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_WR); 1226 else 1227 desc->flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR); 1228 } 1229 1230 static int hns_roce_cmq_csq_done(struct hns_roce_dev *hr_dev) 1231 { 1232 u32 tail = roce_read(hr_dev, ROCEE_TX_CMQ_CI_REG); 1233 struct hns_roce_v2_priv *priv = hr_dev->priv; 1234 1235 return tail == priv->cmq.csq.head; 1236 } 1237 1238 static void update_cmdq_status(struct hns_roce_dev *hr_dev) 1239 { 1240 struct hns_roce_v2_priv *priv = hr_dev->priv; 1241 struct hnae3_handle *handle = priv->handle; 1242 1243 if (handle->rinfo.reset_state == HNS_ROCE_STATE_RST_INIT || 1244 handle->rinfo.instance_state == HNS_ROCE_STATE_INIT) 1245 hr_dev->cmd.state = HNS_ROCE_CMDQ_STATE_FATAL_ERR; 1246 } 1247 1248 static int hns_roce_cmd_err_convert_errno(u16 desc_ret) 1249 { 1250 struct hns_roce_cmd_errcode errcode_table[] = { 1251 {CMD_EXEC_SUCCESS, 0}, 1252 {CMD_NO_AUTH, -EPERM}, 1253 {CMD_NOT_EXIST, -EOPNOTSUPP}, 1254 {CMD_CRQ_FULL, -EXFULL}, 1255 {CMD_NEXT_ERR, -ENOSR}, 1256 {CMD_NOT_EXEC, -ENOTBLK}, 1257 {CMD_PARA_ERR, -EINVAL}, 1258 {CMD_RESULT_ERR, -ERANGE}, 1259 {CMD_TIMEOUT, -ETIME}, 1260 {CMD_HILINK_ERR, -ENOLINK}, 1261 {CMD_INFO_ILLEGAL, -ENXIO}, 1262 {CMD_INVALID, -EBADR}, 1263 }; 1264 u16 i; 1265 1266 for (i = 0; i < ARRAY_SIZE(errcode_table); i++) 1267 if (desc_ret == errcode_table[i].return_status) 1268 return errcode_table[i].errno; 1269 return -EIO; 1270 } 1271 1272 static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev, 1273 struct hns_roce_cmq_desc *desc, int num) 1274 { 1275 struct hns_roce_v2_priv *priv = hr_dev->priv; 1276 struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq; 1277 u32 timeout = 0; 1278 u16 desc_ret; 1279 u32 tail; 1280 int ret; 1281 int i; 1282 1283 spin_lock_bh(&csq->lock); 1284 1285 tail = csq->head; 1286 1287 for (i = 0; i < num; i++) { 1288 csq->desc[csq->head++] = desc[i]; 1289 if (csq->head == csq->desc_num) 1290 csq->head = 0; 1291 } 1292 1293 /* Write to hardware */ 1294 roce_write(hr_dev, ROCEE_TX_CMQ_PI_REG, csq->head); 1295 1296 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_CMDS_CNT]); 1297 1298 do { 1299 if (hns_roce_cmq_csq_done(hr_dev)) 1300 break; 1301 udelay(1); 1302 } while (++timeout < priv->cmq.tx_timeout); 1303 1304 if (hns_roce_cmq_csq_done(hr_dev)) { 1305 ret = 0; 1306 for (i = 0; i < num; i++) { 1307 /* check the result of hardware write back */ 1308 desc[i] = csq->desc[tail++]; 1309 if (tail == csq->desc_num) 1310 tail = 0; 1311 1312 desc_ret = le16_to_cpu(desc[i].retval); 1313 if (likely(desc_ret == CMD_EXEC_SUCCESS)) 1314 continue; 1315 1316 dev_err_ratelimited(hr_dev->dev, 1317 "Cmdq IO error, opcode = 0x%x, return = 0x%x.\n", 1318 desc->opcode, desc_ret); 1319 ret = hns_roce_cmd_err_convert_errno(desc_ret); 1320 } 1321 } else { 1322 /* FW/HW reset or incorrect number of desc */ 1323 tail = roce_read(hr_dev, ROCEE_TX_CMQ_CI_REG); 1324 dev_warn(hr_dev->dev, "CMDQ move tail from %u to %u.\n", 1325 csq->head, tail); 1326 csq->head = tail; 1327 1328 update_cmdq_status(hr_dev); 1329 1330 ret = -EAGAIN; 1331 } 1332 1333 spin_unlock_bh(&csq->lock); 1334 1335 if (ret) 1336 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_CMDS_ERR_CNT]); 1337 1338 return ret; 1339 } 1340 1341 static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev, 1342 struct hns_roce_cmq_desc *desc, int num) 1343 { 1344 bool busy; 1345 int ret; 1346 1347 if (hr_dev->cmd.state == HNS_ROCE_CMDQ_STATE_FATAL_ERR) 1348 return -EIO; 1349 1350 if (!v2_chk_mbox_is_avail(hr_dev, &busy)) 1351 return busy ? -EBUSY : 0; 1352 1353 ret = __hns_roce_cmq_send(hr_dev, desc, num); 1354 if (ret) { 1355 if (!v2_chk_mbox_is_avail(hr_dev, &busy)) 1356 return busy ? -EBUSY : 0; 1357 } 1358 1359 return ret; 1360 } 1361 1362 static int config_hem_ba_to_hw(struct hns_roce_dev *hr_dev, 1363 dma_addr_t base_addr, u8 cmd, unsigned long tag) 1364 { 1365 struct hns_roce_cmd_mailbox *mbox; 1366 int ret; 1367 1368 mbox = hns_roce_alloc_cmd_mailbox(hr_dev); 1369 if (IS_ERR(mbox)) 1370 return PTR_ERR(mbox); 1371 1372 ret = hns_roce_cmd_mbox(hr_dev, base_addr, mbox->dma, cmd, tag); 1373 hns_roce_free_cmd_mailbox(hr_dev, mbox); 1374 return ret; 1375 } 1376 1377 static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev) 1378 { 1379 struct hns_roce_query_version *resp; 1380 struct hns_roce_cmq_desc desc; 1381 int ret; 1382 1383 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_HW_VER, true); 1384 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 1385 if (ret) 1386 return ret; 1387 1388 resp = (struct hns_roce_query_version *)desc.data; 1389 hr_dev->hw_rev = le16_to_cpu(resp->rocee_hw_version); 1390 hr_dev->vendor_id = hr_dev->pci_dev->vendor; 1391 1392 return 0; 1393 } 1394 1395 static void func_clr_hw_resetting_state(struct hns_roce_dev *hr_dev, 1396 struct hnae3_handle *handle) 1397 { 1398 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 1399 unsigned long end; 1400 1401 hr_dev->dis_db = true; 1402 1403 dev_warn(hr_dev->dev, 1404 "func clear is pending, device in resetting state.\n"); 1405 end = HNS_ROCE_V2_HW_RST_TIMEOUT; 1406 while (end) { 1407 if (!ops->get_hw_reset_stat(handle)) { 1408 hr_dev->is_reset = true; 1409 dev_info(hr_dev->dev, 1410 "func clear success after reset.\n"); 1411 return; 1412 } 1413 msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT); 1414 end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT; 1415 } 1416 1417 dev_warn(hr_dev->dev, "func clear failed.\n"); 1418 } 1419 1420 static void func_clr_sw_resetting_state(struct hns_roce_dev *hr_dev, 1421 struct hnae3_handle *handle) 1422 { 1423 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 1424 unsigned long end; 1425 1426 hr_dev->dis_db = true; 1427 1428 dev_warn(hr_dev->dev, 1429 "func clear is pending, device in resetting state.\n"); 1430 end = HNS_ROCE_V2_HW_RST_TIMEOUT; 1431 while (end) { 1432 if (ops->ae_dev_reset_cnt(handle) != 1433 hr_dev->reset_cnt) { 1434 hr_dev->is_reset = true; 1435 dev_info(hr_dev->dev, 1436 "func clear success after sw reset\n"); 1437 return; 1438 } 1439 msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT); 1440 end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT; 1441 } 1442 1443 dev_warn(hr_dev->dev, "func clear failed because of unfinished sw reset\n"); 1444 } 1445 1446 static void hns_roce_func_clr_rst_proc(struct hns_roce_dev *hr_dev, int retval, 1447 int flag) 1448 { 1449 struct hns_roce_v2_priv *priv = hr_dev->priv; 1450 struct hnae3_handle *handle = priv->handle; 1451 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 1452 1453 if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt) { 1454 hr_dev->dis_db = true; 1455 hr_dev->is_reset = true; 1456 dev_info(hr_dev->dev, "func clear success after reset.\n"); 1457 return; 1458 } 1459 1460 if (ops->get_hw_reset_stat(handle)) { 1461 func_clr_hw_resetting_state(hr_dev, handle); 1462 return; 1463 } 1464 1465 if (ops->ae_dev_resetting(handle) && 1466 handle->rinfo.instance_state == HNS_ROCE_STATE_INIT) { 1467 func_clr_sw_resetting_state(hr_dev, handle); 1468 return; 1469 } 1470 1471 if (retval && !flag) 1472 dev_warn(hr_dev->dev, 1473 "func clear read failed, ret = %d.\n", retval); 1474 1475 dev_warn(hr_dev->dev, "func clear failed.\n"); 1476 } 1477 1478 static void __hns_roce_function_clear(struct hns_roce_dev *hr_dev, int vf_id) 1479 { 1480 bool fclr_write_fail_flag = false; 1481 struct hns_roce_func_clear *resp; 1482 struct hns_roce_cmq_desc desc; 1483 unsigned long end; 1484 int ret = 0; 1485 1486 if (check_device_is_in_reset(hr_dev)) 1487 goto out; 1488 1489 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, false); 1490 resp = (struct hns_roce_func_clear *)desc.data; 1491 resp->rst_funcid_en = cpu_to_le32(vf_id); 1492 1493 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 1494 if (ret) { 1495 fclr_write_fail_flag = true; 1496 dev_err(hr_dev->dev, "func clear write failed, ret = %d.\n", 1497 ret); 1498 goto out; 1499 } 1500 1501 msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL); 1502 end = HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS; 1503 while (end) { 1504 if (check_device_is_in_reset(hr_dev)) 1505 goto out; 1506 msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT); 1507 end -= HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT; 1508 1509 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, 1510 true); 1511 1512 resp->rst_funcid_en = cpu_to_le32(vf_id); 1513 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 1514 if (ret) 1515 continue; 1516 1517 if (hr_reg_read(resp, FUNC_CLEAR_RST_FUN_DONE)) { 1518 if (vf_id == 0) 1519 hr_dev->is_reset = true; 1520 return; 1521 } 1522 } 1523 1524 out: 1525 hns_roce_func_clr_rst_proc(hr_dev, ret, fclr_write_fail_flag); 1526 } 1527 1528 static int hns_roce_free_vf_resource(struct hns_roce_dev *hr_dev, int vf_id) 1529 { 1530 enum hns_roce_opcode_type opcode = HNS_ROCE_OPC_ALLOC_VF_RES; 1531 struct hns_roce_cmq_desc desc[2]; 1532 struct hns_roce_cmq_req *req_a; 1533 1534 req_a = (struct hns_roce_cmq_req *)desc[0].data; 1535 hns_roce_cmq_setup_basic_desc(&desc[0], opcode, false); 1536 desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 1537 hns_roce_cmq_setup_basic_desc(&desc[1], opcode, false); 1538 hr_reg_write(req_a, FUNC_RES_A_VF_ID, vf_id); 1539 1540 return hns_roce_cmq_send(hr_dev, desc, 2); 1541 } 1542 1543 static void hns_roce_function_clear(struct hns_roce_dev *hr_dev) 1544 { 1545 int ret; 1546 int i; 1547 1548 if (hr_dev->cmd.state == HNS_ROCE_CMDQ_STATE_FATAL_ERR) 1549 return; 1550 1551 for (i = hr_dev->func_num - 1; i >= 0; i--) { 1552 __hns_roce_function_clear(hr_dev, i); 1553 1554 if (i == 0) 1555 continue; 1556 1557 ret = hns_roce_free_vf_resource(hr_dev, i); 1558 if (ret) 1559 ibdev_err(&hr_dev->ib_dev, 1560 "failed to free vf resource, vf_id = %d, ret = %d.\n", 1561 i, ret); 1562 } 1563 } 1564 1565 static int hns_roce_clear_extdb_list_info(struct hns_roce_dev *hr_dev) 1566 { 1567 struct hns_roce_cmq_desc desc; 1568 int ret; 1569 1570 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CLEAR_EXTDB_LIST_INFO, 1571 false); 1572 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 1573 if (ret) 1574 ibdev_err(&hr_dev->ib_dev, 1575 "failed to clear extended doorbell info, ret = %d.\n", 1576 ret); 1577 1578 return ret; 1579 } 1580 1581 static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev) 1582 { 1583 struct hns_roce_query_fw_info *resp; 1584 struct hns_roce_cmq_desc desc; 1585 int ret; 1586 1587 hns_roce_cmq_setup_basic_desc(&desc, HNS_QUERY_FW_VER, true); 1588 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 1589 if (ret) 1590 return ret; 1591 1592 resp = (struct hns_roce_query_fw_info *)desc.data; 1593 hr_dev->caps.fw_ver = (u64)(le32_to_cpu(resp->fw_ver)); 1594 1595 return 0; 1596 } 1597 1598 static int hns_roce_query_func_info(struct hns_roce_dev *hr_dev) 1599 { 1600 struct hns_roce_cmq_desc desc; 1601 int ret; 1602 1603 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) { 1604 hr_dev->func_num = 1; 1605 return 0; 1606 } 1607 1608 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_FUNC_INFO, 1609 true); 1610 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 1611 if (ret) { 1612 hr_dev->func_num = 1; 1613 return ret; 1614 } 1615 1616 hr_dev->func_num = le32_to_cpu(desc.func_info.own_func_num); 1617 hr_dev->cong_algo_tmpl_id = le32_to_cpu(desc.func_info.own_mac_id); 1618 1619 return 0; 1620 } 1621 1622 static int hns_roce_hw_v2_query_counter(struct hns_roce_dev *hr_dev, 1623 u64 *stats, u32 port, int *num_counters) 1624 { 1625 #define CNT_PER_DESC 3 1626 struct hns_roce_cmq_desc *desc; 1627 int bd_idx, cnt_idx; 1628 __le64 *cnt_data; 1629 int desc_num; 1630 int ret; 1631 int i; 1632 1633 if (port > hr_dev->caps.num_ports) 1634 return -EINVAL; 1635 1636 desc_num = DIV_ROUND_UP(HNS_ROCE_HW_CNT_TOTAL, CNT_PER_DESC); 1637 desc = kcalloc(desc_num, sizeof(*desc), GFP_KERNEL); 1638 if (!desc) 1639 return -ENOMEM; 1640 1641 for (i = 0; i < desc_num; i++) { 1642 hns_roce_cmq_setup_basic_desc(&desc[i], 1643 HNS_ROCE_OPC_QUERY_COUNTER, true); 1644 if (i != desc_num - 1) 1645 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 1646 } 1647 1648 ret = hns_roce_cmq_send(hr_dev, desc, desc_num); 1649 if (ret) { 1650 ibdev_err(&hr_dev->ib_dev, 1651 "failed to get counter, ret = %d.\n", ret); 1652 goto err_out; 1653 } 1654 1655 for (i = 0; i < HNS_ROCE_HW_CNT_TOTAL && i < *num_counters; i++) { 1656 bd_idx = i / CNT_PER_DESC; 1657 if (!(desc[bd_idx].flag & HNS_ROCE_CMD_FLAG_NEXT) && 1658 bd_idx != HNS_ROCE_HW_CNT_TOTAL / CNT_PER_DESC) 1659 break; 1660 1661 cnt_data = (__le64 *)&desc[bd_idx].data[0]; 1662 cnt_idx = i % CNT_PER_DESC; 1663 stats[i] = le64_to_cpu(cnt_data[cnt_idx]); 1664 } 1665 *num_counters = i; 1666 1667 err_out: 1668 kfree(desc); 1669 return ret; 1670 } 1671 1672 static int hns_roce_config_global_param(struct hns_roce_dev *hr_dev) 1673 { 1674 struct hns_roce_cmq_desc desc; 1675 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data; 1676 u32 clock_cycles_of_1us; 1677 1678 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GLOBAL_PARAM, 1679 false); 1680 1681 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) 1682 clock_cycles_of_1us = HNS_ROCE_1NS_CFG; 1683 else 1684 clock_cycles_of_1us = HNS_ROCE_1US_CFG; 1685 1686 hr_reg_write(req, CFG_GLOBAL_PARAM_1US_CYCLES, clock_cycles_of_1us); 1687 hr_reg_write(req, CFG_GLOBAL_PARAM_UDP_PORT, ROCE_V2_UDP_DPORT); 1688 1689 return hns_roce_cmq_send(hr_dev, &desc, 1); 1690 } 1691 1692 static int load_func_res_caps(struct hns_roce_dev *hr_dev, bool is_vf) 1693 { 1694 struct hns_roce_cmq_desc desc[2]; 1695 struct hns_roce_cmq_req *r_a = (struct hns_roce_cmq_req *)desc[0].data; 1696 struct hns_roce_cmq_req *r_b = (struct hns_roce_cmq_req *)desc[1].data; 1697 struct hns_roce_caps *caps = &hr_dev->caps; 1698 enum hns_roce_opcode_type opcode; 1699 u32 func_num; 1700 int ret; 1701 1702 if (is_vf) { 1703 opcode = HNS_ROCE_OPC_QUERY_VF_RES; 1704 func_num = 1; 1705 } else { 1706 opcode = HNS_ROCE_OPC_QUERY_PF_RES; 1707 func_num = hr_dev->func_num; 1708 } 1709 1710 hns_roce_cmq_setup_basic_desc(&desc[0], opcode, true); 1711 desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 1712 hns_roce_cmq_setup_basic_desc(&desc[1], opcode, true); 1713 1714 ret = hns_roce_cmq_send(hr_dev, desc, 2); 1715 if (ret) 1716 return ret; 1717 1718 caps->qpc_bt_num = hr_reg_read(r_a, FUNC_RES_A_QPC_BT_NUM) / func_num; 1719 caps->srqc_bt_num = hr_reg_read(r_a, FUNC_RES_A_SRQC_BT_NUM) / func_num; 1720 caps->cqc_bt_num = hr_reg_read(r_a, FUNC_RES_A_CQC_BT_NUM) / func_num; 1721 caps->mpt_bt_num = hr_reg_read(r_a, FUNC_RES_A_MPT_BT_NUM) / func_num; 1722 caps->eqc_bt_num = hr_reg_read(r_a, FUNC_RES_A_EQC_BT_NUM) / func_num; 1723 caps->smac_bt_num = hr_reg_read(r_b, FUNC_RES_B_SMAC_NUM) / func_num; 1724 caps->sgid_bt_num = hr_reg_read(r_b, FUNC_RES_B_SGID_NUM) / func_num; 1725 caps->sccc_bt_num = hr_reg_read(r_b, FUNC_RES_B_SCCC_BT_NUM) / func_num; 1726 1727 if (is_vf) { 1728 caps->sl_num = hr_reg_read(r_b, FUNC_RES_V_QID_NUM) / func_num; 1729 caps->gmv_bt_num = hr_reg_read(r_b, FUNC_RES_V_GMV_BT_NUM) / 1730 func_num; 1731 } else { 1732 caps->sl_num = hr_reg_read(r_b, FUNC_RES_B_QID_NUM) / func_num; 1733 caps->gmv_bt_num = hr_reg_read(r_b, FUNC_RES_B_GMV_BT_NUM) / 1734 func_num; 1735 } 1736 1737 return 0; 1738 } 1739 1740 static int load_pf_timer_res_caps(struct hns_roce_dev *hr_dev) 1741 { 1742 struct hns_roce_cmq_desc desc; 1743 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data; 1744 struct hns_roce_caps *caps = &hr_dev->caps; 1745 int ret; 1746 1747 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_PF_TIMER_RES, 1748 true); 1749 1750 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 1751 if (ret) 1752 return ret; 1753 1754 caps->qpc_timer_bt_num = hr_reg_read(req, PF_TIMER_RES_QPC_ITEM_NUM); 1755 caps->cqc_timer_bt_num = hr_reg_read(req, PF_TIMER_RES_CQC_ITEM_NUM); 1756 1757 return 0; 1758 } 1759 1760 static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev) 1761 { 1762 struct device *dev = hr_dev->dev; 1763 int ret; 1764 1765 ret = load_func_res_caps(hr_dev, false); 1766 if (ret) { 1767 dev_err(dev, "failed to load pf res caps, ret = %d.\n", ret); 1768 return ret; 1769 } 1770 1771 ret = load_pf_timer_res_caps(hr_dev); 1772 if (ret) 1773 dev_err(dev, "failed to load pf timer resource, ret = %d.\n", 1774 ret); 1775 1776 return ret; 1777 } 1778 1779 static int hns_roce_query_vf_resource(struct hns_roce_dev *hr_dev) 1780 { 1781 struct device *dev = hr_dev->dev; 1782 int ret; 1783 1784 ret = load_func_res_caps(hr_dev, true); 1785 if (ret) 1786 dev_err(dev, "failed to load vf res caps, ret = %d.\n", ret); 1787 1788 return ret; 1789 } 1790 1791 static int __hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev, 1792 u32 vf_id) 1793 { 1794 struct hns_roce_vf_switch *swt; 1795 struct hns_roce_cmq_desc desc; 1796 int ret; 1797 1798 swt = (struct hns_roce_vf_switch *)desc.data; 1799 hns_roce_cmq_setup_basic_desc(&desc, HNS_SWITCH_PARAMETER_CFG, true); 1800 swt->rocee_sel |= cpu_to_le32(HNS_ICL_SWITCH_CMD_ROCEE_SEL); 1801 hr_reg_write(swt, VF_SWITCH_VF_ID, vf_id); 1802 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 1803 if (ret) 1804 return ret; 1805 1806 desc.flag = cpu_to_le16(HNS_ROCE_CMD_FLAG_IN); 1807 desc.flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR); 1808 hr_reg_enable(swt, VF_SWITCH_ALW_LPBK); 1809 hr_reg_clear(swt, VF_SWITCH_ALW_LCL_LPBK); 1810 hr_reg_enable(swt, VF_SWITCH_ALW_DST_OVRD); 1811 1812 return hns_roce_cmq_send(hr_dev, &desc, 1); 1813 } 1814 1815 static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev) 1816 { 1817 u32 vf_id; 1818 int ret; 1819 1820 for (vf_id = 0; vf_id < hr_dev->func_num; vf_id++) { 1821 ret = __hns_roce_set_vf_switch_param(hr_dev, vf_id); 1822 if (ret) 1823 return ret; 1824 } 1825 return 0; 1826 } 1827 1828 static int config_vf_hem_resource(struct hns_roce_dev *hr_dev, int vf_id) 1829 { 1830 struct hns_roce_cmq_desc desc[2]; 1831 struct hns_roce_cmq_req *r_a = (struct hns_roce_cmq_req *)desc[0].data; 1832 struct hns_roce_cmq_req *r_b = (struct hns_roce_cmq_req *)desc[1].data; 1833 enum hns_roce_opcode_type opcode = HNS_ROCE_OPC_ALLOC_VF_RES; 1834 struct hns_roce_caps *caps = &hr_dev->caps; 1835 1836 hns_roce_cmq_setup_basic_desc(&desc[0], opcode, false); 1837 desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 1838 hns_roce_cmq_setup_basic_desc(&desc[1], opcode, false); 1839 1840 hr_reg_write(r_a, FUNC_RES_A_VF_ID, vf_id); 1841 1842 hr_reg_write(r_a, FUNC_RES_A_QPC_BT_NUM, caps->qpc_bt_num); 1843 hr_reg_write(r_a, FUNC_RES_A_QPC_BT_IDX, vf_id * caps->qpc_bt_num); 1844 hr_reg_write(r_a, FUNC_RES_A_SRQC_BT_NUM, caps->srqc_bt_num); 1845 hr_reg_write(r_a, FUNC_RES_A_SRQC_BT_IDX, vf_id * caps->srqc_bt_num); 1846 hr_reg_write(r_a, FUNC_RES_A_CQC_BT_NUM, caps->cqc_bt_num); 1847 hr_reg_write(r_a, FUNC_RES_A_CQC_BT_IDX, vf_id * caps->cqc_bt_num); 1848 hr_reg_write(r_a, FUNC_RES_A_MPT_BT_NUM, caps->mpt_bt_num); 1849 hr_reg_write(r_a, FUNC_RES_A_MPT_BT_IDX, vf_id * caps->mpt_bt_num); 1850 hr_reg_write(r_a, FUNC_RES_A_EQC_BT_NUM, caps->eqc_bt_num); 1851 hr_reg_write(r_a, FUNC_RES_A_EQC_BT_IDX, vf_id * caps->eqc_bt_num); 1852 hr_reg_write(r_b, FUNC_RES_V_QID_NUM, caps->sl_num); 1853 hr_reg_write(r_b, FUNC_RES_B_QID_IDX, vf_id * caps->sl_num); 1854 hr_reg_write(r_b, FUNC_RES_B_SCCC_BT_NUM, caps->sccc_bt_num); 1855 hr_reg_write(r_b, FUNC_RES_B_SCCC_BT_IDX, vf_id * caps->sccc_bt_num); 1856 1857 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) { 1858 hr_reg_write(r_b, FUNC_RES_V_GMV_BT_NUM, caps->gmv_bt_num); 1859 hr_reg_write(r_b, FUNC_RES_B_GMV_BT_IDX, 1860 vf_id * caps->gmv_bt_num); 1861 } else { 1862 hr_reg_write(r_b, FUNC_RES_B_SGID_NUM, caps->sgid_bt_num); 1863 hr_reg_write(r_b, FUNC_RES_B_SGID_IDX, 1864 vf_id * caps->sgid_bt_num); 1865 hr_reg_write(r_b, FUNC_RES_B_SMAC_NUM, caps->smac_bt_num); 1866 hr_reg_write(r_b, FUNC_RES_B_SMAC_IDX, 1867 vf_id * caps->smac_bt_num); 1868 } 1869 1870 return hns_roce_cmq_send(hr_dev, desc, 2); 1871 } 1872 1873 static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev) 1874 { 1875 u32 func_num = max_t(u32, 1, hr_dev->func_num); 1876 u32 vf_id; 1877 int ret; 1878 1879 for (vf_id = 0; vf_id < func_num; vf_id++) { 1880 ret = config_vf_hem_resource(hr_dev, vf_id); 1881 if (ret) { 1882 dev_err(hr_dev->dev, 1883 "failed to config vf-%u hem res, ret = %d.\n", 1884 vf_id, ret); 1885 return ret; 1886 } 1887 } 1888 1889 return 0; 1890 } 1891 1892 static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev) 1893 { 1894 struct hns_roce_cmq_desc desc; 1895 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data; 1896 struct hns_roce_caps *caps = &hr_dev->caps; 1897 1898 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_BT_ATTR, false); 1899 1900 hr_reg_write(req, CFG_BT_ATTR_QPC_BA_PGSZ, 1901 caps->qpc_ba_pg_sz + PG_SHIFT_OFFSET); 1902 hr_reg_write(req, CFG_BT_ATTR_QPC_BUF_PGSZ, 1903 caps->qpc_buf_pg_sz + PG_SHIFT_OFFSET); 1904 hr_reg_write(req, CFG_BT_ATTR_QPC_HOPNUM, 1905 to_hr_hem_hopnum(caps->qpc_hop_num, caps->num_qps)); 1906 1907 hr_reg_write(req, CFG_BT_ATTR_SRQC_BA_PGSZ, 1908 caps->srqc_ba_pg_sz + PG_SHIFT_OFFSET); 1909 hr_reg_write(req, CFG_BT_ATTR_SRQC_BUF_PGSZ, 1910 caps->srqc_buf_pg_sz + PG_SHIFT_OFFSET); 1911 hr_reg_write(req, CFG_BT_ATTR_SRQC_HOPNUM, 1912 to_hr_hem_hopnum(caps->srqc_hop_num, caps->num_srqs)); 1913 1914 hr_reg_write(req, CFG_BT_ATTR_CQC_BA_PGSZ, 1915 caps->cqc_ba_pg_sz + PG_SHIFT_OFFSET); 1916 hr_reg_write(req, CFG_BT_ATTR_CQC_BUF_PGSZ, 1917 caps->cqc_buf_pg_sz + PG_SHIFT_OFFSET); 1918 hr_reg_write(req, CFG_BT_ATTR_CQC_HOPNUM, 1919 to_hr_hem_hopnum(caps->cqc_hop_num, caps->num_cqs)); 1920 1921 hr_reg_write(req, CFG_BT_ATTR_MPT_BA_PGSZ, 1922 caps->mpt_ba_pg_sz + PG_SHIFT_OFFSET); 1923 hr_reg_write(req, CFG_BT_ATTR_MPT_BUF_PGSZ, 1924 caps->mpt_buf_pg_sz + PG_SHIFT_OFFSET); 1925 hr_reg_write(req, CFG_BT_ATTR_MPT_HOPNUM, 1926 to_hr_hem_hopnum(caps->mpt_hop_num, caps->num_mtpts)); 1927 1928 hr_reg_write(req, CFG_BT_ATTR_SCCC_BA_PGSZ, 1929 caps->sccc_ba_pg_sz + PG_SHIFT_OFFSET); 1930 hr_reg_write(req, CFG_BT_ATTR_SCCC_BUF_PGSZ, 1931 caps->sccc_buf_pg_sz + PG_SHIFT_OFFSET); 1932 hr_reg_write(req, CFG_BT_ATTR_SCCC_HOPNUM, 1933 to_hr_hem_hopnum(caps->sccc_hop_num, caps->num_qps)); 1934 1935 return hns_roce_cmq_send(hr_dev, &desc, 1); 1936 } 1937 1938 static void calc_pg_sz(u32 obj_num, u32 obj_size, u32 hop_num, u32 ctx_bt_num, 1939 u32 *buf_page_size, u32 *bt_page_size, u32 hem_type) 1940 { 1941 u64 obj_per_chunk; 1942 u64 bt_chunk_size = PAGE_SIZE; 1943 u64 buf_chunk_size = PAGE_SIZE; 1944 u64 obj_per_chunk_default = buf_chunk_size / obj_size; 1945 1946 *buf_page_size = 0; 1947 *bt_page_size = 0; 1948 1949 switch (hop_num) { 1950 case 3: 1951 obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) * 1952 (bt_chunk_size / BA_BYTE_LEN) * 1953 (bt_chunk_size / BA_BYTE_LEN) * 1954 obj_per_chunk_default; 1955 break; 1956 case 2: 1957 obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) * 1958 (bt_chunk_size / BA_BYTE_LEN) * 1959 obj_per_chunk_default; 1960 break; 1961 case 1: 1962 obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) * 1963 obj_per_chunk_default; 1964 break; 1965 case HNS_ROCE_HOP_NUM_0: 1966 obj_per_chunk = ctx_bt_num * obj_per_chunk_default; 1967 break; 1968 default: 1969 pr_err("table %u not support hop_num = %u!\n", hem_type, 1970 hop_num); 1971 return; 1972 } 1973 1974 if (hem_type >= HEM_TYPE_MTT) 1975 *bt_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk)); 1976 else 1977 *buf_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk)); 1978 } 1979 1980 static void set_hem_page_size(struct hns_roce_dev *hr_dev) 1981 { 1982 struct hns_roce_caps *caps = &hr_dev->caps; 1983 1984 /* EQ */ 1985 caps->eqe_ba_pg_sz = 0; 1986 caps->eqe_buf_pg_sz = 0; 1987 1988 /* Link Table */ 1989 caps->llm_buf_pg_sz = 0; 1990 1991 /* MR */ 1992 caps->mpt_ba_pg_sz = 0; 1993 caps->mpt_buf_pg_sz = 0; 1994 caps->pbl_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_16K; 1995 caps->pbl_buf_pg_sz = 0; 1996 calc_pg_sz(caps->num_mtpts, caps->mtpt_entry_sz, caps->mpt_hop_num, 1997 caps->mpt_bt_num, &caps->mpt_buf_pg_sz, &caps->mpt_ba_pg_sz, 1998 HEM_TYPE_MTPT); 1999 2000 /* QP */ 2001 caps->qpc_ba_pg_sz = 0; 2002 caps->qpc_buf_pg_sz = 0; 2003 caps->qpc_timer_ba_pg_sz = 0; 2004 caps->qpc_timer_buf_pg_sz = 0; 2005 caps->sccc_ba_pg_sz = 0; 2006 caps->sccc_buf_pg_sz = 0; 2007 caps->mtt_ba_pg_sz = 0; 2008 caps->mtt_buf_pg_sz = 0; 2009 calc_pg_sz(caps->num_qps, caps->qpc_sz, caps->qpc_hop_num, 2010 caps->qpc_bt_num, &caps->qpc_buf_pg_sz, &caps->qpc_ba_pg_sz, 2011 HEM_TYPE_QPC); 2012 2013 if (caps->flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) 2014 calc_pg_sz(caps->num_qps, caps->sccc_sz, caps->sccc_hop_num, 2015 caps->sccc_bt_num, &caps->sccc_buf_pg_sz, 2016 &caps->sccc_ba_pg_sz, HEM_TYPE_SCCC); 2017 2018 /* CQ */ 2019 caps->cqc_ba_pg_sz = 0; 2020 caps->cqc_buf_pg_sz = 0; 2021 caps->cqc_timer_ba_pg_sz = 0; 2022 caps->cqc_timer_buf_pg_sz = 0; 2023 caps->cqe_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_256K; 2024 caps->cqe_buf_pg_sz = 0; 2025 calc_pg_sz(caps->num_cqs, caps->cqc_entry_sz, caps->cqc_hop_num, 2026 caps->cqc_bt_num, &caps->cqc_buf_pg_sz, &caps->cqc_ba_pg_sz, 2027 HEM_TYPE_CQC); 2028 calc_pg_sz(caps->max_cqes, caps->cqe_sz, caps->cqe_hop_num, 2029 1, &caps->cqe_buf_pg_sz, &caps->cqe_ba_pg_sz, HEM_TYPE_CQE); 2030 2031 /* SRQ */ 2032 if (caps->flags & HNS_ROCE_CAP_FLAG_SRQ) { 2033 caps->srqc_ba_pg_sz = 0; 2034 caps->srqc_buf_pg_sz = 0; 2035 caps->srqwqe_ba_pg_sz = 0; 2036 caps->srqwqe_buf_pg_sz = 0; 2037 caps->idx_ba_pg_sz = 0; 2038 caps->idx_buf_pg_sz = 0; 2039 calc_pg_sz(caps->num_srqs, caps->srqc_entry_sz, 2040 caps->srqc_hop_num, caps->srqc_bt_num, 2041 &caps->srqc_buf_pg_sz, &caps->srqc_ba_pg_sz, 2042 HEM_TYPE_SRQC); 2043 calc_pg_sz(caps->num_srqwqe_segs, caps->mtt_entry_sz, 2044 caps->srqwqe_hop_num, 1, &caps->srqwqe_buf_pg_sz, 2045 &caps->srqwqe_ba_pg_sz, HEM_TYPE_SRQWQE); 2046 calc_pg_sz(caps->num_idx_segs, caps->idx_entry_sz, 2047 caps->idx_hop_num, 1, &caps->idx_buf_pg_sz, 2048 &caps->idx_ba_pg_sz, HEM_TYPE_IDX); 2049 } 2050 2051 /* GMV */ 2052 caps->gmv_ba_pg_sz = 0; 2053 caps->gmv_buf_pg_sz = 0; 2054 } 2055 2056 /* Apply all loaded caps before setting to hardware */ 2057 static void apply_func_caps(struct hns_roce_dev *hr_dev) 2058 { 2059 #define MAX_GID_TBL_LEN 256 2060 struct hns_roce_caps *caps = &hr_dev->caps; 2061 struct hns_roce_v2_priv *priv = hr_dev->priv; 2062 2063 /* The following configurations don't need to be got from firmware. */ 2064 caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ; 2065 caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ; 2066 caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ; 2067 2068 caps->pbl_hop_num = HNS_ROCE_PBL_HOP_NUM; 2069 caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0; 2070 caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0; 2071 2072 caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS; 2073 caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS; 2074 2075 if (!caps->num_comp_vectors) 2076 caps->num_comp_vectors = 2077 min_t(u32, caps->eqc_bt_num - HNS_ROCE_V2_AEQE_VEC_NUM, 2078 (u32)priv->handle->rinfo.num_vectors - 2079 (HNS_ROCE_V2_AEQE_VEC_NUM + HNS_ROCE_V2_ABNORMAL_VEC_NUM)); 2080 2081 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) { 2082 caps->eqe_hop_num = HNS_ROCE_V3_EQE_HOP_NUM; 2083 caps->ceqe_size = HNS_ROCE_V3_EQE_SIZE; 2084 caps->aeqe_size = HNS_ROCE_V3_EQE_SIZE; 2085 2086 /* The following configurations will be overwritten */ 2087 caps->qpc_sz = HNS_ROCE_V3_QPC_SZ; 2088 caps->cqe_sz = HNS_ROCE_V3_CQE_SIZE; 2089 caps->sccc_sz = HNS_ROCE_V3_SCCC_SZ; 2090 2091 /* The following configurations are not got from firmware */ 2092 caps->gmv_entry_sz = HNS_ROCE_V3_GMV_ENTRY_SZ; 2093 2094 caps->gmv_hop_num = HNS_ROCE_HOP_NUM_0; 2095 2096 /* It's meaningless to support excessively large gid_table_len, 2097 * as the type of sgid_index in kernel struct ib_global_route 2098 * and userspace struct ibv_global_route are u8/uint8_t (0-255). 2099 */ 2100 caps->gid_table_len[0] = min_t(u32, MAX_GID_TBL_LEN, 2101 caps->gmv_bt_num * 2102 (HNS_HW_PAGE_SIZE / caps->gmv_entry_sz)); 2103 2104 caps->gmv_entry_num = caps->gmv_bt_num * (HNS_HW_PAGE_SIZE / 2105 caps->gmv_entry_sz); 2106 } else { 2107 u32 func_num = max_t(u32, 1, hr_dev->func_num); 2108 2109 caps->eqe_hop_num = HNS_ROCE_V2_EQE_HOP_NUM; 2110 caps->ceqe_size = HNS_ROCE_CEQE_SIZE; 2111 caps->aeqe_size = HNS_ROCE_AEQE_SIZE; 2112 caps->gid_table_len[0] /= func_num; 2113 } 2114 2115 if (hr_dev->is_vf) { 2116 caps->default_aeq_arm_st = 0x3; 2117 caps->default_ceq_arm_st = 0x3; 2118 caps->default_ceq_max_cnt = 0x1; 2119 caps->default_ceq_period = 0x10; 2120 caps->default_aeq_max_cnt = 0x1; 2121 caps->default_aeq_period = 0x10; 2122 } 2123 2124 set_hem_page_size(hr_dev); 2125 } 2126 2127 static int hns_roce_query_caps(struct hns_roce_dev *hr_dev) 2128 { 2129 struct hns_roce_cmq_desc desc[HNS_ROCE_QUERY_PF_CAPS_CMD_NUM]; 2130 struct hns_roce_caps *caps = &hr_dev->caps; 2131 struct hns_roce_query_pf_caps_a *resp_a; 2132 struct hns_roce_query_pf_caps_b *resp_b; 2133 struct hns_roce_query_pf_caps_c *resp_c; 2134 struct hns_roce_query_pf_caps_d *resp_d; 2135 struct hns_roce_query_pf_caps_e *resp_e; 2136 enum hns_roce_opcode_type cmd; 2137 int ctx_hop_num; 2138 int pbl_hop_num; 2139 int ret; 2140 int i; 2141 2142 cmd = hr_dev->is_vf ? HNS_ROCE_OPC_QUERY_VF_CAPS_NUM : 2143 HNS_ROCE_OPC_QUERY_PF_CAPS_NUM; 2144 2145 for (i = 0; i < HNS_ROCE_QUERY_PF_CAPS_CMD_NUM; i++) { 2146 hns_roce_cmq_setup_basic_desc(&desc[i], cmd, true); 2147 if (i < (HNS_ROCE_QUERY_PF_CAPS_CMD_NUM - 1)) 2148 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 2149 else 2150 desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 2151 } 2152 2153 ret = hns_roce_cmq_send(hr_dev, desc, HNS_ROCE_QUERY_PF_CAPS_CMD_NUM); 2154 if (ret) 2155 return ret; 2156 2157 resp_a = (struct hns_roce_query_pf_caps_a *)desc[0].data; 2158 resp_b = (struct hns_roce_query_pf_caps_b *)desc[1].data; 2159 resp_c = (struct hns_roce_query_pf_caps_c *)desc[2].data; 2160 resp_d = (struct hns_roce_query_pf_caps_d *)desc[3].data; 2161 resp_e = (struct hns_roce_query_pf_caps_e *)desc[4].data; 2162 2163 caps->local_ca_ack_delay = resp_a->local_ca_ack_delay; 2164 caps->max_sq_sg = le16_to_cpu(resp_a->max_sq_sg); 2165 caps->max_sq_inline = le16_to_cpu(resp_a->max_sq_inline); 2166 caps->max_rq_sg = le16_to_cpu(resp_a->max_rq_sg); 2167 caps->max_rq_sg = roundup_pow_of_two(caps->max_rq_sg); 2168 caps->max_srq_sges = le16_to_cpu(resp_a->max_srq_sges); 2169 caps->max_srq_sges = roundup_pow_of_two(caps->max_srq_sges); 2170 caps->num_aeq_vectors = resp_a->num_aeq_vectors; 2171 caps->num_other_vectors = resp_a->num_other_vectors; 2172 caps->max_sq_desc_sz = resp_a->max_sq_desc_sz; 2173 caps->max_rq_desc_sz = resp_a->max_rq_desc_sz; 2174 2175 caps->mtpt_entry_sz = resp_b->mtpt_entry_sz; 2176 caps->irrl_entry_sz = resp_b->irrl_entry_sz; 2177 caps->trrl_entry_sz = resp_b->trrl_entry_sz; 2178 caps->cqc_entry_sz = resp_b->cqc_entry_sz; 2179 caps->srqc_entry_sz = resp_b->srqc_entry_sz; 2180 caps->idx_entry_sz = resp_b->idx_entry_sz; 2181 caps->sccc_sz = resp_b->sccc_sz; 2182 caps->max_mtu = resp_b->max_mtu; 2183 caps->min_cqes = resp_b->min_cqes; 2184 caps->min_wqes = resp_b->min_wqes; 2185 caps->page_size_cap = le32_to_cpu(resp_b->page_size_cap); 2186 caps->pkey_table_len[0] = resp_b->pkey_table_len; 2187 caps->phy_num_uars = resp_b->phy_num_uars; 2188 ctx_hop_num = resp_b->ctx_hop_num; 2189 pbl_hop_num = resp_b->pbl_hop_num; 2190 2191 caps->num_pds = 1 << hr_reg_read(resp_c, PF_CAPS_C_NUM_PDS); 2192 2193 caps->flags = hr_reg_read(resp_c, PF_CAPS_C_CAP_FLAGS); 2194 caps->flags |= le16_to_cpu(resp_d->cap_flags_ex) << 2195 HNS_ROCE_CAP_FLAGS_EX_SHIFT; 2196 2197 caps->num_cqs = 1 << hr_reg_read(resp_c, PF_CAPS_C_NUM_CQS); 2198 caps->gid_table_len[0] = hr_reg_read(resp_c, PF_CAPS_C_MAX_GID); 2199 caps->max_cqes = 1 << hr_reg_read(resp_c, PF_CAPS_C_CQ_DEPTH); 2200 caps->num_xrcds = 1 << hr_reg_read(resp_c, PF_CAPS_C_NUM_XRCDS); 2201 caps->num_mtpts = 1 << hr_reg_read(resp_c, PF_CAPS_C_NUM_MRWS); 2202 caps->num_qps = 1 << hr_reg_read(resp_c, PF_CAPS_C_NUM_QPS); 2203 caps->max_qp_init_rdma = hr_reg_read(resp_c, PF_CAPS_C_MAX_ORD); 2204 caps->max_qp_dest_rdma = caps->max_qp_init_rdma; 2205 caps->max_wqes = 1 << le16_to_cpu(resp_c->sq_depth); 2206 2207 caps->num_srqs = 1 << hr_reg_read(resp_d, PF_CAPS_D_NUM_SRQS); 2208 caps->cong_cap = hr_reg_read(resp_d, PF_CAPS_D_CONG_CAP); 2209 caps->max_srq_wrs = 1 << le16_to_cpu(resp_d->srq_depth); 2210 caps->ceqe_depth = 1 << hr_reg_read(resp_d, PF_CAPS_D_CEQ_DEPTH); 2211 caps->num_comp_vectors = hr_reg_read(resp_d, PF_CAPS_D_NUM_CEQS); 2212 caps->aeqe_depth = 1 << hr_reg_read(resp_d, PF_CAPS_D_AEQ_DEPTH); 2213 caps->default_cong_type = hr_reg_read(resp_d, PF_CAPS_D_DEFAULT_ALG); 2214 caps->reserved_pds = hr_reg_read(resp_d, PF_CAPS_D_RSV_PDS); 2215 caps->num_uars = 1 << hr_reg_read(resp_d, PF_CAPS_D_NUM_UARS); 2216 caps->reserved_qps = hr_reg_read(resp_d, PF_CAPS_D_RSV_QPS); 2217 caps->reserved_uars = hr_reg_read(resp_d, PF_CAPS_D_RSV_UARS); 2218 2219 caps->reserved_mrws = hr_reg_read(resp_e, PF_CAPS_E_RSV_MRWS); 2220 caps->chunk_sz = 1 << hr_reg_read(resp_e, PF_CAPS_E_CHUNK_SIZE_SHIFT); 2221 caps->reserved_cqs = hr_reg_read(resp_e, PF_CAPS_E_RSV_CQS); 2222 caps->reserved_xrcds = hr_reg_read(resp_e, PF_CAPS_E_RSV_XRCDS); 2223 caps->reserved_srqs = hr_reg_read(resp_e, PF_CAPS_E_RSV_SRQS); 2224 caps->reserved_lkey = hr_reg_read(resp_e, PF_CAPS_E_RSV_LKEYS); 2225 2226 caps->qpc_hop_num = ctx_hop_num; 2227 caps->sccc_hop_num = ctx_hop_num; 2228 caps->srqc_hop_num = ctx_hop_num; 2229 caps->cqc_hop_num = ctx_hop_num; 2230 caps->mpt_hop_num = ctx_hop_num; 2231 caps->mtt_hop_num = pbl_hop_num; 2232 caps->cqe_hop_num = pbl_hop_num; 2233 caps->srqwqe_hop_num = pbl_hop_num; 2234 caps->idx_hop_num = pbl_hop_num; 2235 caps->wqe_sq_hop_num = hr_reg_read(resp_d, PF_CAPS_D_SQWQE_HOP_NUM); 2236 caps->wqe_sge_hop_num = hr_reg_read(resp_d, PF_CAPS_D_EX_SGE_HOP_NUM); 2237 caps->wqe_rq_hop_num = hr_reg_read(resp_d, PF_CAPS_D_RQWQE_HOP_NUM); 2238 2239 if (!(caps->page_size_cap & PAGE_SIZE)) 2240 caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED; 2241 2242 if (!hr_dev->is_vf) { 2243 caps->cqe_sz = resp_a->cqe_sz; 2244 caps->qpc_sz = le16_to_cpu(resp_b->qpc_sz); 2245 caps->default_aeq_arm_st = 2246 hr_reg_read(resp_d, PF_CAPS_D_AEQ_ARM_ST); 2247 caps->default_ceq_arm_st = 2248 hr_reg_read(resp_d, PF_CAPS_D_CEQ_ARM_ST); 2249 caps->default_ceq_max_cnt = le16_to_cpu(resp_e->ceq_max_cnt); 2250 caps->default_ceq_period = le16_to_cpu(resp_e->ceq_period); 2251 caps->default_aeq_max_cnt = le16_to_cpu(resp_e->aeq_max_cnt); 2252 caps->default_aeq_period = le16_to_cpu(resp_e->aeq_period); 2253 } 2254 2255 return 0; 2256 } 2257 2258 static int config_hem_entry_size(struct hns_roce_dev *hr_dev, u32 type, u32 val) 2259 { 2260 struct hns_roce_cmq_desc desc; 2261 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data; 2262 2263 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_ENTRY_SIZE, 2264 false); 2265 2266 hr_reg_write(req, CFG_HEM_ENTRY_SIZE_TYPE, type); 2267 hr_reg_write(req, CFG_HEM_ENTRY_SIZE_VALUE, val); 2268 2269 return hns_roce_cmq_send(hr_dev, &desc, 1); 2270 } 2271 2272 static int hns_roce_config_entry_size(struct hns_roce_dev *hr_dev) 2273 { 2274 struct hns_roce_caps *caps = &hr_dev->caps; 2275 int ret; 2276 2277 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) 2278 return 0; 2279 2280 ret = config_hem_entry_size(hr_dev, HNS_ROCE_CFG_QPC_SIZE, 2281 caps->qpc_sz); 2282 if (ret) { 2283 dev_err(hr_dev->dev, "failed to cfg qpc sz, ret = %d.\n", ret); 2284 return ret; 2285 } 2286 2287 ret = config_hem_entry_size(hr_dev, HNS_ROCE_CFG_SCCC_SIZE, 2288 caps->sccc_sz); 2289 if (ret) 2290 dev_err(hr_dev->dev, "failed to cfg sccc sz, ret = %d.\n", ret); 2291 2292 return ret; 2293 } 2294 2295 static int hns_roce_v2_vf_profile(struct hns_roce_dev *hr_dev) 2296 { 2297 struct device *dev = hr_dev->dev; 2298 int ret; 2299 2300 hr_dev->func_num = 1; 2301 2302 ret = hns_roce_query_caps(hr_dev); 2303 if (ret) { 2304 dev_err(dev, "failed to query VF caps, ret = %d.\n", ret); 2305 return ret; 2306 } 2307 2308 ret = hns_roce_query_vf_resource(hr_dev); 2309 if (ret) { 2310 dev_err(dev, "failed to query VF resource, ret = %d.\n", ret); 2311 return ret; 2312 } 2313 2314 apply_func_caps(hr_dev); 2315 2316 ret = hns_roce_v2_set_bt(hr_dev); 2317 if (ret) 2318 dev_err(dev, "failed to config VF BA table, ret = %d.\n", ret); 2319 2320 return ret; 2321 } 2322 2323 static int hns_roce_v2_pf_profile(struct hns_roce_dev *hr_dev) 2324 { 2325 struct device *dev = hr_dev->dev; 2326 int ret; 2327 2328 ret = hns_roce_query_func_info(hr_dev); 2329 if (ret) { 2330 dev_err(dev, "failed to query func info, ret = %d.\n", ret); 2331 return ret; 2332 } 2333 2334 ret = hns_roce_config_global_param(hr_dev); 2335 if (ret) { 2336 dev_err(dev, "failed to config global param, ret = %d.\n", ret); 2337 return ret; 2338 } 2339 2340 ret = hns_roce_set_vf_switch_param(hr_dev); 2341 if (ret) { 2342 dev_err(dev, "failed to set switch param, ret = %d.\n", ret); 2343 return ret; 2344 } 2345 2346 ret = hns_roce_query_caps(hr_dev); 2347 if (ret) { 2348 dev_err(dev, "failed to query PF caps, ret = %d.\n", ret); 2349 return ret; 2350 } 2351 2352 ret = hns_roce_query_pf_resource(hr_dev); 2353 if (ret) { 2354 dev_err(dev, "failed to query pf resource, ret = %d.\n", ret); 2355 return ret; 2356 } 2357 2358 apply_func_caps(hr_dev); 2359 2360 ret = hns_roce_alloc_vf_resource(hr_dev); 2361 if (ret) { 2362 dev_err(dev, "failed to alloc vf resource, ret = %d.\n", ret); 2363 return ret; 2364 } 2365 2366 ret = hns_roce_v2_set_bt(hr_dev); 2367 if (ret) { 2368 dev_err(dev, "failed to config BA table, ret = %d.\n", ret); 2369 return ret; 2370 } 2371 2372 /* Configure the size of QPC, SCCC, etc. */ 2373 return hns_roce_config_entry_size(hr_dev); 2374 } 2375 2376 static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) 2377 { 2378 struct device *dev = hr_dev->dev; 2379 int ret; 2380 2381 ret = hns_roce_cmq_query_hw_info(hr_dev); 2382 if (ret) { 2383 dev_err(dev, "failed to query hardware info, ret = %d.\n", ret); 2384 return ret; 2385 } 2386 2387 ret = hns_roce_query_fw_ver(hr_dev); 2388 if (ret) { 2389 dev_err(dev, "failed to query firmware info, ret = %d.\n", ret); 2390 return ret; 2391 } 2392 2393 hr_dev->vendor_part_id = hr_dev->pci_dev->device; 2394 hr_dev->sys_image_guid = be64_to_cpu(hr_dev->ib_dev.node_guid); 2395 2396 if (hr_dev->is_vf) 2397 return hns_roce_v2_vf_profile(hr_dev); 2398 else 2399 return hns_roce_v2_pf_profile(hr_dev); 2400 } 2401 2402 static void config_llm_table(struct hns_roce_buf *data_buf, void *cfg_buf) 2403 { 2404 u32 i, next_ptr, page_num; 2405 __le64 *entry = cfg_buf; 2406 dma_addr_t addr; 2407 u64 val; 2408 2409 page_num = data_buf->npages; 2410 for (i = 0; i < page_num; i++) { 2411 addr = hns_roce_buf_page(data_buf, i); 2412 if (i == (page_num - 1)) 2413 next_ptr = 0; 2414 else 2415 next_ptr = i + 1; 2416 2417 val = HNS_ROCE_EXT_LLM_ENTRY(addr, (u64)next_ptr); 2418 entry[i] = cpu_to_le64(val); 2419 } 2420 } 2421 2422 static int set_llm_cfg_to_hw(struct hns_roce_dev *hr_dev, 2423 struct hns_roce_link_table *table) 2424 { 2425 struct hns_roce_cmq_desc desc[2]; 2426 struct hns_roce_cmq_req *r_a = (struct hns_roce_cmq_req *)desc[0].data; 2427 struct hns_roce_cmq_req *r_b = (struct hns_roce_cmq_req *)desc[1].data; 2428 struct hns_roce_buf *buf = table->buf; 2429 enum hns_roce_opcode_type opcode; 2430 dma_addr_t addr; 2431 2432 opcode = HNS_ROCE_OPC_CFG_EXT_LLM; 2433 hns_roce_cmq_setup_basic_desc(&desc[0], opcode, false); 2434 desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 2435 hns_roce_cmq_setup_basic_desc(&desc[1], opcode, false); 2436 2437 hr_reg_write(r_a, CFG_LLM_A_BA_L, lower_32_bits(table->table.map)); 2438 hr_reg_write(r_a, CFG_LLM_A_BA_H, upper_32_bits(table->table.map)); 2439 hr_reg_write(r_a, CFG_LLM_A_DEPTH, buf->npages); 2440 hr_reg_write(r_a, CFG_LLM_A_PGSZ, to_hr_hw_page_shift(buf->page_shift)); 2441 hr_reg_enable(r_a, CFG_LLM_A_INIT_EN); 2442 2443 addr = to_hr_hw_page_addr(hns_roce_buf_page(buf, 0)); 2444 hr_reg_write(r_a, CFG_LLM_A_HEAD_BA_L, lower_32_bits(addr)); 2445 hr_reg_write(r_a, CFG_LLM_A_HEAD_BA_H, upper_32_bits(addr)); 2446 hr_reg_write(r_a, CFG_LLM_A_HEAD_NXTPTR, 1); 2447 hr_reg_write(r_a, CFG_LLM_A_HEAD_PTR, 0); 2448 2449 addr = to_hr_hw_page_addr(hns_roce_buf_page(buf, buf->npages - 1)); 2450 hr_reg_write(r_b, CFG_LLM_B_TAIL_BA_L, lower_32_bits(addr)); 2451 hr_reg_write(r_b, CFG_LLM_B_TAIL_BA_H, upper_32_bits(addr)); 2452 hr_reg_write(r_b, CFG_LLM_B_TAIL_PTR, buf->npages - 1); 2453 2454 return hns_roce_cmq_send(hr_dev, desc, 2); 2455 } 2456 2457 static struct hns_roce_link_table * 2458 alloc_link_table_buf(struct hns_roce_dev *hr_dev) 2459 { 2460 struct hns_roce_v2_priv *priv = hr_dev->priv; 2461 struct hns_roce_link_table *link_tbl; 2462 u32 pg_shift, size, min_size; 2463 2464 link_tbl = &priv->ext_llm; 2465 pg_shift = hr_dev->caps.llm_buf_pg_sz + PAGE_SHIFT; 2466 size = hr_dev->caps.num_qps * HNS_ROCE_V2_EXT_LLM_ENTRY_SZ; 2467 min_size = HNS_ROCE_EXT_LLM_MIN_PAGES(hr_dev->caps.sl_num) << pg_shift; 2468 2469 /* Alloc data table */ 2470 size = max(size, min_size); 2471 link_tbl->buf = hns_roce_buf_alloc(hr_dev, size, pg_shift, 0); 2472 if (IS_ERR(link_tbl->buf)) 2473 return ERR_PTR(-ENOMEM); 2474 2475 /* Alloc config table */ 2476 size = link_tbl->buf->npages * sizeof(u64); 2477 link_tbl->table.buf = dma_alloc_coherent(hr_dev->dev, size, 2478 &link_tbl->table.map, 2479 GFP_KERNEL); 2480 if (!link_tbl->table.buf) { 2481 hns_roce_buf_free(hr_dev, link_tbl->buf); 2482 return ERR_PTR(-ENOMEM); 2483 } 2484 2485 return link_tbl; 2486 } 2487 2488 static void free_link_table_buf(struct hns_roce_dev *hr_dev, 2489 struct hns_roce_link_table *tbl) 2490 { 2491 if (tbl->buf) { 2492 u32 size = tbl->buf->npages * sizeof(u64); 2493 2494 dma_free_coherent(hr_dev->dev, size, tbl->table.buf, 2495 tbl->table.map); 2496 } 2497 2498 hns_roce_buf_free(hr_dev, tbl->buf); 2499 } 2500 2501 static int hns_roce_init_link_table(struct hns_roce_dev *hr_dev) 2502 { 2503 struct hns_roce_link_table *link_tbl; 2504 int ret; 2505 2506 link_tbl = alloc_link_table_buf(hr_dev); 2507 if (IS_ERR(link_tbl)) 2508 return -ENOMEM; 2509 2510 if (WARN_ON(link_tbl->buf->npages > HNS_ROCE_V2_EXT_LLM_MAX_DEPTH)) { 2511 ret = -EINVAL; 2512 goto err_alloc; 2513 } 2514 2515 config_llm_table(link_tbl->buf, link_tbl->table.buf); 2516 ret = set_llm_cfg_to_hw(hr_dev, link_tbl); 2517 if (ret) 2518 goto err_alloc; 2519 2520 return 0; 2521 2522 err_alloc: 2523 free_link_table_buf(hr_dev, link_tbl); 2524 return ret; 2525 } 2526 2527 static void hns_roce_free_link_table(struct hns_roce_dev *hr_dev) 2528 { 2529 struct hns_roce_v2_priv *priv = hr_dev->priv; 2530 2531 free_link_table_buf(hr_dev, &priv->ext_llm); 2532 } 2533 2534 static void free_dip_list(struct hns_roce_dev *hr_dev) 2535 { 2536 struct hns_roce_dip *hr_dip; 2537 struct hns_roce_dip *tmp; 2538 unsigned long flags; 2539 2540 spin_lock_irqsave(&hr_dev->dip_list_lock, flags); 2541 2542 list_for_each_entry_safe(hr_dip, tmp, &hr_dev->dip_list, node) { 2543 list_del(&hr_dip->node); 2544 kfree(hr_dip); 2545 } 2546 2547 spin_unlock_irqrestore(&hr_dev->dip_list_lock, flags); 2548 } 2549 2550 static struct ib_pd *free_mr_init_pd(struct hns_roce_dev *hr_dev) 2551 { 2552 struct hns_roce_v2_priv *priv = hr_dev->priv; 2553 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr; 2554 struct ib_device *ibdev = &hr_dev->ib_dev; 2555 struct hns_roce_pd *hr_pd; 2556 struct ib_pd *pd; 2557 2558 hr_pd = kzalloc(sizeof(*hr_pd), GFP_KERNEL); 2559 if (ZERO_OR_NULL_PTR(hr_pd)) 2560 return NULL; 2561 pd = &hr_pd->ibpd; 2562 pd->device = ibdev; 2563 2564 if (hns_roce_alloc_pd(pd, NULL)) { 2565 ibdev_err(ibdev, "failed to create pd for free mr.\n"); 2566 kfree(hr_pd); 2567 return NULL; 2568 } 2569 free_mr->rsv_pd = to_hr_pd(pd); 2570 free_mr->rsv_pd->ibpd.device = &hr_dev->ib_dev; 2571 free_mr->rsv_pd->ibpd.uobject = NULL; 2572 free_mr->rsv_pd->ibpd.__internal_mr = NULL; 2573 atomic_set(&free_mr->rsv_pd->ibpd.usecnt, 0); 2574 2575 return pd; 2576 } 2577 2578 static struct ib_cq *free_mr_init_cq(struct hns_roce_dev *hr_dev) 2579 { 2580 struct hns_roce_v2_priv *priv = hr_dev->priv; 2581 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr; 2582 struct ib_device *ibdev = &hr_dev->ib_dev; 2583 struct ib_cq_init_attr cq_init_attr = {}; 2584 struct hns_roce_cq *hr_cq; 2585 struct ib_cq *cq; 2586 2587 cq_init_attr.cqe = HNS_ROCE_FREE_MR_USED_CQE_NUM; 2588 2589 hr_cq = kzalloc(sizeof(*hr_cq), GFP_KERNEL); 2590 if (ZERO_OR_NULL_PTR(hr_cq)) 2591 return NULL; 2592 2593 cq = &hr_cq->ib_cq; 2594 cq->device = ibdev; 2595 2596 if (hns_roce_create_cq(cq, &cq_init_attr, NULL)) { 2597 ibdev_err(ibdev, "failed to create cq for free mr.\n"); 2598 kfree(hr_cq); 2599 return NULL; 2600 } 2601 free_mr->rsv_cq = to_hr_cq(cq); 2602 free_mr->rsv_cq->ib_cq.device = &hr_dev->ib_dev; 2603 free_mr->rsv_cq->ib_cq.uobject = NULL; 2604 free_mr->rsv_cq->ib_cq.comp_handler = NULL; 2605 free_mr->rsv_cq->ib_cq.event_handler = NULL; 2606 free_mr->rsv_cq->ib_cq.cq_context = NULL; 2607 atomic_set(&free_mr->rsv_cq->ib_cq.usecnt, 0); 2608 2609 return cq; 2610 } 2611 2612 static int free_mr_init_qp(struct hns_roce_dev *hr_dev, struct ib_cq *cq, 2613 struct ib_qp_init_attr *init_attr, int i) 2614 { 2615 struct hns_roce_v2_priv *priv = hr_dev->priv; 2616 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr; 2617 struct ib_device *ibdev = &hr_dev->ib_dev; 2618 struct hns_roce_qp *hr_qp; 2619 struct ib_qp *qp; 2620 int ret; 2621 2622 hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL); 2623 if (ZERO_OR_NULL_PTR(hr_qp)) 2624 return -ENOMEM; 2625 2626 qp = &hr_qp->ibqp; 2627 qp->device = ibdev; 2628 2629 ret = hns_roce_create_qp(qp, init_attr, NULL); 2630 if (ret) { 2631 ibdev_err(ibdev, "failed to create qp for free mr.\n"); 2632 kfree(hr_qp); 2633 return ret; 2634 } 2635 2636 free_mr->rsv_qp[i] = hr_qp; 2637 free_mr->rsv_qp[i]->ibqp.recv_cq = cq; 2638 free_mr->rsv_qp[i]->ibqp.send_cq = cq; 2639 2640 return 0; 2641 } 2642 2643 static void free_mr_exit(struct hns_roce_dev *hr_dev) 2644 { 2645 struct hns_roce_v2_priv *priv = hr_dev->priv; 2646 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr; 2647 struct ib_qp *qp; 2648 int i; 2649 2650 for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) { 2651 if (free_mr->rsv_qp[i]) { 2652 qp = &free_mr->rsv_qp[i]->ibqp; 2653 hns_roce_v2_destroy_qp(qp, NULL); 2654 kfree(free_mr->rsv_qp[i]); 2655 free_mr->rsv_qp[i] = NULL; 2656 } 2657 } 2658 2659 if (free_mr->rsv_cq) { 2660 hns_roce_destroy_cq(&free_mr->rsv_cq->ib_cq, NULL); 2661 kfree(free_mr->rsv_cq); 2662 free_mr->rsv_cq = NULL; 2663 } 2664 2665 if (free_mr->rsv_pd) { 2666 hns_roce_dealloc_pd(&free_mr->rsv_pd->ibpd, NULL); 2667 kfree(free_mr->rsv_pd); 2668 free_mr->rsv_pd = NULL; 2669 } 2670 2671 mutex_destroy(&free_mr->mutex); 2672 } 2673 2674 static int free_mr_alloc_res(struct hns_roce_dev *hr_dev) 2675 { 2676 struct hns_roce_v2_priv *priv = hr_dev->priv; 2677 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr; 2678 struct ib_qp_init_attr qp_init_attr = {}; 2679 struct ib_pd *pd; 2680 struct ib_cq *cq; 2681 int ret; 2682 int i; 2683 2684 pd = free_mr_init_pd(hr_dev); 2685 if (!pd) 2686 return -ENOMEM; 2687 2688 cq = free_mr_init_cq(hr_dev); 2689 if (!cq) { 2690 ret = -ENOMEM; 2691 goto create_failed_cq; 2692 } 2693 2694 qp_init_attr.qp_type = IB_QPT_RC; 2695 qp_init_attr.sq_sig_type = IB_SIGNAL_ALL_WR; 2696 qp_init_attr.send_cq = cq; 2697 qp_init_attr.recv_cq = cq; 2698 for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) { 2699 qp_init_attr.cap.max_send_wr = HNS_ROCE_FREE_MR_USED_SQWQE_NUM; 2700 qp_init_attr.cap.max_send_sge = HNS_ROCE_FREE_MR_USED_SQSGE_NUM; 2701 qp_init_attr.cap.max_recv_wr = HNS_ROCE_FREE_MR_USED_RQWQE_NUM; 2702 qp_init_attr.cap.max_recv_sge = HNS_ROCE_FREE_MR_USED_RQSGE_NUM; 2703 2704 ret = free_mr_init_qp(hr_dev, cq, &qp_init_attr, i); 2705 if (ret) 2706 goto create_failed_qp; 2707 } 2708 2709 return 0; 2710 2711 create_failed_qp: 2712 for (i--; i >= 0; i--) { 2713 hns_roce_v2_destroy_qp(&free_mr->rsv_qp[i]->ibqp, NULL); 2714 kfree(free_mr->rsv_qp[i]); 2715 } 2716 hns_roce_destroy_cq(cq, NULL); 2717 kfree(cq); 2718 2719 create_failed_cq: 2720 hns_roce_dealloc_pd(pd, NULL); 2721 kfree(pd); 2722 2723 return ret; 2724 } 2725 2726 static int free_mr_modify_rsv_qp(struct hns_roce_dev *hr_dev, 2727 struct ib_qp_attr *attr, int sl_num) 2728 { 2729 struct hns_roce_v2_priv *priv = hr_dev->priv; 2730 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr; 2731 struct ib_device *ibdev = &hr_dev->ib_dev; 2732 struct hns_roce_qp *hr_qp; 2733 int loopback; 2734 int mask; 2735 int ret; 2736 2737 hr_qp = to_hr_qp(&free_mr->rsv_qp[sl_num]->ibqp); 2738 hr_qp->free_mr_en = 1; 2739 hr_qp->ibqp.device = ibdev; 2740 hr_qp->ibqp.qp_type = IB_QPT_RC; 2741 2742 mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS; 2743 attr->qp_state = IB_QPS_INIT; 2744 attr->port_num = 1; 2745 attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE; 2746 ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_INIT, 2747 IB_QPS_INIT, NULL); 2748 if (ret) { 2749 ibdev_err(ibdev, "failed to modify qp to init, ret = %d.\n", 2750 ret); 2751 return ret; 2752 } 2753 2754 loopback = hr_dev->loop_idc; 2755 /* Set qpc lbi = 1 incidate loopback IO */ 2756 hr_dev->loop_idc = 1; 2757 2758 mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU | IB_QP_DEST_QPN | 2759 IB_QP_RQ_PSN | IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER; 2760 attr->qp_state = IB_QPS_RTR; 2761 attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 2762 attr->path_mtu = IB_MTU_256; 2763 attr->dest_qp_num = hr_qp->qpn; 2764 attr->rq_psn = HNS_ROCE_FREE_MR_USED_PSN; 2765 2766 rdma_ah_set_sl(&attr->ah_attr, (u8)sl_num); 2767 2768 ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_INIT, 2769 IB_QPS_RTR, NULL); 2770 hr_dev->loop_idc = loopback; 2771 if (ret) { 2772 ibdev_err(ibdev, "failed to modify qp to rtr, ret = %d.\n", 2773 ret); 2774 return ret; 2775 } 2776 2777 mask = IB_QP_STATE | IB_QP_SQ_PSN | IB_QP_RETRY_CNT | IB_QP_TIMEOUT | 2778 IB_QP_RNR_RETRY | IB_QP_MAX_QP_RD_ATOMIC; 2779 attr->qp_state = IB_QPS_RTS; 2780 attr->sq_psn = HNS_ROCE_FREE_MR_USED_PSN; 2781 attr->retry_cnt = HNS_ROCE_FREE_MR_USED_QP_RETRY_CNT; 2782 attr->timeout = HNS_ROCE_FREE_MR_USED_QP_TIMEOUT; 2783 ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_RTR, 2784 IB_QPS_RTS, NULL); 2785 if (ret) 2786 ibdev_err(ibdev, "failed to modify qp to rts, ret = %d.\n", 2787 ret); 2788 2789 return ret; 2790 } 2791 2792 static int free_mr_modify_qp(struct hns_roce_dev *hr_dev) 2793 { 2794 struct hns_roce_v2_priv *priv = hr_dev->priv; 2795 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr; 2796 struct ib_qp_attr attr = {}; 2797 int ret; 2798 int i; 2799 2800 rdma_ah_set_grh(&attr.ah_attr, NULL, 0, 0, 1, 0); 2801 rdma_ah_set_static_rate(&attr.ah_attr, 3); 2802 rdma_ah_set_port_num(&attr.ah_attr, 1); 2803 2804 for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) { 2805 ret = free_mr_modify_rsv_qp(hr_dev, &attr, i); 2806 if (ret) 2807 return ret; 2808 } 2809 2810 return 0; 2811 } 2812 2813 static int free_mr_init(struct hns_roce_dev *hr_dev) 2814 { 2815 struct hns_roce_v2_priv *priv = hr_dev->priv; 2816 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr; 2817 int ret; 2818 2819 mutex_init(&free_mr->mutex); 2820 2821 ret = free_mr_alloc_res(hr_dev); 2822 if (ret) { 2823 mutex_destroy(&free_mr->mutex); 2824 return ret; 2825 } 2826 2827 ret = free_mr_modify_qp(hr_dev); 2828 if (ret) 2829 goto err_modify_qp; 2830 2831 return 0; 2832 2833 err_modify_qp: 2834 free_mr_exit(hr_dev); 2835 2836 return ret; 2837 } 2838 2839 static int get_hem_table(struct hns_roce_dev *hr_dev) 2840 { 2841 unsigned int qpc_count; 2842 unsigned int cqc_count; 2843 unsigned int gmv_count; 2844 int ret; 2845 int i; 2846 2847 /* Alloc memory for source address table buffer space chunk */ 2848 for (gmv_count = 0; gmv_count < hr_dev->caps.gmv_entry_num; 2849 gmv_count++) { 2850 ret = hns_roce_table_get(hr_dev, &hr_dev->gmv_table, gmv_count); 2851 if (ret) 2852 goto err_gmv_failed; 2853 } 2854 2855 if (hr_dev->is_vf) 2856 return 0; 2857 2858 /* Alloc memory for QPC Timer buffer space chunk */ 2859 for (qpc_count = 0; qpc_count < hr_dev->caps.qpc_timer_bt_num; 2860 qpc_count++) { 2861 ret = hns_roce_table_get(hr_dev, &hr_dev->qpc_timer_table, 2862 qpc_count); 2863 if (ret) { 2864 dev_err(hr_dev->dev, "QPC Timer get failed\n"); 2865 goto err_qpc_timer_failed; 2866 } 2867 } 2868 2869 /* Alloc memory for CQC Timer buffer space chunk */ 2870 for (cqc_count = 0; cqc_count < hr_dev->caps.cqc_timer_bt_num; 2871 cqc_count++) { 2872 ret = hns_roce_table_get(hr_dev, &hr_dev->cqc_timer_table, 2873 cqc_count); 2874 if (ret) { 2875 dev_err(hr_dev->dev, "CQC Timer get failed\n"); 2876 goto err_cqc_timer_failed; 2877 } 2878 } 2879 2880 return 0; 2881 2882 err_cqc_timer_failed: 2883 for (i = 0; i < cqc_count; i++) 2884 hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i); 2885 2886 err_qpc_timer_failed: 2887 for (i = 0; i < qpc_count; i++) 2888 hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i); 2889 2890 err_gmv_failed: 2891 for (i = 0; i < gmv_count; i++) 2892 hns_roce_table_put(hr_dev, &hr_dev->gmv_table, i); 2893 2894 return ret; 2895 } 2896 2897 static void put_hem_table(struct hns_roce_dev *hr_dev) 2898 { 2899 int i; 2900 2901 for (i = 0; i < hr_dev->caps.gmv_entry_num; i++) 2902 hns_roce_table_put(hr_dev, &hr_dev->gmv_table, i); 2903 2904 if (hr_dev->is_vf) 2905 return; 2906 2907 for (i = 0; i < hr_dev->caps.qpc_timer_bt_num; i++) 2908 hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i); 2909 2910 for (i = 0; i < hr_dev->caps.cqc_timer_bt_num; i++) 2911 hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i); 2912 } 2913 2914 static int hns_roce_v2_init(struct hns_roce_dev *hr_dev) 2915 { 2916 int ret; 2917 2918 /* The hns ROCEE requires the extdb info to be cleared before using */ 2919 ret = hns_roce_clear_extdb_list_info(hr_dev); 2920 if (ret) 2921 return ret; 2922 2923 ret = get_hem_table(hr_dev); 2924 if (ret) 2925 return ret; 2926 2927 if (hr_dev->is_vf) 2928 return 0; 2929 2930 ret = hns_roce_init_link_table(hr_dev); 2931 if (ret) { 2932 dev_err(hr_dev->dev, "failed to init llm, ret = %d.\n", ret); 2933 goto err_llm_init_failed; 2934 } 2935 2936 return 0; 2937 2938 err_llm_init_failed: 2939 put_hem_table(hr_dev); 2940 2941 return ret; 2942 } 2943 2944 static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev) 2945 { 2946 hns_roce_function_clear(hr_dev); 2947 2948 if (!hr_dev->is_vf) 2949 hns_roce_free_link_table(hr_dev); 2950 2951 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP09) 2952 free_dip_list(hr_dev); 2953 } 2954 2955 static int hns_roce_mbox_post(struct hns_roce_dev *hr_dev, 2956 struct hns_roce_mbox_msg *mbox_msg) 2957 { 2958 struct hns_roce_cmq_desc desc; 2959 struct hns_roce_post_mbox *mb = (struct hns_roce_post_mbox *)desc.data; 2960 2961 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_POST_MB, false); 2962 2963 mb->in_param_l = cpu_to_le32(mbox_msg->in_param); 2964 mb->in_param_h = cpu_to_le32(mbox_msg->in_param >> 32); 2965 mb->out_param_l = cpu_to_le32(mbox_msg->out_param); 2966 mb->out_param_h = cpu_to_le32(mbox_msg->out_param >> 32); 2967 mb->cmd_tag = cpu_to_le32(mbox_msg->tag << 8 | mbox_msg->cmd); 2968 mb->token_event_en = cpu_to_le32(mbox_msg->event_en << 16 | 2969 mbox_msg->token); 2970 2971 return hns_roce_cmq_send(hr_dev, &desc, 1); 2972 } 2973 2974 static int v2_wait_mbox_complete(struct hns_roce_dev *hr_dev, u32 timeout, 2975 u8 *complete_status) 2976 { 2977 struct hns_roce_mbox_status *mb_st; 2978 struct hns_roce_cmq_desc desc; 2979 unsigned long end; 2980 int ret = -EBUSY; 2981 u32 status; 2982 bool busy; 2983 2984 mb_st = (struct hns_roce_mbox_status *)desc.data; 2985 end = msecs_to_jiffies(timeout) + jiffies; 2986 while (v2_chk_mbox_is_avail(hr_dev, &busy)) { 2987 if (hr_dev->cmd.state == HNS_ROCE_CMDQ_STATE_FATAL_ERR) 2988 return -EIO; 2989 2990 status = 0; 2991 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_MB_ST, 2992 true); 2993 ret = __hns_roce_cmq_send(hr_dev, &desc, 1); 2994 if (!ret) { 2995 status = le32_to_cpu(mb_st->mb_status_hw_run); 2996 /* No pending message exists in ROCEE mbox. */ 2997 if (!(status & MB_ST_HW_RUN_M)) 2998 break; 2999 } else if (!v2_chk_mbox_is_avail(hr_dev, &busy)) { 3000 break; 3001 } 3002 3003 if (time_after(jiffies, end)) { 3004 dev_err_ratelimited(hr_dev->dev, 3005 "failed to wait mbox status 0x%x\n", 3006 status); 3007 return -ETIMEDOUT; 3008 } 3009 3010 cond_resched(); 3011 ret = -EBUSY; 3012 } 3013 3014 if (!ret) { 3015 *complete_status = (u8)(status & MB_ST_COMPLETE_M); 3016 } else if (!v2_chk_mbox_is_avail(hr_dev, &busy)) { 3017 /* Ignore all errors if the mbox is unavailable. */ 3018 ret = 0; 3019 *complete_status = MB_ST_COMPLETE_M; 3020 } 3021 3022 return ret; 3023 } 3024 3025 static int v2_post_mbox(struct hns_roce_dev *hr_dev, 3026 struct hns_roce_mbox_msg *mbox_msg) 3027 { 3028 u8 status = 0; 3029 int ret; 3030 3031 /* Waiting for the mbox to be idle */ 3032 ret = v2_wait_mbox_complete(hr_dev, HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS, 3033 &status); 3034 if (unlikely(ret)) { 3035 dev_err_ratelimited(hr_dev->dev, 3036 "failed to check post mbox status = 0x%x, ret = %d.\n", 3037 status, ret); 3038 return ret; 3039 } 3040 3041 /* Post new message to mbox */ 3042 ret = hns_roce_mbox_post(hr_dev, mbox_msg); 3043 if (ret) 3044 dev_err_ratelimited(hr_dev->dev, 3045 "failed to post mailbox, ret = %d.\n", ret); 3046 3047 return ret; 3048 } 3049 3050 static int v2_poll_mbox_done(struct hns_roce_dev *hr_dev) 3051 { 3052 u8 status = 0; 3053 int ret; 3054 3055 ret = v2_wait_mbox_complete(hr_dev, HNS_ROCE_CMD_TIMEOUT_MSECS, 3056 &status); 3057 if (!ret) { 3058 if (status != MB_ST_COMPLETE_SUCC) 3059 return -EBUSY; 3060 } else { 3061 dev_err_ratelimited(hr_dev->dev, 3062 "failed to check mbox status = 0x%x, ret = %d.\n", 3063 status, ret); 3064 } 3065 3066 return ret; 3067 } 3068 3069 static void copy_gid(void *dest, const union ib_gid *gid) 3070 { 3071 #define GID_SIZE 4 3072 const union ib_gid *src = gid; 3073 __le32 (*p)[GID_SIZE] = dest; 3074 int i; 3075 3076 if (!gid) 3077 src = &zgid; 3078 3079 for (i = 0; i < GID_SIZE; i++) 3080 (*p)[i] = cpu_to_le32(*(u32 *)&src->raw[i * sizeof(u32)]); 3081 } 3082 3083 static int config_sgid_table(struct hns_roce_dev *hr_dev, 3084 int gid_index, const union ib_gid *gid, 3085 enum hns_roce_sgid_type sgid_type) 3086 { 3087 struct hns_roce_cmq_desc desc; 3088 struct hns_roce_cfg_sgid_tb *sgid_tb = 3089 (struct hns_roce_cfg_sgid_tb *)desc.data; 3090 3091 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SGID_TB, false); 3092 3093 hr_reg_write(sgid_tb, CFG_SGID_TB_TABLE_IDX, gid_index); 3094 hr_reg_write(sgid_tb, CFG_SGID_TB_VF_SGID_TYPE, sgid_type); 3095 3096 copy_gid(&sgid_tb->vf_sgid_l, gid); 3097 3098 return hns_roce_cmq_send(hr_dev, &desc, 1); 3099 } 3100 3101 static int config_gmv_table(struct hns_roce_dev *hr_dev, 3102 int gid_index, const union ib_gid *gid, 3103 enum hns_roce_sgid_type sgid_type, 3104 const struct ib_gid_attr *attr) 3105 { 3106 struct hns_roce_cmq_desc desc[2]; 3107 struct hns_roce_cfg_gmv_tb_a *tb_a = 3108 (struct hns_roce_cfg_gmv_tb_a *)desc[0].data; 3109 struct hns_roce_cfg_gmv_tb_b *tb_b = 3110 (struct hns_roce_cfg_gmv_tb_b *)desc[1].data; 3111 3112 u16 vlan_id = VLAN_CFI_MASK; 3113 u8 mac[ETH_ALEN] = {}; 3114 int ret; 3115 3116 if (gid) { 3117 ret = rdma_read_gid_l2_fields(attr, &vlan_id, mac); 3118 if (ret) 3119 return ret; 3120 } 3121 3122 hns_roce_cmq_setup_basic_desc(&desc[0], HNS_ROCE_OPC_CFG_GMV_TBL, false); 3123 desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 3124 3125 hns_roce_cmq_setup_basic_desc(&desc[1], HNS_ROCE_OPC_CFG_GMV_TBL, false); 3126 3127 copy_gid(&tb_a->vf_sgid_l, gid); 3128 3129 hr_reg_write(tb_a, GMV_TB_A_VF_SGID_TYPE, sgid_type); 3130 hr_reg_write(tb_a, GMV_TB_A_VF_VLAN_EN, vlan_id < VLAN_CFI_MASK); 3131 hr_reg_write(tb_a, GMV_TB_A_VF_VLAN_ID, vlan_id); 3132 3133 tb_b->vf_smac_l = cpu_to_le32(*(u32 *)mac); 3134 3135 hr_reg_write(tb_b, GMV_TB_B_SMAC_H, *(u16 *)&mac[4]); 3136 hr_reg_write(tb_b, GMV_TB_B_SGID_IDX, gid_index); 3137 3138 return hns_roce_cmq_send(hr_dev, desc, 2); 3139 } 3140 3141 static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, int gid_index, 3142 const union ib_gid *gid, 3143 const struct ib_gid_attr *attr) 3144 { 3145 enum hns_roce_sgid_type sgid_type = GID_TYPE_FLAG_ROCE_V1; 3146 int ret; 3147 3148 if (gid) { 3149 if (attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) { 3150 if (ipv6_addr_v4mapped((void *)gid)) 3151 sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV4; 3152 else 3153 sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV6; 3154 } else if (attr->gid_type == IB_GID_TYPE_ROCE) { 3155 sgid_type = GID_TYPE_FLAG_ROCE_V1; 3156 } 3157 } 3158 3159 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) 3160 ret = config_gmv_table(hr_dev, gid_index, gid, sgid_type, attr); 3161 else 3162 ret = config_sgid_table(hr_dev, gid_index, gid, sgid_type); 3163 3164 if (ret) 3165 ibdev_err(&hr_dev->ib_dev, "failed to set gid, ret = %d!\n", 3166 ret); 3167 3168 return ret; 3169 } 3170 3171 static int hns_roce_v2_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port, 3172 const u8 *addr) 3173 { 3174 struct hns_roce_cmq_desc desc; 3175 struct hns_roce_cfg_smac_tb *smac_tb = 3176 (struct hns_roce_cfg_smac_tb *)desc.data; 3177 u16 reg_smac_h; 3178 u32 reg_smac_l; 3179 3180 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SMAC_TB, false); 3181 3182 reg_smac_l = *(u32 *)(&addr[0]); 3183 reg_smac_h = *(u16 *)(&addr[4]); 3184 3185 hr_reg_write(smac_tb, CFG_SMAC_TB_IDX, phy_port); 3186 hr_reg_write(smac_tb, CFG_SMAC_TB_VF_SMAC_H, reg_smac_h); 3187 smac_tb->vf_smac_l = cpu_to_le32(reg_smac_l); 3188 3189 return hns_roce_cmq_send(hr_dev, &desc, 1); 3190 } 3191 3192 static int set_mtpt_pbl(struct hns_roce_dev *hr_dev, 3193 struct hns_roce_v2_mpt_entry *mpt_entry, 3194 struct hns_roce_mr *mr) 3195 { 3196 u64 pages[HNS_ROCE_V2_MAX_INNER_MTPT_NUM] = { 0 }; 3197 struct ib_device *ibdev = &hr_dev->ib_dev; 3198 dma_addr_t pbl_ba; 3199 int ret; 3200 int i; 3201 3202 ret = hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, pages, 3203 min_t(int, ARRAY_SIZE(pages), mr->npages)); 3204 if (ret) { 3205 ibdev_err(ibdev, "failed to find PBL mtr, ret = %d.\n", ret); 3206 return ret; 3207 } 3208 3209 /* Aligned to the hardware address access unit */ 3210 for (i = 0; i < ARRAY_SIZE(pages); i++) 3211 pages[i] >>= MPT_PBL_BUF_ADDR_S; 3212 3213 pbl_ba = hns_roce_get_mtr_ba(&mr->pbl_mtr); 3214 3215 mpt_entry->pbl_size = cpu_to_le32(mr->npages); 3216 mpt_entry->pbl_ba_l = cpu_to_le32(pbl_ba >> MPT_PBL_BA_ADDR_S); 3217 hr_reg_write(mpt_entry, MPT_PBL_BA_H, 3218 upper_32_bits(pbl_ba >> MPT_PBL_BA_ADDR_S)); 3219 3220 mpt_entry->pa0_l = cpu_to_le32(lower_32_bits(pages[0])); 3221 hr_reg_write(mpt_entry, MPT_PA0_H, upper_32_bits(pages[0])); 3222 3223 mpt_entry->pa1_l = cpu_to_le32(lower_32_bits(pages[1])); 3224 hr_reg_write(mpt_entry, MPT_PA1_H, upper_32_bits(pages[1])); 3225 hr_reg_write(mpt_entry, MPT_PBL_BUF_PG_SZ, 3226 to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift)); 3227 3228 return 0; 3229 } 3230 3231 static int hns_roce_v2_write_mtpt(struct hns_roce_dev *hr_dev, 3232 void *mb_buf, struct hns_roce_mr *mr) 3233 { 3234 struct hns_roce_v2_mpt_entry *mpt_entry; 3235 3236 mpt_entry = mb_buf; 3237 memset(mpt_entry, 0, sizeof(*mpt_entry)); 3238 3239 hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_VALID); 3240 hr_reg_write(mpt_entry, MPT_PD, mr->pd); 3241 3242 hr_reg_write_bool(mpt_entry, MPT_BIND_EN, 3243 mr->access & IB_ACCESS_MW_BIND); 3244 hr_reg_write_bool(mpt_entry, MPT_ATOMIC_EN, 3245 mr->access & IB_ACCESS_REMOTE_ATOMIC); 3246 hr_reg_write_bool(mpt_entry, MPT_RR_EN, 3247 mr->access & IB_ACCESS_REMOTE_READ); 3248 hr_reg_write_bool(mpt_entry, MPT_RW_EN, 3249 mr->access & IB_ACCESS_REMOTE_WRITE); 3250 hr_reg_write_bool(mpt_entry, MPT_LW_EN, 3251 mr->access & IB_ACCESS_LOCAL_WRITE); 3252 3253 mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size)); 3254 mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size)); 3255 mpt_entry->lkey = cpu_to_le32(mr->key); 3256 mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova)); 3257 mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova)); 3258 3259 if (mr->type != MR_TYPE_MR) 3260 hr_reg_enable(mpt_entry, MPT_PA); 3261 3262 if (mr->type == MR_TYPE_DMA) 3263 return 0; 3264 3265 if (mr->pbl_hop_num != HNS_ROCE_HOP_NUM_0) 3266 hr_reg_write(mpt_entry, MPT_PBL_HOP_NUM, mr->pbl_hop_num); 3267 3268 hr_reg_write(mpt_entry, MPT_PBL_BA_PG_SZ, 3269 to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.ba_pg_shift)); 3270 hr_reg_enable(mpt_entry, MPT_INNER_PA_VLD); 3271 3272 return set_mtpt_pbl(hr_dev, mpt_entry, mr); 3273 } 3274 3275 static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev, 3276 struct hns_roce_mr *mr, int flags, 3277 void *mb_buf) 3278 { 3279 struct hns_roce_v2_mpt_entry *mpt_entry = mb_buf; 3280 u32 mr_access_flags = mr->access; 3281 int ret = 0; 3282 3283 hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_VALID); 3284 hr_reg_write(mpt_entry, MPT_PD, mr->pd); 3285 3286 if (flags & IB_MR_REREG_ACCESS) { 3287 hr_reg_write(mpt_entry, MPT_BIND_EN, 3288 (mr_access_flags & IB_ACCESS_MW_BIND ? 1 : 0)); 3289 hr_reg_write(mpt_entry, MPT_ATOMIC_EN, 3290 mr_access_flags & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0); 3291 hr_reg_write(mpt_entry, MPT_RR_EN, 3292 mr_access_flags & IB_ACCESS_REMOTE_READ ? 1 : 0); 3293 hr_reg_write(mpt_entry, MPT_RW_EN, 3294 mr_access_flags & IB_ACCESS_REMOTE_WRITE ? 1 : 0); 3295 hr_reg_write(mpt_entry, MPT_LW_EN, 3296 mr_access_flags & IB_ACCESS_LOCAL_WRITE ? 1 : 0); 3297 } 3298 3299 if (flags & IB_MR_REREG_TRANS) { 3300 mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova)); 3301 mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova)); 3302 mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size)); 3303 mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size)); 3304 3305 ret = set_mtpt_pbl(hr_dev, mpt_entry, mr); 3306 } 3307 3308 return ret; 3309 } 3310 3311 static int hns_roce_v2_frmr_write_mtpt(void *mb_buf, struct hns_roce_mr *mr) 3312 { 3313 dma_addr_t pbl_ba = hns_roce_get_mtr_ba(&mr->pbl_mtr); 3314 struct hns_roce_v2_mpt_entry *mpt_entry; 3315 3316 mpt_entry = mb_buf; 3317 memset(mpt_entry, 0, sizeof(*mpt_entry)); 3318 3319 hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_FREE); 3320 hr_reg_write(mpt_entry, MPT_PD, mr->pd); 3321 3322 hr_reg_enable(mpt_entry, MPT_RA_EN); 3323 hr_reg_enable(mpt_entry, MPT_R_INV_EN); 3324 3325 hr_reg_enable(mpt_entry, MPT_FRE); 3326 hr_reg_clear(mpt_entry, MPT_MR_MW); 3327 hr_reg_enable(mpt_entry, MPT_BPD); 3328 hr_reg_clear(mpt_entry, MPT_PA); 3329 3330 hr_reg_write(mpt_entry, MPT_PBL_HOP_NUM, 1); 3331 hr_reg_write(mpt_entry, MPT_PBL_BA_PG_SZ, 3332 to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.ba_pg_shift)); 3333 hr_reg_write(mpt_entry, MPT_PBL_BUF_PG_SZ, 3334 to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift)); 3335 3336 mpt_entry->pbl_size = cpu_to_le32(mr->npages); 3337 3338 mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(pbl_ba >> 3339 MPT_PBL_BA_ADDR_S)); 3340 hr_reg_write(mpt_entry, MPT_PBL_BA_H, 3341 upper_32_bits(pbl_ba >> MPT_PBL_BA_ADDR_S)); 3342 3343 return 0; 3344 } 3345 3346 static int hns_roce_v2_mw_write_mtpt(void *mb_buf, struct hns_roce_mw *mw) 3347 { 3348 struct hns_roce_v2_mpt_entry *mpt_entry; 3349 3350 mpt_entry = mb_buf; 3351 memset(mpt_entry, 0, sizeof(*mpt_entry)); 3352 3353 hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_FREE); 3354 hr_reg_write(mpt_entry, MPT_PD, mw->pdn); 3355 3356 hr_reg_enable(mpt_entry, MPT_R_INV_EN); 3357 hr_reg_enable(mpt_entry, MPT_LW_EN); 3358 3359 hr_reg_enable(mpt_entry, MPT_MR_MW); 3360 hr_reg_enable(mpt_entry, MPT_BPD); 3361 hr_reg_clear(mpt_entry, MPT_PA); 3362 hr_reg_write(mpt_entry, MPT_BQP, 3363 mw->ibmw.type == IB_MW_TYPE_1 ? 0 : 1); 3364 3365 mpt_entry->lkey = cpu_to_le32(mw->rkey); 3366 3367 hr_reg_write(mpt_entry, MPT_PBL_HOP_NUM, 3368 mw->pbl_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : 3369 mw->pbl_hop_num); 3370 hr_reg_write(mpt_entry, MPT_PBL_BA_PG_SZ, 3371 mw->pbl_ba_pg_sz + PG_SHIFT_OFFSET); 3372 hr_reg_write(mpt_entry, MPT_PBL_BUF_PG_SZ, 3373 mw->pbl_buf_pg_sz + PG_SHIFT_OFFSET); 3374 3375 return 0; 3376 } 3377 3378 static int free_mr_post_send_lp_wqe(struct hns_roce_qp *hr_qp) 3379 { 3380 struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp->ibqp.device); 3381 struct ib_device *ibdev = &hr_dev->ib_dev; 3382 const struct ib_send_wr *bad_wr; 3383 struct ib_rdma_wr rdma_wr = {}; 3384 struct ib_send_wr *send_wr; 3385 int ret; 3386 3387 send_wr = &rdma_wr.wr; 3388 send_wr->opcode = IB_WR_RDMA_WRITE; 3389 3390 ret = hns_roce_v2_post_send(&hr_qp->ibqp, send_wr, &bad_wr); 3391 if (ret) { 3392 ibdev_err(ibdev, "failed to post wqe for free mr, ret = %d.\n", 3393 ret); 3394 return ret; 3395 } 3396 3397 return 0; 3398 } 3399 3400 static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries, 3401 struct ib_wc *wc); 3402 3403 static void free_mr_send_cmd_to_hw(struct hns_roce_dev *hr_dev) 3404 { 3405 struct hns_roce_v2_priv *priv = hr_dev->priv; 3406 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr; 3407 struct ib_wc wc[ARRAY_SIZE(free_mr->rsv_qp)]; 3408 struct ib_device *ibdev = &hr_dev->ib_dev; 3409 struct hns_roce_qp *hr_qp; 3410 unsigned long end; 3411 int cqe_cnt = 0; 3412 int npolled; 3413 int ret; 3414 int i; 3415 3416 /* 3417 * If the device initialization is not complete or in the uninstall 3418 * process, then there is no need to execute free mr. 3419 */ 3420 if (priv->handle->rinfo.reset_state == HNS_ROCE_STATE_RST_INIT || 3421 priv->handle->rinfo.instance_state == HNS_ROCE_STATE_INIT || 3422 hr_dev->state == HNS_ROCE_DEVICE_STATE_UNINIT) 3423 return; 3424 3425 mutex_lock(&free_mr->mutex); 3426 3427 for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) { 3428 hr_qp = free_mr->rsv_qp[i]; 3429 3430 ret = free_mr_post_send_lp_wqe(hr_qp); 3431 if (ret) { 3432 ibdev_err(ibdev, 3433 "failed to send wqe (qp:0x%lx) for free mr, ret = %d.\n", 3434 hr_qp->qpn, ret); 3435 break; 3436 } 3437 3438 cqe_cnt++; 3439 } 3440 3441 end = msecs_to_jiffies(HNS_ROCE_V2_FREE_MR_TIMEOUT) + jiffies; 3442 while (cqe_cnt) { 3443 npolled = hns_roce_v2_poll_cq(&free_mr->rsv_cq->ib_cq, cqe_cnt, wc); 3444 if (npolled < 0) { 3445 ibdev_err(ibdev, 3446 "failed to poll cqe for free mr, remain %d cqe.\n", 3447 cqe_cnt); 3448 goto out; 3449 } 3450 3451 if (time_after(jiffies, end)) { 3452 ibdev_err(ibdev, 3453 "failed to poll cqe for free mr and timeout, remain %d cqe.\n", 3454 cqe_cnt); 3455 goto out; 3456 } 3457 cqe_cnt -= npolled; 3458 } 3459 3460 out: 3461 mutex_unlock(&free_mr->mutex); 3462 } 3463 3464 static void hns_roce_v2_dereg_mr(struct hns_roce_dev *hr_dev) 3465 { 3466 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) 3467 free_mr_send_cmd_to_hw(hr_dev); 3468 } 3469 3470 static void *get_cqe_v2(struct hns_roce_cq *hr_cq, int n) 3471 { 3472 return hns_roce_buf_offset(hr_cq->mtr.kmem, n * hr_cq->cqe_size); 3473 } 3474 3475 static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, unsigned int n) 3476 { 3477 struct hns_roce_v2_cqe *cqe = get_cqe_v2(hr_cq, n & hr_cq->ib_cq.cqe); 3478 3479 /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */ 3480 return (hr_reg_read(cqe, CQE_OWNER) ^ !!(n & hr_cq->cq_depth)) ? cqe : 3481 NULL; 3482 } 3483 3484 static inline void update_cq_db(struct hns_roce_dev *hr_dev, 3485 struct hns_roce_cq *hr_cq) 3486 { 3487 if (likely(hr_cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB)) { 3488 *hr_cq->set_ci_db = hr_cq->cons_index & V2_CQ_DB_CONS_IDX_M; 3489 } else { 3490 struct hns_roce_v2_db cq_db = {}; 3491 3492 hr_reg_write(&cq_db, DB_TAG, hr_cq->cqn); 3493 hr_reg_write(&cq_db, DB_CMD, HNS_ROCE_V2_CQ_DB); 3494 hr_reg_write(&cq_db, DB_CQ_CI, hr_cq->cons_index); 3495 hr_reg_write(&cq_db, DB_CQ_CMD_SN, 1); 3496 3497 hns_roce_write64(hr_dev, (__le32 *)&cq_db, hr_cq->db_reg); 3498 } 3499 } 3500 3501 static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn, 3502 struct hns_roce_srq *srq) 3503 { 3504 struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device); 3505 struct hns_roce_v2_cqe *cqe, *dest; 3506 u32 prod_index; 3507 int nfreed = 0; 3508 int wqe_index; 3509 u8 owner_bit; 3510 3511 for (prod_index = hr_cq->cons_index; get_sw_cqe_v2(hr_cq, prod_index); 3512 ++prod_index) { 3513 if (prod_index > hr_cq->cons_index + hr_cq->ib_cq.cqe) 3514 break; 3515 } 3516 3517 /* 3518 * Now backwards through the CQ, removing CQ entries 3519 * that match our QP by overwriting them with next entries. 3520 */ 3521 while ((int) --prod_index - (int) hr_cq->cons_index >= 0) { 3522 cqe = get_cqe_v2(hr_cq, prod_index & hr_cq->ib_cq.cqe); 3523 if (hr_reg_read(cqe, CQE_LCL_QPN) == qpn) { 3524 if (srq && hr_reg_read(cqe, CQE_S_R)) { 3525 wqe_index = hr_reg_read(cqe, CQE_WQE_IDX); 3526 hns_roce_free_srq_wqe(srq, wqe_index); 3527 } 3528 ++nfreed; 3529 } else if (nfreed) { 3530 dest = get_cqe_v2(hr_cq, (prod_index + nfreed) & 3531 hr_cq->ib_cq.cqe); 3532 owner_bit = hr_reg_read(dest, CQE_OWNER); 3533 memcpy(dest, cqe, hr_cq->cqe_size); 3534 hr_reg_write(dest, CQE_OWNER, owner_bit); 3535 } 3536 } 3537 3538 if (nfreed) { 3539 hr_cq->cons_index += nfreed; 3540 update_cq_db(hr_dev, hr_cq); 3541 } 3542 } 3543 3544 static void hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn, 3545 struct hns_roce_srq *srq) 3546 { 3547 spin_lock_irq(&hr_cq->lock); 3548 __hns_roce_v2_cq_clean(hr_cq, qpn, srq); 3549 spin_unlock_irq(&hr_cq->lock); 3550 } 3551 3552 static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev, 3553 struct hns_roce_cq *hr_cq, void *mb_buf, 3554 u64 *mtts, dma_addr_t dma_handle) 3555 { 3556 struct hns_roce_v2_cq_context *cq_context; 3557 3558 cq_context = mb_buf; 3559 memset(cq_context, 0, sizeof(*cq_context)); 3560 3561 hr_reg_write(cq_context, CQC_CQ_ST, V2_CQ_STATE_VALID); 3562 hr_reg_write(cq_context, CQC_ARM_ST, NO_ARMED); 3563 hr_reg_write(cq_context, CQC_SHIFT, ilog2(hr_cq->cq_depth)); 3564 hr_reg_write(cq_context, CQC_CEQN, hr_cq->vector); 3565 hr_reg_write(cq_context, CQC_CQN, hr_cq->cqn); 3566 3567 if (hr_cq->cqe_size == HNS_ROCE_V3_CQE_SIZE) 3568 hr_reg_write(cq_context, CQC_CQE_SIZE, CQE_SIZE_64B); 3569 3570 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_STASH) 3571 hr_reg_enable(cq_context, CQC_STASH); 3572 3573 hr_reg_write(cq_context, CQC_CQE_CUR_BLK_ADDR_L, 3574 to_hr_hw_page_addr(mtts[0])); 3575 hr_reg_write(cq_context, CQC_CQE_CUR_BLK_ADDR_H, 3576 upper_32_bits(to_hr_hw_page_addr(mtts[0]))); 3577 hr_reg_write(cq_context, CQC_CQE_HOP_NUM, hr_dev->caps.cqe_hop_num == 3578 HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev->caps.cqe_hop_num); 3579 hr_reg_write(cq_context, CQC_CQE_NEX_BLK_ADDR_L, 3580 to_hr_hw_page_addr(mtts[1])); 3581 hr_reg_write(cq_context, CQC_CQE_NEX_BLK_ADDR_H, 3582 upper_32_bits(to_hr_hw_page_addr(mtts[1]))); 3583 hr_reg_write(cq_context, CQC_CQE_BAR_PG_SZ, 3584 to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.ba_pg_shift)); 3585 hr_reg_write(cq_context, CQC_CQE_BUF_PG_SZ, 3586 to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.buf_pg_shift)); 3587 hr_reg_write(cq_context, CQC_CQE_BA_L, dma_handle >> CQC_CQE_BA_L_S); 3588 hr_reg_write(cq_context, CQC_CQE_BA_H, dma_handle >> CQC_CQE_BA_H_S); 3589 hr_reg_write_bool(cq_context, CQC_DB_RECORD_EN, 3590 hr_cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB); 3591 hr_reg_write(cq_context, CQC_CQE_DB_RECORD_ADDR_L, 3592 ((u32)hr_cq->db.dma) >> 1); 3593 hr_reg_write(cq_context, CQC_CQE_DB_RECORD_ADDR_H, 3594 hr_cq->db.dma >> CQC_CQE_DB_RECORD_ADDR_H_S); 3595 hr_reg_write(cq_context, CQC_CQ_MAX_CNT, 3596 HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM); 3597 hr_reg_write(cq_context, CQC_CQ_PERIOD, 3598 HNS_ROCE_V2_CQ_DEFAULT_INTERVAL); 3599 } 3600 3601 static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq, 3602 enum ib_cq_notify_flags flags) 3603 { 3604 struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device); 3605 struct hns_roce_cq *hr_cq = to_hr_cq(ibcq); 3606 struct hns_roce_v2_db cq_db = {}; 3607 u32 notify_flag; 3608 3609 /* 3610 * flags = 0, then notify_flag : next 3611 * flags = 1, then notify flag : solocited 3612 */ 3613 notify_flag = (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ? 3614 V2_CQ_DB_REQ_NOT : V2_CQ_DB_REQ_NOT_SOL; 3615 3616 hr_reg_write(&cq_db, DB_TAG, hr_cq->cqn); 3617 hr_reg_write(&cq_db, DB_CMD, HNS_ROCE_V2_CQ_DB_NOTIFY); 3618 hr_reg_write(&cq_db, DB_CQ_CI, hr_cq->cons_index); 3619 hr_reg_write(&cq_db, DB_CQ_CMD_SN, hr_cq->arm_sn); 3620 hr_reg_write(&cq_db, DB_CQ_NOTIFY, notify_flag); 3621 3622 hns_roce_write64(hr_dev, (__le32 *)&cq_db, hr_cq->db_reg); 3623 3624 return 0; 3625 } 3626 3627 static int sw_comp(struct hns_roce_qp *hr_qp, struct hns_roce_wq *wq, 3628 int num_entries, struct ib_wc *wc) 3629 { 3630 unsigned int left; 3631 int npolled = 0; 3632 3633 left = wq->head - wq->tail; 3634 if (left == 0) 3635 return 0; 3636 3637 left = min_t(unsigned int, (unsigned int)num_entries, left); 3638 while (npolled < left) { 3639 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; 3640 wc->status = IB_WC_WR_FLUSH_ERR; 3641 wc->vendor_err = 0; 3642 wc->qp = &hr_qp->ibqp; 3643 3644 wq->tail++; 3645 wc++; 3646 npolled++; 3647 } 3648 3649 return npolled; 3650 } 3651 3652 static int hns_roce_v2_sw_poll_cq(struct hns_roce_cq *hr_cq, int num_entries, 3653 struct ib_wc *wc) 3654 { 3655 struct hns_roce_qp *hr_qp; 3656 int npolled = 0; 3657 3658 list_for_each_entry(hr_qp, &hr_cq->sq_list, sq_node) { 3659 npolled += sw_comp(hr_qp, &hr_qp->sq, 3660 num_entries - npolled, wc + npolled); 3661 if (npolled >= num_entries) 3662 goto out; 3663 } 3664 3665 list_for_each_entry(hr_qp, &hr_cq->rq_list, rq_node) { 3666 npolled += sw_comp(hr_qp, &hr_qp->rq, 3667 num_entries - npolled, wc + npolled); 3668 if (npolled >= num_entries) 3669 goto out; 3670 } 3671 3672 out: 3673 return npolled; 3674 } 3675 3676 static void get_cqe_status(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp, 3677 struct hns_roce_cq *cq, struct hns_roce_v2_cqe *cqe, 3678 struct ib_wc *wc) 3679 { 3680 static const struct { 3681 u32 cqe_status; 3682 enum ib_wc_status wc_status; 3683 } map[] = { 3684 { HNS_ROCE_CQE_V2_SUCCESS, IB_WC_SUCCESS }, 3685 { HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR, IB_WC_LOC_LEN_ERR }, 3686 { HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR, IB_WC_LOC_QP_OP_ERR }, 3687 { HNS_ROCE_CQE_V2_LOCAL_PROT_ERR, IB_WC_LOC_PROT_ERR }, 3688 { HNS_ROCE_CQE_V2_WR_FLUSH_ERR, IB_WC_WR_FLUSH_ERR }, 3689 { HNS_ROCE_CQE_V2_MW_BIND_ERR, IB_WC_MW_BIND_ERR }, 3690 { HNS_ROCE_CQE_V2_BAD_RESP_ERR, IB_WC_BAD_RESP_ERR }, 3691 { HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR, IB_WC_LOC_ACCESS_ERR }, 3692 { HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR, IB_WC_REM_INV_REQ_ERR }, 3693 { HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR, IB_WC_REM_ACCESS_ERR }, 3694 { HNS_ROCE_CQE_V2_REMOTE_OP_ERR, IB_WC_REM_OP_ERR }, 3695 { HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR, 3696 IB_WC_RETRY_EXC_ERR }, 3697 { HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR, IB_WC_RNR_RETRY_EXC_ERR }, 3698 { HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR, IB_WC_REM_ABORT_ERR }, 3699 { HNS_ROCE_CQE_V2_GENERAL_ERR, IB_WC_GENERAL_ERR} 3700 }; 3701 3702 u32 cqe_status = hr_reg_read(cqe, CQE_STATUS); 3703 int i; 3704 3705 wc->status = IB_WC_GENERAL_ERR; 3706 for (i = 0; i < ARRAY_SIZE(map); i++) 3707 if (cqe_status == map[i].cqe_status) { 3708 wc->status = map[i].wc_status; 3709 break; 3710 } 3711 3712 if (likely(wc->status == IB_WC_SUCCESS || 3713 wc->status == IB_WC_WR_FLUSH_ERR)) 3714 return; 3715 3716 ibdev_err_ratelimited(&hr_dev->ib_dev, "error cqe status 0x%x:\n", 3717 cqe_status); 3718 print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_NONE, 16, 4, cqe, 3719 cq->cqe_size, false); 3720 wc->vendor_err = hr_reg_read(cqe, CQE_SUB_STATUS); 3721 3722 /* 3723 * For hns ROCEE, GENERAL_ERR is an error type that is not defined in 3724 * the standard protocol, the driver must ignore it and needn't to set 3725 * the QP to an error state. 3726 */ 3727 if (cqe_status == HNS_ROCE_CQE_V2_GENERAL_ERR) 3728 return; 3729 3730 flush_cqe(hr_dev, qp); 3731 } 3732 3733 static int get_cur_qp(struct hns_roce_cq *hr_cq, struct hns_roce_v2_cqe *cqe, 3734 struct hns_roce_qp **cur_qp) 3735 { 3736 struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device); 3737 struct hns_roce_qp *hr_qp = *cur_qp; 3738 u32 qpn; 3739 3740 qpn = hr_reg_read(cqe, CQE_LCL_QPN); 3741 3742 if (!hr_qp || qpn != hr_qp->qpn) { 3743 hr_qp = __hns_roce_qp_lookup(hr_dev, qpn); 3744 if (unlikely(!hr_qp)) { 3745 ibdev_err(&hr_dev->ib_dev, 3746 "CQ %06lx with entry for unknown QPN %06x\n", 3747 hr_cq->cqn, qpn); 3748 return -EINVAL; 3749 } 3750 *cur_qp = hr_qp; 3751 } 3752 3753 return 0; 3754 } 3755 3756 /* 3757 * mapped-value = 1 + real-value 3758 * The ib wc opcode's real value is start from 0, In order to distinguish 3759 * between initialized and uninitialized map values, we plus 1 to the actual 3760 * value when defining the mapping, so that the validity can be identified by 3761 * checking whether the mapped value is greater than 0. 3762 */ 3763 #define HR_WC_OP_MAP(hr_key, ib_key) \ 3764 [HNS_ROCE_V2_WQE_OP_ ## hr_key] = 1 + IB_WC_ ## ib_key 3765 3766 static const u32 wc_send_op_map[] = { 3767 HR_WC_OP_MAP(SEND, SEND), 3768 HR_WC_OP_MAP(SEND_WITH_INV, SEND), 3769 HR_WC_OP_MAP(SEND_WITH_IMM, SEND), 3770 HR_WC_OP_MAP(RDMA_READ, RDMA_READ), 3771 HR_WC_OP_MAP(RDMA_WRITE, RDMA_WRITE), 3772 HR_WC_OP_MAP(RDMA_WRITE_WITH_IMM, RDMA_WRITE), 3773 HR_WC_OP_MAP(ATOM_CMP_AND_SWAP, COMP_SWAP), 3774 HR_WC_OP_MAP(ATOM_FETCH_AND_ADD, FETCH_ADD), 3775 HR_WC_OP_MAP(ATOM_MSK_CMP_AND_SWAP, MASKED_COMP_SWAP), 3776 HR_WC_OP_MAP(ATOM_MSK_FETCH_AND_ADD, MASKED_FETCH_ADD), 3777 HR_WC_OP_MAP(FAST_REG_PMR, REG_MR), 3778 HR_WC_OP_MAP(BIND_MW, REG_MR), 3779 }; 3780 3781 static int to_ib_wc_send_op(u32 hr_opcode) 3782 { 3783 if (hr_opcode >= ARRAY_SIZE(wc_send_op_map)) 3784 return -EINVAL; 3785 3786 return wc_send_op_map[hr_opcode] ? wc_send_op_map[hr_opcode] - 1 : 3787 -EINVAL; 3788 } 3789 3790 static const u32 wc_recv_op_map[] = { 3791 HR_WC_OP_MAP(RDMA_WRITE_WITH_IMM, WITH_IMM), 3792 HR_WC_OP_MAP(SEND, RECV), 3793 HR_WC_OP_MAP(SEND_WITH_IMM, WITH_IMM), 3794 HR_WC_OP_MAP(SEND_WITH_INV, RECV), 3795 }; 3796 3797 static int to_ib_wc_recv_op(u32 hr_opcode) 3798 { 3799 if (hr_opcode >= ARRAY_SIZE(wc_recv_op_map)) 3800 return -EINVAL; 3801 3802 return wc_recv_op_map[hr_opcode] ? wc_recv_op_map[hr_opcode] - 1 : 3803 -EINVAL; 3804 } 3805 3806 static void fill_send_wc(struct ib_wc *wc, struct hns_roce_v2_cqe *cqe) 3807 { 3808 u32 hr_opcode; 3809 int ib_opcode; 3810 3811 wc->wc_flags = 0; 3812 3813 hr_opcode = hr_reg_read(cqe, CQE_OPCODE); 3814 switch (hr_opcode) { 3815 case HNS_ROCE_V2_WQE_OP_RDMA_READ: 3816 wc->byte_len = le32_to_cpu(cqe->byte_cnt); 3817 break; 3818 case HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM: 3819 case HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM: 3820 wc->wc_flags |= IB_WC_WITH_IMM; 3821 break; 3822 case HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP: 3823 case HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD: 3824 case HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP: 3825 case HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD: 3826 wc->byte_len = 8; 3827 break; 3828 default: 3829 break; 3830 } 3831 3832 ib_opcode = to_ib_wc_send_op(hr_opcode); 3833 if (ib_opcode < 0) 3834 wc->status = IB_WC_GENERAL_ERR; 3835 else 3836 wc->opcode = ib_opcode; 3837 } 3838 3839 static int fill_recv_wc(struct ib_wc *wc, struct hns_roce_v2_cqe *cqe) 3840 { 3841 u32 hr_opcode; 3842 int ib_opcode; 3843 3844 wc->byte_len = le32_to_cpu(cqe->byte_cnt); 3845 3846 hr_opcode = hr_reg_read(cqe, CQE_OPCODE); 3847 switch (hr_opcode) { 3848 case HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM: 3849 case HNS_ROCE_V2_OPCODE_SEND_WITH_IMM: 3850 wc->wc_flags = IB_WC_WITH_IMM; 3851 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(cqe->immtdata)); 3852 break; 3853 case HNS_ROCE_V2_OPCODE_SEND_WITH_INV: 3854 wc->wc_flags = IB_WC_WITH_INVALIDATE; 3855 wc->ex.invalidate_rkey = le32_to_cpu(cqe->rkey); 3856 break; 3857 default: 3858 wc->wc_flags = 0; 3859 } 3860 3861 ib_opcode = to_ib_wc_recv_op(hr_opcode); 3862 if (ib_opcode < 0) 3863 wc->status = IB_WC_GENERAL_ERR; 3864 else 3865 wc->opcode = ib_opcode; 3866 3867 wc->sl = hr_reg_read(cqe, CQE_SL); 3868 wc->src_qp = hr_reg_read(cqe, CQE_RMT_QPN); 3869 wc->slid = 0; 3870 wc->wc_flags |= hr_reg_read(cqe, CQE_GRH) ? IB_WC_GRH : 0; 3871 wc->port_num = hr_reg_read(cqe, CQE_PORTN); 3872 wc->pkey_index = 0; 3873 3874 if (hr_reg_read(cqe, CQE_VID_VLD)) { 3875 wc->vlan_id = hr_reg_read(cqe, CQE_VID); 3876 wc->wc_flags |= IB_WC_WITH_VLAN; 3877 } else { 3878 wc->vlan_id = 0xffff; 3879 } 3880 3881 wc->network_hdr_type = hr_reg_read(cqe, CQE_PORT_TYPE); 3882 3883 return 0; 3884 } 3885 3886 static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq, 3887 struct hns_roce_qp **cur_qp, struct ib_wc *wc) 3888 { 3889 struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device); 3890 struct hns_roce_qp *qp = *cur_qp; 3891 struct hns_roce_srq *srq = NULL; 3892 struct hns_roce_v2_cqe *cqe; 3893 struct hns_roce_wq *wq; 3894 int is_send; 3895 u16 wqe_idx; 3896 int ret; 3897 3898 cqe = get_sw_cqe_v2(hr_cq, hr_cq->cons_index); 3899 if (!cqe) 3900 return -EAGAIN; 3901 3902 ++hr_cq->cons_index; 3903 /* Memory barrier */ 3904 rmb(); 3905 3906 ret = get_cur_qp(hr_cq, cqe, &qp); 3907 if (ret) 3908 return ret; 3909 3910 wc->qp = &qp->ibqp; 3911 wc->vendor_err = 0; 3912 3913 wqe_idx = hr_reg_read(cqe, CQE_WQE_IDX); 3914 3915 is_send = !hr_reg_read(cqe, CQE_S_R); 3916 if (is_send) { 3917 wq = &qp->sq; 3918 3919 /* If sg_signal_bit is set, tail pointer will be updated to 3920 * the WQE corresponding to the current CQE. 3921 */ 3922 if (qp->sq_signal_bits) 3923 wq->tail += (wqe_idx - (u16)wq->tail) & 3924 (wq->wqe_cnt - 1); 3925 3926 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; 3927 ++wq->tail; 3928 3929 fill_send_wc(wc, cqe); 3930 } else { 3931 if (qp->ibqp.srq) { 3932 srq = to_hr_srq(qp->ibqp.srq); 3933 wc->wr_id = srq->wrid[wqe_idx]; 3934 hns_roce_free_srq_wqe(srq, wqe_idx); 3935 } else { 3936 wq = &qp->rq; 3937 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; 3938 ++wq->tail; 3939 } 3940 3941 ret = fill_recv_wc(wc, cqe); 3942 } 3943 3944 get_cqe_status(hr_dev, qp, hr_cq, cqe, wc); 3945 if (unlikely(wc->status != IB_WC_SUCCESS)) 3946 return 0; 3947 3948 return ret; 3949 } 3950 3951 static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries, 3952 struct ib_wc *wc) 3953 { 3954 struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device); 3955 struct hns_roce_cq *hr_cq = to_hr_cq(ibcq); 3956 struct hns_roce_qp *cur_qp = NULL; 3957 unsigned long flags; 3958 int npolled; 3959 3960 spin_lock_irqsave(&hr_cq->lock, flags); 3961 3962 /* 3963 * When the device starts to reset, the state is RST_DOWN. At this time, 3964 * there may still be some valid CQEs in the hardware that are not 3965 * polled. Therefore, it is not allowed to switch to the software mode 3966 * immediately. When the state changes to UNINIT, CQE no longer exists 3967 * in the hardware, and then switch to software mode. 3968 */ 3969 if (hr_dev->state == HNS_ROCE_DEVICE_STATE_UNINIT) { 3970 npolled = hns_roce_v2_sw_poll_cq(hr_cq, num_entries, wc); 3971 goto out; 3972 } 3973 3974 for (npolled = 0; npolled < num_entries; ++npolled) { 3975 if (hns_roce_v2_poll_one(hr_cq, &cur_qp, wc + npolled)) 3976 break; 3977 } 3978 3979 if (npolled) 3980 update_cq_db(hr_dev, hr_cq); 3981 3982 out: 3983 spin_unlock_irqrestore(&hr_cq->lock, flags); 3984 3985 return npolled; 3986 } 3987 3988 static int get_op_for_set_hem(struct hns_roce_dev *hr_dev, u32 type, 3989 u32 step_idx, u8 *mbox_cmd) 3990 { 3991 u8 cmd; 3992 3993 switch (type) { 3994 case HEM_TYPE_QPC: 3995 cmd = HNS_ROCE_CMD_WRITE_QPC_BT0; 3996 break; 3997 case HEM_TYPE_MTPT: 3998 cmd = HNS_ROCE_CMD_WRITE_MPT_BT0; 3999 break; 4000 case HEM_TYPE_CQC: 4001 cmd = HNS_ROCE_CMD_WRITE_CQC_BT0; 4002 break; 4003 case HEM_TYPE_SRQC: 4004 cmd = HNS_ROCE_CMD_WRITE_SRQC_BT0; 4005 break; 4006 case HEM_TYPE_SCCC: 4007 cmd = HNS_ROCE_CMD_WRITE_SCCC_BT0; 4008 break; 4009 case HEM_TYPE_QPC_TIMER: 4010 cmd = HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0; 4011 break; 4012 case HEM_TYPE_CQC_TIMER: 4013 cmd = HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0; 4014 break; 4015 default: 4016 dev_warn(hr_dev->dev, "failed to check hem type %u.\n", type); 4017 return -EINVAL; 4018 } 4019 4020 *mbox_cmd = cmd + step_idx; 4021 4022 return 0; 4023 } 4024 4025 static int config_gmv_ba_to_hw(struct hns_roce_dev *hr_dev, unsigned long obj, 4026 dma_addr_t base_addr) 4027 { 4028 struct hns_roce_cmq_desc desc; 4029 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data; 4030 u32 idx = obj / (HNS_HW_PAGE_SIZE / hr_dev->caps.gmv_entry_sz); 4031 u64 addr = to_hr_hw_page_addr(base_addr); 4032 4033 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT, false); 4034 4035 hr_reg_write(req, CFG_GMV_BT_BA_L, lower_32_bits(addr)); 4036 hr_reg_write(req, CFG_GMV_BT_BA_H, upper_32_bits(addr)); 4037 hr_reg_write(req, CFG_GMV_BT_IDX, idx); 4038 4039 return hns_roce_cmq_send(hr_dev, &desc, 1); 4040 } 4041 4042 static int set_hem_to_hw(struct hns_roce_dev *hr_dev, int obj, 4043 dma_addr_t base_addr, u32 hem_type, u32 step_idx) 4044 { 4045 int ret; 4046 u8 cmd; 4047 4048 if (unlikely(hem_type == HEM_TYPE_GMV)) 4049 return config_gmv_ba_to_hw(hr_dev, obj, base_addr); 4050 4051 if (unlikely(hem_type == HEM_TYPE_SCCC && step_idx)) 4052 return 0; 4053 4054 ret = get_op_for_set_hem(hr_dev, hem_type, step_idx, &cmd); 4055 if (ret < 0) 4056 return ret; 4057 4058 return config_hem_ba_to_hw(hr_dev, base_addr, cmd, obj); 4059 } 4060 4061 static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev, 4062 struct hns_roce_hem_table *table, int obj, 4063 u32 step_idx) 4064 { 4065 struct hns_roce_hem_mhop mhop; 4066 struct hns_roce_hem *hem; 4067 unsigned long mhop_obj = obj; 4068 int i, j, k; 4069 int ret = 0; 4070 u64 hem_idx = 0; 4071 u64 l1_idx = 0; 4072 u64 bt_ba = 0; 4073 u32 chunk_ba_num; 4074 u32 hop_num; 4075 4076 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) 4077 return 0; 4078 4079 hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop); 4080 i = mhop.l0_idx; 4081 j = mhop.l1_idx; 4082 k = mhop.l2_idx; 4083 hop_num = mhop.hop_num; 4084 chunk_ba_num = mhop.bt_chunk_size / 8; 4085 4086 if (hop_num == 2) { 4087 hem_idx = i * chunk_ba_num * chunk_ba_num + j * chunk_ba_num + 4088 k; 4089 l1_idx = i * chunk_ba_num + j; 4090 } else if (hop_num == 1) { 4091 hem_idx = i * chunk_ba_num + j; 4092 } else if (hop_num == HNS_ROCE_HOP_NUM_0) { 4093 hem_idx = i; 4094 } 4095 4096 if (table->type == HEM_TYPE_SCCC) 4097 obj = mhop.l0_idx; 4098 4099 if (check_whether_last_step(hop_num, step_idx)) { 4100 hem = table->hem[hem_idx]; 4101 4102 ret = set_hem_to_hw(hr_dev, obj, hem->dma, table->type, step_idx); 4103 } else { 4104 if (step_idx == 0) 4105 bt_ba = table->bt_l0_dma_addr[i]; 4106 else if (step_idx == 1 && hop_num == 2) 4107 bt_ba = table->bt_l1_dma_addr[l1_idx]; 4108 4109 ret = set_hem_to_hw(hr_dev, obj, bt_ba, table->type, step_idx); 4110 } 4111 4112 return ret; 4113 } 4114 4115 static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev, 4116 struct hns_roce_hem_table *table, 4117 int tag, u32 step_idx) 4118 { 4119 struct hns_roce_cmd_mailbox *mailbox; 4120 struct device *dev = hr_dev->dev; 4121 u8 cmd = 0xff; 4122 int ret; 4123 4124 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) 4125 return 0; 4126 4127 switch (table->type) { 4128 case HEM_TYPE_QPC: 4129 cmd = HNS_ROCE_CMD_DESTROY_QPC_BT0; 4130 break; 4131 case HEM_TYPE_MTPT: 4132 cmd = HNS_ROCE_CMD_DESTROY_MPT_BT0; 4133 break; 4134 case HEM_TYPE_CQC: 4135 cmd = HNS_ROCE_CMD_DESTROY_CQC_BT0; 4136 break; 4137 case HEM_TYPE_SRQC: 4138 cmd = HNS_ROCE_CMD_DESTROY_SRQC_BT0; 4139 break; 4140 case HEM_TYPE_SCCC: 4141 case HEM_TYPE_QPC_TIMER: 4142 case HEM_TYPE_CQC_TIMER: 4143 case HEM_TYPE_GMV: 4144 return 0; 4145 default: 4146 dev_warn(dev, "table %u not to be destroyed by mailbox!\n", 4147 table->type); 4148 return 0; 4149 } 4150 4151 cmd += step_idx; 4152 4153 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 4154 if (IS_ERR(mailbox)) 4155 return PTR_ERR(mailbox); 4156 4157 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, cmd, tag); 4158 4159 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 4160 return ret; 4161 } 4162 4163 static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev, 4164 struct hns_roce_v2_qp_context *context, 4165 struct hns_roce_v2_qp_context *qpc_mask, 4166 struct hns_roce_qp *hr_qp) 4167 { 4168 struct hns_roce_cmd_mailbox *mailbox; 4169 int qpc_size; 4170 int ret; 4171 4172 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 4173 if (IS_ERR(mailbox)) 4174 return PTR_ERR(mailbox); 4175 4176 /* The qpc size of HIP08 is only 256B, which is half of HIP09 */ 4177 qpc_size = hr_dev->caps.qpc_sz; 4178 memcpy(mailbox->buf, context, qpc_size); 4179 memcpy(mailbox->buf + qpc_size, qpc_mask, qpc_size); 4180 4181 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, 4182 HNS_ROCE_CMD_MODIFY_QPC, hr_qp->qpn); 4183 4184 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 4185 4186 return ret; 4187 } 4188 4189 static void set_access_flags(struct hns_roce_qp *hr_qp, 4190 struct hns_roce_v2_qp_context *context, 4191 struct hns_roce_v2_qp_context *qpc_mask, 4192 const struct ib_qp_attr *attr, int attr_mask) 4193 { 4194 u8 dest_rd_atomic; 4195 u32 access_flags; 4196 4197 dest_rd_atomic = (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) ? 4198 attr->max_dest_rd_atomic : hr_qp->resp_depth; 4199 4200 access_flags = (attr_mask & IB_QP_ACCESS_FLAGS) ? 4201 attr->qp_access_flags : hr_qp->atomic_rd_en; 4202 4203 if (!dest_rd_atomic) 4204 access_flags &= IB_ACCESS_REMOTE_WRITE; 4205 4206 hr_reg_write_bool(context, QPC_RRE, 4207 access_flags & IB_ACCESS_REMOTE_READ); 4208 hr_reg_clear(qpc_mask, QPC_RRE); 4209 4210 hr_reg_write_bool(context, QPC_RWE, 4211 access_flags & IB_ACCESS_REMOTE_WRITE); 4212 hr_reg_clear(qpc_mask, QPC_RWE); 4213 4214 hr_reg_write_bool(context, QPC_ATE, 4215 access_flags & IB_ACCESS_REMOTE_ATOMIC); 4216 hr_reg_clear(qpc_mask, QPC_ATE); 4217 hr_reg_write_bool(context, QPC_EXT_ATE, 4218 access_flags & IB_ACCESS_REMOTE_ATOMIC); 4219 hr_reg_clear(qpc_mask, QPC_EXT_ATE); 4220 } 4221 4222 static void set_qpc_wqe_cnt(struct hns_roce_qp *hr_qp, 4223 struct hns_roce_v2_qp_context *context) 4224 { 4225 hr_reg_write(context, QPC_SGE_SHIFT, 4226 to_hr_hem_entries_shift(hr_qp->sge.sge_cnt, 4227 hr_qp->sge.sge_shift)); 4228 4229 hr_reg_write(context, QPC_SQ_SHIFT, ilog2(hr_qp->sq.wqe_cnt)); 4230 4231 hr_reg_write(context, QPC_RQ_SHIFT, ilog2(hr_qp->rq.wqe_cnt)); 4232 } 4233 4234 static inline int get_cqn(struct ib_cq *ib_cq) 4235 { 4236 return ib_cq ? to_hr_cq(ib_cq)->cqn : 0; 4237 } 4238 4239 static inline int get_pdn(struct ib_pd *ib_pd) 4240 { 4241 return ib_pd ? to_hr_pd(ib_pd)->pdn : 0; 4242 } 4243 4244 static void modify_qp_reset_to_init(struct ib_qp *ibqp, 4245 struct hns_roce_v2_qp_context *context, 4246 struct hns_roce_v2_qp_context *qpc_mask) 4247 { 4248 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 4249 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 4250 4251 /* 4252 * In v2 engine, software pass context and context mask to hardware 4253 * when modifying qp. If software need modify some fields in context, 4254 * we should set all bits of the relevant fields in context mask to 4255 * 0 at the same time, else set them to 0x1. 4256 */ 4257 hr_reg_write(context, QPC_TST, to_hr_qp_type(ibqp->qp_type)); 4258 4259 hr_reg_write(context, QPC_PD, get_pdn(ibqp->pd)); 4260 4261 hr_reg_write(context, QPC_RQWS, ilog2(hr_qp->rq.max_gs)); 4262 4263 set_qpc_wqe_cnt(hr_qp, context); 4264 4265 /* No VLAN need to set 0xFFF */ 4266 hr_reg_write(context, QPC_VLAN_ID, 0xfff); 4267 4268 if (ibqp->qp_type == IB_QPT_XRC_TGT) { 4269 context->qkey_xrcd = cpu_to_le32(hr_qp->xrcdn); 4270 4271 hr_reg_enable(context, QPC_XRC_QP_TYPE); 4272 } 4273 4274 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) 4275 hr_reg_enable(context, QPC_RQ_RECORD_EN); 4276 4277 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_OWNER_DB) 4278 hr_reg_enable(context, QPC_OWNER_MODE); 4279 4280 hr_reg_write(context, QPC_RQ_DB_RECORD_ADDR_L, 4281 lower_32_bits(hr_qp->rdb.dma) >> 1); 4282 hr_reg_write(context, QPC_RQ_DB_RECORD_ADDR_H, 4283 upper_32_bits(hr_qp->rdb.dma)); 4284 4285 hr_reg_write(context, QPC_RX_CQN, get_cqn(ibqp->recv_cq)); 4286 4287 if (ibqp->srq) { 4288 hr_reg_enable(context, QPC_SRQ_EN); 4289 hr_reg_write(context, QPC_SRQN, to_hr_srq(ibqp->srq)->srqn); 4290 } 4291 4292 hr_reg_enable(context, QPC_FRE); 4293 4294 hr_reg_write(context, QPC_TX_CQN, get_cqn(ibqp->send_cq)); 4295 4296 if (hr_dev->caps.qpc_sz < HNS_ROCE_V3_QPC_SZ) 4297 return; 4298 4299 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_STASH) 4300 hr_reg_enable(&context->ext, QPCEX_STASH); 4301 } 4302 4303 static void modify_qp_init_to_init(struct ib_qp *ibqp, 4304 struct hns_roce_v2_qp_context *context, 4305 struct hns_roce_v2_qp_context *qpc_mask) 4306 { 4307 /* 4308 * In v2 engine, software pass context and context mask to hardware 4309 * when modifying qp. If software need modify some fields in context, 4310 * we should set all bits of the relevant fields in context mask to 4311 * 0 at the same time, else set them to 0x1. 4312 */ 4313 hr_reg_write(context, QPC_TST, to_hr_qp_type(ibqp->qp_type)); 4314 hr_reg_clear(qpc_mask, QPC_TST); 4315 4316 hr_reg_write(context, QPC_PD, get_pdn(ibqp->pd)); 4317 hr_reg_clear(qpc_mask, QPC_PD); 4318 4319 hr_reg_write(context, QPC_RX_CQN, get_cqn(ibqp->recv_cq)); 4320 hr_reg_clear(qpc_mask, QPC_RX_CQN); 4321 4322 hr_reg_write(context, QPC_TX_CQN, get_cqn(ibqp->send_cq)); 4323 hr_reg_clear(qpc_mask, QPC_TX_CQN); 4324 4325 if (ibqp->srq) { 4326 hr_reg_enable(context, QPC_SRQ_EN); 4327 hr_reg_clear(qpc_mask, QPC_SRQ_EN); 4328 hr_reg_write(context, QPC_SRQN, to_hr_srq(ibqp->srq)->srqn); 4329 hr_reg_clear(qpc_mask, QPC_SRQN); 4330 } 4331 } 4332 4333 static int config_qp_rq_buf(struct hns_roce_dev *hr_dev, 4334 struct hns_roce_qp *hr_qp, 4335 struct hns_roce_v2_qp_context *context, 4336 struct hns_roce_v2_qp_context *qpc_mask) 4337 { 4338 u64 mtts[MTT_MIN_COUNT] = { 0 }; 4339 u64 wqe_sge_ba; 4340 int ret; 4341 4342 /* Search qp buf's mtts */ 4343 ret = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->rq.offset, mtts, 4344 MTT_MIN_COUNT); 4345 if (hr_qp->rq.wqe_cnt && ret) { 4346 ibdev_err(&hr_dev->ib_dev, 4347 "failed to find QP(0x%lx) RQ WQE buf, ret = %d.\n", 4348 hr_qp->qpn, ret); 4349 return ret; 4350 } 4351 4352 wqe_sge_ba = hns_roce_get_mtr_ba(&hr_qp->mtr); 4353 4354 context->wqe_sge_ba = cpu_to_le32(wqe_sge_ba >> 3); 4355 qpc_mask->wqe_sge_ba = 0; 4356 4357 /* 4358 * In v2 engine, software pass context and context mask to hardware 4359 * when modifying qp. If software need modify some fields in context, 4360 * we should set all bits of the relevant fields in context mask to 4361 * 0 at the same time, else set them to 0x1. 4362 */ 4363 hr_reg_write(context, QPC_WQE_SGE_BA_H, wqe_sge_ba >> (32 + 3)); 4364 hr_reg_clear(qpc_mask, QPC_WQE_SGE_BA_H); 4365 4366 hr_reg_write(context, QPC_SQ_HOP_NUM, 4367 to_hr_hem_hopnum(hr_dev->caps.wqe_sq_hop_num, 4368 hr_qp->sq.wqe_cnt)); 4369 hr_reg_clear(qpc_mask, QPC_SQ_HOP_NUM); 4370 4371 hr_reg_write(context, QPC_SGE_HOP_NUM, 4372 to_hr_hem_hopnum(hr_dev->caps.wqe_sge_hop_num, 4373 hr_qp->sge.sge_cnt)); 4374 hr_reg_clear(qpc_mask, QPC_SGE_HOP_NUM); 4375 4376 hr_reg_write(context, QPC_RQ_HOP_NUM, 4377 to_hr_hem_hopnum(hr_dev->caps.wqe_rq_hop_num, 4378 hr_qp->rq.wqe_cnt)); 4379 4380 hr_reg_clear(qpc_mask, QPC_RQ_HOP_NUM); 4381 4382 hr_reg_write(context, QPC_WQE_SGE_BA_PG_SZ, 4383 to_hr_hw_page_shift(hr_qp->mtr.hem_cfg.ba_pg_shift)); 4384 hr_reg_clear(qpc_mask, QPC_WQE_SGE_BA_PG_SZ); 4385 4386 hr_reg_write(context, QPC_WQE_SGE_BUF_PG_SZ, 4387 to_hr_hw_page_shift(hr_qp->mtr.hem_cfg.buf_pg_shift)); 4388 hr_reg_clear(qpc_mask, QPC_WQE_SGE_BUF_PG_SZ); 4389 4390 context->rq_cur_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[0])); 4391 qpc_mask->rq_cur_blk_addr = 0; 4392 4393 hr_reg_write(context, QPC_RQ_CUR_BLK_ADDR_H, 4394 upper_32_bits(to_hr_hw_page_addr(mtts[0]))); 4395 hr_reg_clear(qpc_mask, QPC_RQ_CUR_BLK_ADDR_H); 4396 4397 context->rq_nxt_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[1])); 4398 qpc_mask->rq_nxt_blk_addr = 0; 4399 4400 hr_reg_write(context, QPC_RQ_NXT_BLK_ADDR_H, 4401 upper_32_bits(to_hr_hw_page_addr(mtts[1]))); 4402 hr_reg_clear(qpc_mask, QPC_RQ_NXT_BLK_ADDR_H); 4403 4404 return 0; 4405 } 4406 4407 static int config_qp_sq_buf(struct hns_roce_dev *hr_dev, 4408 struct hns_roce_qp *hr_qp, 4409 struct hns_roce_v2_qp_context *context, 4410 struct hns_roce_v2_qp_context *qpc_mask) 4411 { 4412 struct ib_device *ibdev = &hr_dev->ib_dev; 4413 u64 sge_cur_blk = 0; 4414 u64 sq_cur_blk = 0; 4415 int ret; 4416 4417 /* search qp buf's mtts */ 4418 ret = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->sq.offset, 4419 &sq_cur_blk, 1); 4420 if (ret) { 4421 ibdev_err(ibdev, "failed to find QP(0x%lx) SQ WQE buf, ret = %d.\n", 4422 hr_qp->qpn, ret); 4423 return ret; 4424 } 4425 if (hr_qp->sge.sge_cnt > 0) { 4426 ret = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 4427 hr_qp->sge.offset, &sge_cur_blk, 1); 4428 if (ret) { 4429 ibdev_err(ibdev, "failed to find QP(0x%lx) SGE buf, ret = %d.\n", 4430 hr_qp->qpn, ret); 4431 return ret; 4432 } 4433 } 4434 4435 /* 4436 * In v2 engine, software pass context and context mask to hardware 4437 * when modifying qp. If software need modify some fields in context, 4438 * we should set all bits of the relevant fields in context mask to 4439 * 0 at the same time, else set them to 0x1. 4440 */ 4441 hr_reg_write(context, QPC_SQ_CUR_BLK_ADDR_L, 4442 lower_32_bits(to_hr_hw_page_addr(sq_cur_blk))); 4443 hr_reg_write(context, QPC_SQ_CUR_BLK_ADDR_H, 4444 upper_32_bits(to_hr_hw_page_addr(sq_cur_blk))); 4445 hr_reg_clear(qpc_mask, QPC_SQ_CUR_BLK_ADDR_L); 4446 hr_reg_clear(qpc_mask, QPC_SQ_CUR_BLK_ADDR_H); 4447 4448 hr_reg_write(context, QPC_SQ_CUR_SGE_BLK_ADDR_L, 4449 lower_32_bits(to_hr_hw_page_addr(sge_cur_blk))); 4450 hr_reg_write(context, QPC_SQ_CUR_SGE_BLK_ADDR_H, 4451 upper_32_bits(to_hr_hw_page_addr(sge_cur_blk))); 4452 hr_reg_clear(qpc_mask, QPC_SQ_CUR_SGE_BLK_ADDR_L); 4453 hr_reg_clear(qpc_mask, QPC_SQ_CUR_SGE_BLK_ADDR_H); 4454 4455 hr_reg_write(context, QPC_RX_SQ_CUR_BLK_ADDR_L, 4456 lower_32_bits(to_hr_hw_page_addr(sq_cur_blk))); 4457 hr_reg_write(context, QPC_RX_SQ_CUR_BLK_ADDR_H, 4458 upper_32_bits(to_hr_hw_page_addr(sq_cur_blk))); 4459 hr_reg_clear(qpc_mask, QPC_RX_SQ_CUR_BLK_ADDR_L); 4460 hr_reg_clear(qpc_mask, QPC_RX_SQ_CUR_BLK_ADDR_H); 4461 4462 return 0; 4463 } 4464 4465 static inline enum ib_mtu get_mtu(struct ib_qp *ibqp, 4466 const struct ib_qp_attr *attr) 4467 { 4468 if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_UD) 4469 return IB_MTU_4096; 4470 4471 return attr->path_mtu; 4472 } 4473 4474 static int modify_qp_init_to_rtr(struct ib_qp *ibqp, 4475 const struct ib_qp_attr *attr, int attr_mask, 4476 struct hns_roce_v2_qp_context *context, 4477 struct hns_roce_v2_qp_context *qpc_mask, 4478 struct ib_udata *udata) 4479 { 4480 struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(udata, 4481 struct hns_roce_ucontext, ibucontext); 4482 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 4483 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 4484 struct ib_device *ibdev = &hr_dev->ib_dev; 4485 dma_addr_t trrl_ba; 4486 dma_addr_t irrl_ba; 4487 enum ib_mtu ib_mtu; 4488 const u8 *smac; 4489 u8 lp_pktn_ini; 4490 u64 *mtts; 4491 u8 *dmac; 4492 u32 port; 4493 int mtu; 4494 int ret; 4495 4496 ret = config_qp_rq_buf(hr_dev, hr_qp, context, qpc_mask); 4497 if (ret) { 4498 ibdev_err(ibdev, "failed to config rq buf, ret = %d.\n", ret); 4499 return ret; 4500 } 4501 4502 /* Search IRRL's mtts */ 4503 mtts = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table, 4504 hr_qp->qpn, &irrl_ba); 4505 if (!mtts) { 4506 ibdev_err(ibdev, "failed to find qp irrl_table.\n"); 4507 return -EINVAL; 4508 } 4509 4510 /* Search TRRL's mtts */ 4511 mtts = hns_roce_table_find(hr_dev, &hr_dev->qp_table.trrl_table, 4512 hr_qp->qpn, &trrl_ba); 4513 if (!mtts) { 4514 ibdev_err(ibdev, "failed to find qp trrl_table.\n"); 4515 return -EINVAL; 4516 } 4517 4518 if (attr_mask & IB_QP_ALT_PATH) { 4519 ibdev_err(ibdev, "INIT2RTR attr_mask (0x%x) error.\n", 4520 attr_mask); 4521 return -EINVAL; 4522 } 4523 4524 hr_reg_write(context, QPC_TRRL_BA_L, trrl_ba >> QPC_TRRL_BA_L_S); 4525 hr_reg_clear(qpc_mask, QPC_TRRL_BA_L); 4526 context->trrl_ba = cpu_to_le32(trrl_ba >> QPC_TRRL_BA_M_S); 4527 qpc_mask->trrl_ba = 0; 4528 hr_reg_write(context, QPC_TRRL_BA_H, trrl_ba >> QPC_TRRL_BA_H_S); 4529 hr_reg_clear(qpc_mask, QPC_TRRL_BA_H); 4530 4531 context->irrl_ba = cpu_to_le32(irrl_ba >> QPC_IRRL_BA_L_S); 4532 qpc_mask->irrl_ba = 0; 4533 hr_reg_write(context, QPC_IRRL_BA_H, irrl_ba >> QPC_IRRL_BA_H_S); 4534 hr_reg_clear(qpc_mask, QPC_IRRL_BA_H); 4535 4536 hr_reg_enable(context, QPC_RMT_E2E); 4537 hr_reg_clear(qpc_mask, QPC_RMT_E2E); 4538 4539 hr_reg_write(context, QPC_SIG_TYPE, hr_qp->sq_signal_bits); 4540 hr_reg_clear(qpc_mask, QPC_SIG_TYPE); 4541 4542 port = (attr_mask & IB_QP_PORT) ? (attr->port_num - 1) : hr_qp->port; 4543 4544 smac = (const u8 *)hr_dev->dev_addr[port]; 4545 dmac = (u8 *)attr->ah_attr.roce.dmac; 4546 /* when dmac equals smac or loop_idc is 1, it should loopback */ 4547 if (ether_addr_equal_unaligned(dmac, smac) || 4548 hr_dev->loop_idc == 0x1) { 4549 hr_reg_write(context, QPC_LBI, hr_dev->loop_idc); 4550 hr_reg_clear(qpc_mask, QPC_LBI); 4551 } 4552 4553 if (attr_mask & IB_QP_DEST_QPN) { 4554 hr_reg_write(context, QPC_DQPN, attr->dest_qp_num); 4555 hr_reg_clear(qpc_mask, QPC_DQPN); 4556 } 4557 4558 memcpy(&context->dmac, dmac, sizeof(u32)); 4559 hr_reg_write(context, QPC_DMAC_H, *((u16 *)(&dmac[4]))); 4560 qpc_mask->dmac = 0; 4561 hr_reg_clear(qpc_mask, QPC_DMAC_H); 4562 4563 ib_mtu = get_mtu(ibqp, attr); 4564 hr_qp->path_mtu = ib_mtu; 4565 4566 mtu = ib_mtu_enum_to_int(ib_mtu); 4567 if (WARN_ON(mtu <= 0)) 4568 return -EINVAL; 4569 #define MIN_LP_MSG_LEN 1024 4570 /* mtu * (2 ^ lp_pktn_ini) should be in the range of 1024 to mtu */ 4571 lp_pktn_ini = ilog2(max(mtu, MIN_LP_MSG_LEN) / mtu); 4572 4573 if (attr_mask & IB_QP_PATH_MTU) { 4574 hr_reg_write(context, QPC_MTU, ib_mtu); 4575 hr_reg_clear(qpc_mask, QPC_MTU); 4576 } 4577 4578 hr_reg_write(context, QPC_LP_PKTN_INI, lp_pktn_ini); 4579 hr_reg_clear(qpc_mask, QPC_LP_PKTN_INI); 4580 4581 /* ACK_REQ_FREQ should be larger than or equal to LP_PKTN_INI */ 4582 hr_reg_write(context, QPC_ACK_REQ_FREQ, lp_pktn_ini); 4583 hr_reg_clear(qpc_mask, QPC_ACK_REQ_FREQ); 4584 4585 hr_reg_clear(qpc_mask, QPC_RX_REQ_PSN_ERR); 4586 hr_reg_clear(qpc_mask, QPC_RX_REQ_MSN); 4587 hr_reg_clear(qpc_mask, QPC_RX_REQ_LAST_OPTYPE); 4588 4589 context->rq_rnr_timer = 0; 4590 qpc_mask->rq_rnr_timer = 0; 4591 4592 hr_reg_clear(qpc_mask, QPC_TRRL_HEAD_MAX); 4593 hr_reg_clear(qpc_mask, QPC_TRRL_TAIL_MAX); 4594 4595 #define MAX_LP_SGEN 3 4596 /* rocee send 2^lp_sgen_ini segs every time */ 4597 hr_reg_write(context, QPC_LP_SGEN_INI, MAX_LP_SGEN); 4598 hr_reg_clear(qpc_mask, QPC_LP_SGEN_INI); 4599 4600 if (udata && ibqp->qp_type == IB_QPT_RC && 4601 (uctx->config & HNS_ROCE_RQ_INLINE_FLAGS)) { 4602 hr_reg_write_bool(context, QPC_RQIE, 4603 hr_dev->caps.flags & 4604 HNS_ROCE_CAP_FLAG_RQ_INLINE); 4605 hr_reg_clear(qpc_mask, QPC_RQIE); 4606 } 4607 4608 if (udata && 4609 (ibqp->qp_type == IB_QPT_RC || ibqp->qp_type == IB_QPT_XRC_TGT) && 4610 (uctx->config & HNS_ROCE_CQE_INLINE_FLAGS)) { 4611 hr_reg_write_bool(context, QPC_CQEIE, 4612 hr_dev->caps.flags & 4613 HNS_ROCE_CAP_FLAG_CQE_INLINE); 4614 hr_reg_clear(qpc_mask, QPC_CQEIE); 4615 4616 hr_reg_write(context, QPC_CQEIS, 0); 4617 hr_reg_clear(qpc_mask, QPC_CQEIS); 4618 } 4619 4620 return 0; 4621 } 4622 4623 static int modify_qp_rtr_to_rts(struct ib_qp *ibqp, int attr_mask, 4624 struct hns_roce_v2_qp_context *context, 4625 struct hns_roce_v2_qp_context *qpc_mask) 4626 { 4627 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 4628 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 4629 struct ib_device *ibdev = &hr_dev->ib_dev; 4630 int ret; 4631 4632 /* Not support alternate path and path migration */ 4633 if (attr_mask & (IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE)) { 4634 ibdev_err(ibdev, "RTR2RTS attr_mask (0x%x)error\n", attr_mask); 4635 return -EINVAL; 4636 } 4637 4638 ret = config_qp_sq_buf(hr_dev, hr_qp, context, qpc_mask); 4639 if (ret) { 4640 ibdev_err(ibdev, "failed to config sq buf, ret = %d.\n", ret); 4641 return ret; 4642 } 4643 4644 /* 4645 * Set some fields in context to zero, Because the default values 4646 * of all fields in context are zero, we need not set them to 0 again. 4647 * but we should set the relevant fields of context mask to 0. 4648 */ 4649 hr_reg_clear(qpc_mask, QPC_IRRL_SGE_IDX); 4650 4651 hr_reg_clear(qpc_mask, QPC_RX_ACK_MSN); 4652 4653 hr_reg_clear(qpc_mask, QPC_ACK_LAST_OPTYPE); 4654 hr_reg_clear(qpc_mask, QPC_IRRL_PSN_VLD); 4655 hr_reg_clear(qpc_mask, QPC_IRRL_PSN); 4656 4657 hr_reg_clear(qpc_mask, QPC_IRRL_TAIL_REAL); 4658 4659 hr_reg_clear(qpc_mask, QPC_RETRY_MSG_MSN); 4660 4661 hr_reg_clear(qpc_mask, QPC_RNR_RETRY_FLAG); 4662 4663 hr_reg_clear(qpc_mask, QPC_CHECK_FLG); 4664 4665 hr_reg_clear(qpc_mask, QPC_V2_IRRL_HEAD); 4666 4667 return 0; 4668 } 4669 4670 static int get_dip_ctx_idx(struct ib_qp *ibqp, const struct ib_qp_attr *attr, 4671 u32 *dip_idx) 4672 { 4673 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 4674 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 4675 u32 *spare_idx = hr_dev->qp_table.idx_table.spare_idx; 4676 u32 *head = &hr_dev->qp_table.idx_table.head; 4677 u32 *tail = &hr_dev->qp_table.idx_table.tail; 4678 struct hns_roce_dip *hr_dip; 4679 unsigned long flags; 4680 int ret = 0; 4681 4682 spin_lock_irqsave(&hr_dev->dip_list_lock, flags); 4683 4684 spare_idx[*tail] = ibqp->qp_num; 4685 *tail = (*tail == hr_dev->caps.num_qps - 1) ? 0 : (*tail + 1); 4686 4687 list_for_each_entry(hr_dip, &hr_dev->dip_list, node) { 4688 if (!memcmp(grh->dgid.raw, hr_dip->dgid, GID_LEN_V2)) { 4689 *dip_idx = hr_dip->dip_idx; 4690 goto out; 4691 } 4692 } 4693 4694 /* If no dgid is found, a new dip and a mapping between dgid and 4695 * dip_idx will be created. 4696 */ 4697 hr_dip = kzalloc(sizeof(*hr_dip), GFP_ATOMIC); 4698 if (!hr_dip) { 4699 ret = -ENOMEM; 4700 goto out; 4701 } 4702 4703 memcpy(hr_dip->dgid, grh->dgid.raw, sizeof(grh->dgid.raw)); 4704 hr_dip->dip_idx = *dip_idx = spare_idx[*head]; 4705 *head = (*head == hr_dev->caps.num_qps - 1) ? 0 : (*head + 1); 4706 list_add_tail(&hr_dip->node, &hr_dev->dip_list); 4707 4708 out: 4709 spin_unlock_irqrestore(&hr_dev->dip_list_lock, flags); 4710 return ret; 4711 } 4712 4713 enum { 4714 CONG_DCQCN, 4715 CONG_WINDOW, 4716 }; 4717 4718 enum { 4719 UNSUPPORT_CONG_LEVEL, 4720 SUPPORT_CONG_LEVEL, 4721 }; 4722 4723 enum { 4724 CONG_LDCP, 4725 CONG_HC3, 4726 }; 4727 4728 enum { 4729 DIP_INVALID, 4730 DIP_VALID, 4731 }; 4732 4733 enum { 4734 WND_LIMIT, 4735 WND_UNLIMIT, 4736 }; 4737 4738 static int check_cong_type(struct ib_qp *ibqp, 4739 struct hns_roce_congestion_algorithm *cong_alg) 4740 { 4741 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 4742 4743 /* different congestion types match different configurations */ 4744 switch (hr_qp->cong_type) { 4745 case CONG_TYPE_DCQCN: 4746 cong_alg->alg_sel = CONG_DCQCN; 4747 cong_alg->alg_sub_sel = UNSUPPORT_CONG_LEVEL; 4748 cong_alg->dip_vld = DIP_INVALID; 4749 cong_alg->wnd_mode_sel = WND_LIMIT; 4750 break; 4751 case CONG_TYPE_LDCP: 4752 cong_alg->alg_sel = CONG_WINDOW; 4753 cong_alg->alg_sub_sel = CONG_LDCP; 4754 cong_alg->dip_vld = DIP_INVALID; 4755 cong_alg->wnd_mode_sel = WND_UNLIMIT; 4756 break; 4757 case CONG_TYPE_HC3: 4758 cong_alg->alg_sel = CONG_WINDOW; 4759 cong_alg->alg_sub_sel = CONG_HC3; 4760 cong_alg->dip_vld = DIP_INVALID; 4761 cong_alg->wnd_mode_sel = WND_LIMIT; 4762 break; 4763 case CONG_TYPE_DIP: 4764 cong_alg->alg_sel = CONG_DCQCN; 4765 cong_alg->alg_sub_sel = UNSUPPORT_CONG_LEVEL; 4766 cong_alg->dip_vld = DIP_VALID; 4767 cong_alg->wnd_mode_sel = WND_LIMIT; 4768 break; 4769 default: 4770 hr_qp->cong_type = CONG_TYPE_DCQCN; 4771 cong_alg->alg_sel = CONG_DCQCN; 4772 cong_alg->alg_sub_sel = UNSUPPORT_CONG_LEVEL; 4773 cong_alg->dip_vld = DIP_INVALID; 4774 cong_alg->wnd_mode_sel = WND_LIMIT; 4775 break; 4776 } 4777 4778 return 0; 4779 } 4780 4781 static int fill_cong_field(struct ib_qp *ibqp, const struct ib_qp_attr *attr, 4782 struct hns_roce_v2_qp_context *context, 4783 struct hns_roce_v2_qp_context *qpc_mask) 4784 { 4785 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 4786 struct hns_roce_congestion_algorithm cong_field; 4787 struct ib_device *ibdev = ibqp->device; 4788 struct hns_roce_dev *hr_dev = to_hr_dev(ibdev); 4789 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 4790 u32 dip_idx = 0; 4791 int ret; 4792 4793 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 || 4794 grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE) 4795 return 0; 4796 4797 ret = check_cong_type(ibqp, &cong_field); 4798 if (ret) 4799 return ret; 4800 4801 hr_reg_write(context, QPC_CONG_ALGO_TMPL_ID, hr_dev->cong_algo_tmpl_id + 4802 hr_qp->cong_type * HNS_ROCE_CONG_SIZE); 4803 hr_reg_clear(qpc_mask, QPC_CONG_ALGO_TMPL_ID); 4804 hr_reg_write(&context->ext, QPCEX_CONG_ALG_SEL, cong_field.alg_sel); 4805 hr_reg_clear(&qpc_mask->ext, QPCEX_CONG_ALG_SEL); 4806 hr_reg_write(&context->ext, QPCEX_CONG_ALG_SUB_SEL, 4807 cong_field.alg_sub_sel); 4808 hr_reg_clear(&qpc_mask->ext, QPCEX_CONG_ALG_SUB_SEL); 4809 hr_reg_write(&context->ext, QPCEX_DIP_CTX_IDX_VLD, cong_field.dip_vld); 4810 hr_reg_clear(&qpc_mask->ext, QPCEX_DIP_CTX_IDX_VLD); 4811 hr_reg_write(&context->ext, QPCEX_SQ_RQ_NOT_FORBID_EN, 4812 cong_field.wnd_mode_sel); 4813 hr_reg_clear(&qpc_mask->ext, QPCEX_SQ_RQ_NOT_FORBID_EN); 4814 4815 /* if dip is disabled, there is no need to set dip idx */ 4816 if (cong_field.dip_vld == 0) 4817 return 0; 4818 4819 ret = get_dip_ctx_idx(ibqp, attr, &dip_idx); 4820 if (ret) { 4821 ibdev_err(ibdev, "failed to fill cong field, ret = %d.\n", ret); 4822 return ret; 4823 } 4824 4825 hr_reg_write(&context->ext, QPCEX_DIP_CTX_IDX, dip_idx); 4826 hr_reg_write(&qpc_mask->ext, QPCEX_DIP_CTX_IDX, 0); 4827 4828 return 0; 4829 } 4830 4831 static int hns_roce_hw_v2_get_dscp(struct hns_roce_dev *hr_dev, u8 dscp, 4832 u8 *tc_mode, u8 *priority) 4833 { 4834 struct hns_roce_v2_priv *priv = hr_dev->priv; 4835 struct hnae3_handle *handle = priv->handle; 4836 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 4837 4838 if (!ops->get_dscp_prio) 4839 return -EOPNOTSUPP; 4840 4841 return ops->get_dscp_prio(handle, dscp, tc_mode, priority); 4842 } 4843 4844 bool check_sl_valid(struct hns_roce_dev *hr_dev, u8 sl) 4845 { 4846 u32 max_sl; 4847 4848 max_sl = min_t(u32, MAX_SERVICE_LEVEL, hr_dev->caps.sl_num - 1); 4849 if (unlikely(sl > max_sl)) { 4850 ibdev_err_ratelimited(&hr_dev->ib_dev, 4851 "failed to set SL(%u). Shouldn't be larger than %u.\n", 4852 sl, max_sl); 4853 return false; 4854 } 4855 4856 return true; 4857 } 4858 4859 static int hns_roce_set_sl(struct ib_qp *ibqp, 4860 const struct ib_qp_attr *attr, 4861 struct hns_roce_v2_qp_context *context, 4862 struct hns_roce_v2_qp_context *qpc_mask) 4863 { 4864 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 4865 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 4866 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 4867 struct ib_device *ibdev = &hr_dev->ib_dev; 4868 int ret; 4869 4870 ret = hns_roce_hw_v2_get_dscp(hr_dev, get_tclass(&attr->ah_attr.grh), 4871 &hr_qp->tc_mode, &hr_qp->priority); 4872 if (ret && ret != -EOPNOTSUPP && 4873 grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) { 4874 ibdev_err_ratelimited(ibdev, 4875 "failed to get dscp, ret = %d.\n", ret); 4876 return ret; 4877 } 4878 4879 if (hr_qp->tc_mode == HNAE3_TC_MAP_MODE_DSCP && 4880 grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) 4881 hr_qp->sl = hr_qp->priority; 4882 else 4883 hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr); 4884 4885 if (!check_sl_valid(hr_dev, hr_qp->sl)) 4886 return -EINVAL; 4887 4888 hr_reg_write(context, QPC_SL, hr_qp->sl); 4889 hr_reg_clear(qpc_mask, QPC_SL); 4890 4891 return 0; 4892 } 4893 4894 static int hns_roce_v2_set_path(struct ib_qp *ibqp, 4895 const struct ib_qp_attr *attr, 4896 int attr_mask, 4897 struct hns_roce_v2_qp_context *context, 4898 struct hns_roce_v2_qp_context *qpc_mask) 4899 { 4900 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 4901 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 4902 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 4903 struct ib_device *ibdev = &hr_dev->ib_dev; 4904 const struct ib_gid_attr *gid_attr = NULL; 4905 u8 sl = rdma_ah_get_sl(&attr->ah_attr); 4906 int is_roce_protocol; 4907 u16 vlan_id = 0xffff; 4908 bool is_udp = false; 4909 u8 ib_port; 4910 u8 hr_port; 4911 int ret; 4912 4913 /* 4914 * If free_mr_en of qp is set, it means that this qp comes from 4915 * free mr. This qp will perform the loopback operation. 4916 * In the loopback scenario, only sl needs to be set. 4917 */ 4918 if (hr_qp->free_mr_en) { 4919 if (!check_sl_valid(hr_dev, sl)) 4920 return -EINVAL; 4921 hr_reg_write(context, QPC_SL, sl); 4922 hr_reg_clear(qpc_mask, QPC_SL); 4923 hr_qp->sl = sl; 4924 return 0; 4925 } 4926 4927 ib_port = (attr_mask & IB_QP_PORT) ? attr->port_num : hr_qp->port + 1; 4928 hr_port = ib_port - 1; 4929 is_roce_protocol = rdma_cap_eth_ah(&hr_dev->ib_dev, ib_port) && 4930 rdma_ah_get_ah_flags(&attr->ah_attr) & IB_AH_GRH; 4931 4932 if (is_roce_protocol) { 4933 gid_attr = attr->ah_attr.grh.sgid_attr; 4934 ret = rdma_read_gid_l2_fields(gid_attr, &vlan_id, NULL); 4935 if (ret) 4936 return ret; 4937 4938 is_udp = (gid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP); 4939 } 4940 4941 /* Only HIP08 needs to set the vlan_en bits in QPC */ 4942 if (vlan_id < VLAN_N_VID && 4943 hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) { 4944 hr_reg_enable(context, QPC_RQ_VLAN_EN); 4945 hr_reg_clear(qpc_mask, QPC_RQ_VLAN_EN); 4946 hr_reg_enable(context, QPC_SQ_VLAN_EN); 4947 hr_reg_clear(qpc_mask, QPC_SQ_VLAN_EN); 4948 } 4949 4950 hr_reg_write(context, QPC_VLAN_ID, vlan_id); 4951 hr_reg_clear(qpc_mask, QPC_VLAN_ID); 4952 4953 if (grh->sgid_index >= hr_dev->caps.gid_table_len[hr_port]) { 4954 ibdev_err(ibdev, "sgid_index(%u) too large. max is %d\n", 4955 grh->sgid_index, hr_dev->caps.gid_table_len[hr_port]); 4956 return -EINVAL; 4957 } 4958 4959 if (attr->ah_attr.type != RDMA_AH_ATTR_TYPE_ROCE) { 4960 ibdev_err(ibdev, "ah attr is not RDMA roce type\n"); 4961 return -EINVAL; 4962 } 4963 4964 hr_reg_write(context, QPC_UDPSPN, 4965 is_udp ? rdma_get_udp_sport(grh->flow_label, ibqp->qp_num, 4966 attr->dest_qp_num) : 4967 0); 4968 4969 hr_reg_clear(qpc_mask, QPC_UDPSPN); 4970 4971 hr_reg_write(context, QPC_GMV_IDX, grh->sgid_index); 4972 4973 hr_reg_clear(qpc_mask, QPC_GMV_IDX); 4974 4975 hr_reg_write(context, QPC_HOPLIMIT, grh->hop_limit); 4976 hr_reg_clear(qpc_mask, QPC_HOPLIMIT); 4977 4978 ret = fill_cong_field(ibqp, attr, context, qpc_mask); 4979 if (ret) 4980 return ret; 4981 4982 hr_reg_write(context, QPC_TC, get_tclass(&attr->ah_attr.grh)); 4983 hr_reg_clear(qpc_mask, QPC_TC); 4984 4985 hr_reg_write(context, QPC_FL, grh->flow_label); 4986 hr_reg_clear(qpc_mask, QPC_FL); 4987 memcpy(context->dgid, grh->dgid.raw, sizeof(grh->dgid.raw)); 4988 memset(qpc_mask->dgid, 0, sizeof(grh->dgid.raw)); 4989 4990 return hns_roce_set_sl(ibqp, attr, context, qpc_mask); 4991 } 4992 4993 static bool check_qp_state(enum ib_qp_state cur_state, 4994 enum ib_qp_state new_state) 4995 { 4996 static const bool sm[][IB_QPS_ERR + 1] = { 4997 [IB_QPS_RESET] = { [IB_QPS_RESET] = true, 4998 [IB_QPS_INIT] = true }, 4999 [IB_QPS_INIT] = { [IB_QPS_RESET] = true, 5000 [IB_QPS_INIT] = true, 5001 [IB_QPS_RTR] = true, 5002 [IB_QPS_ERR] = true }, 5003 [IB_QPS_RTR] = { [IB_QPS_RESET] = true, 5004 [IB_QPS_RTS] = true, 5005 [IB_QPS_ERR] = true }, 5006 [IB_QPS_RTS] = { [IB_QPS_RESET] = true, 5007 [IB_QPS_RTS] = true, 5008 [IB_QPS_ERR] = true }, 5009 [IB_QPS_SQD] = {}, 5010 [IB_QPS_SQE] = {}, 5011 [IB_QPS_ERR] = { [IB_QPS_RESET] = true, 5012 [IB_QPS_ERR] = true } 5013 }; 5014 5015 return sm[cur_state][new_state]; 5016 } 5017 5018 static int hns_roce_v2_set_abs_fields(struct ib_qp *ibqp, 5019 const struct ib_qp_attr *attr, 5020 int attr_mask, 5021 enum ib_qp_state cur_state, 5022 enum ib_qp_state new_state, 5023 struct hns_roce_v2_qp_context *context, 5024 struct hns_roce_v2_qp_context *qpc_mask, 5025 struct ib_udata *udata) 5026 { 5027 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 5028 int ret = 0; 5029 5030 if (!check_qp_state(cur_state, new_state)) { 5031 ibdev_err(&hr_dev->ib_dev, "Illegal state for QP!\n"); 5032 return -EINVAL; 5033 } 5034 5035 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { 5036 memset(qpc_mask, 0, hr_dev->caps.qpc_sz); 5037 modify_qp_reset_to_init(ibqp, context, qpc_mask); 5038 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) { 5039 modify_qp_init_to_init(ibqp, context, qpc_mask); 5040 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) { 5041 ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context, 5042 qpc_mask, udata); 5043 } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) { 5044 ret = modify_qp_rtr_to_rts(ibqp, attr_mask, context, qpc_mask); 5045 } 5046 5047 return ret; 5048 } 5049 5050 static bool check_qp_timeout_cfg_range(struct hns_roce_dev *hr_dev, u8 *timeout) 5051 { 5052 #define QP_ACK_TIMEOUT_MAX_HIP08 20 5053 #define QP_ACK_TIMEOUT_MAX 31 5054 5055 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) { 5056 if (*timeout > QP_ACK_TIMEOUT_MAX_HIP08) { 5057 ibdev_warn(&hr_dev->ib_dev, 5058 "local ACK timeout shall be 0 to 20.\n"); 5059 return false; 5060 } 5061 *timeout += HNS_ROCE_V2_QP_ACK_TIMEOUT_OFS_HIP08; 5062 } else if (hr_dev->pci_dev->revision > PCI_REVISION_ID_HIP08) { 5063 if (*timeout > QP_ACK_TIMEOUT_MAX) { 5064 ibdev_warn(&hr_dev->ib_dev, 5065 "local ACK timeout shall be 0 to 31.\n"); 5066 return false; 5067 } 5068 } 5069 5070 return true; 5071 } 5072 5073 static int hns_roce_v2_set_opt_fields(struct ib_qp *ibqp, 5074 const struct ib_qp_attr *attr, 5075 int attr_mask, 5076 struct hns_roce_v2_qp_context *context, 5077 struct hns_roce_v2_qp_context *qpc_mask) 5078 { 5079 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 5080 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 5081 int ret = 0; 5082 u8 timeout; 5083 5084 if (attr_mask & IB_QP_AV) { 5085 ret = hns_roce_v2_set_path(ibqp, attr, attr_mask, context, 5086 qpc_mask); 5087 if (ret) 5088 return ret; 5089 } 5090 5091 if (attr_mask & IB_QP_TIMEOUT) { 5092 timeout = attr->timeout; 5093 if (check_qp_timeout_cfg_range(hr_dev, &timeout)) { 5094 hr_reg_write(context, QPC_AT, timeout); 5095 hr_reg_clear(qpc_mask, QPC_AT); 5096 } 5097 } 5098 5099 if (attr_mask & IB_QP_RETRY_CNT) { 5100 hr_reg_write(context, QPC_RETRY_NUM_INIT, attr->retry_cnt); 5101 hr_reg_clear(qpc_mask, QPC_RETRY_NUM_INIT); 5102 5103 hr_reg_write(context, QPC_RETRY_CNT, attr->retry_cnt); 5104 hr_reg_clear(qpc_mask, QPC_RETRY_CNT); 5105 } 5106 5107 if (attr_mask & IB_QP_RNR_RETRY) { 5108 hr_reg_write(context, QPC_RNR_NUM_INIT, attr->rnr_retry); 5109 hr_reg_clear(qpc_mask, QPC_RNR_NUM_INIT); 5110 5111 hr_reg_write(context, QPC_RNR_CNT, attr->rnr_retry); 5112 hr_reg_clear(qpc_mask, QPC_RNR_CNT); 5113 } 5114 5115 if (attr_mask & IB_QP_SQ_PSN) { 5116 hr_reg_write(context, QPC_SQ_CUR_PSN, attr->sq_psn); 5117 hr_reg_clear(qpc_mask, QPC_SQ_CUR_PSN); 5118 5119 hr_reg_write(context, QPC_SQ_MAX_PSN, attr->sq_psn); 5120 hr_reg_clear(qpc_mask, QPC_SQ_MAX_PSN); 5121 5122 hr_reg_write(context, QPC_RETRY_MSG_PSN_L, attr->sq_psn); 5123 hr_reg_clear(qpc_mask, QPC_RETRY_MSG_PSN_L); 5124 5125 hr_reg_write(context, QPC_RETRY_MSG_PSN_H, 5126 attr->sq_psn >> RETRY_MSG_PSN_SHIFT); 5127 hr_reg_clear(qpc_mask, QPC_RETRY_MSG_PSN_H); 5128 5129 hr_reg_write(context, QPC_RETRY_MSG_FPKT_PSN, attr->sq_psn); 5130 hr_reg_clear(qpc_mask, QPC_RETRY_MSG_FPKT_PSN); 5131 5132 hr_reg_write(context, QPC_RX_ACK_EPSN, attr->sq_psn); 5133 hr_reg_clear(qpc_mask, QPC_RX_ACK_EPSN); 5134 } 5135 5136 if ((attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) && 5137 attr->max_dest_rd_atomic) { 5138 hr_reg_write(context, QPC_RR_MAX, 5139 fls(attr->max_dest_rd_atomic - 1)); 5140 hr_reg_clear(qpc_mask, QPC_RR_MAX); 5141 } 5142 5143 if ((attr_mask & IB_QP_MAX_QP_RD_ATOMIC) && attr->max_rd_atomic) { 5144 hr_reg_write(context, QPC_SR_MAX, fls(attr->max_rd_atomic - 1)); 5145 hr_reg_clear(qpc_mask, QPC_SR_MAX); 5146 } 5147 5148 if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) 5149 set_access_flags(hr_qp, context, qpc_mask, attr, attr_mask); 5150 5151 if (attr_mask & IB_QP_MIN_RNR_TIMER) { 5152 hr_reg_write(context, QPC_MIN_RNR_TIME, 5153 hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 ? 5154 HNS_ROCE_RNR_TIMER_10NS : attr->min_rnr_timer); 5155 hr_reg_clear(qpc_mask, QPC_MIN_RNR_TIME); 5156 } 5157 5158 if (attr_mask & IB_QP_RQ_PSN) { 5159 hr_reg_write(context, QPC_RX_REQ_EPSN, attr->rq_psn); 5160 hr_reg_clear(qpc_mask, QPC_RX_REQ_EPSN); 5161 5162 hr_reg_write(context, QPC_RAQ_PSN, attr->rq_psn - 1); 5163 hr_reg_clear(qpc_mask, QPC_RAQ_PSN); 5164 } 5165 5166 if (attr_mask & IB_QP_QKEY) { 5167 context->qkey_xrcd = cpu_to_le32(attr->qkey); 5168 qpc_mask->qkey_xrcd = 0; 5169 hr_qp->qkey = attr->qkey; 5170 } 5171 5172 return ret; 5173 } 5174 5175 static void hns_roce_v2_record_opt_fields(struct ib_qp *ibqp, 5176 const struct ib_qp_attr *attr, 5177 int attr_mask) 5178 { 5179 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 5180 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 5181 5182 if (attr_mask & IB_QP_ACCESS_FLAGS) 5183 hr_qp->atomic_rd_en = attr->qp_access_flags; 5184 5185 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) 5186 hr_qp->resp_depth = attr->max_dest_rd_atomic; 5187 if (attr_mask & IB_QP_PORT) { 5188 hr_qp->port = attr->port_num - 1; 5189 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; 5190 } 5191 } 5192 5193 static void clear_qp(struct hns_roce_qp *hr_qp) 5194 { 5195 struct ib_qp *ibqp = &hr_qp->ibqp; 5196 5197 if (ibqp->send_cq) 5198 hns_roce_v2_cq_clean(to_hr_cq(ibqp->send_cq), 5199 hr_qp->qpn, NULL); 5200 5201 if (ibqp->recv_cq && ibqp->recv_cq != ibqp->send_cq) 5202 hns_roce_v2_cq_clean(to_hr_cq(ibqp->recv_cq), 5203 hr_qp->qpn, ibqp->srq ? 5204 to_hr_srq(ibqp->srq) : NULL); 5205 5206 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) 5207 *hr_qp->rdb.db_record = 0; 5208 5209 hr_qp->rq.head = 0; 5210 hr_qp->rq.tail = 0; 5211 hr_qp->sq.head = 0; 5212 hr_qp->sq.tail = 0; 5213 hr_qp->next_sge = 0; 5214 } 5215 5216 static void v2_set_flushed_fields(struct ib_qp *ibqp, 5217 struct hns_roce_v2_qp_context *context, 5218 struct hns_roce_v2_qp_context *qpc_mask) 5219 { 5220 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 5221 unsigned long sq_flag = 0; 5222 unsigned long rq_flag = 0; 5223 5224 if (ibqp->qp_type == IB_QPT_XRC_TGT) 5225 return; 5226 5227 spin_lock_irqsave(&hr_qp->sq.lock, sq_flag); 5228 hr_reg_write(context, QPC_SQ_PRODUCER_IDX, hr_qp->sq.head); 5229 hr_reg_clear(qpc_mask, QPC_SQ_PRODUCER_IDX); 5230 hr_qp->state = IB_QPS_ERR; 5231 spin_unlock_irqrestore(&hr_qp->sq.lock, sq_flag); 5232 5233 if (ibqp->srq || ibqp->qp_type == IB_QPT_XRC_INI) /* no RQ */ 5234 return; 5235 5236 spin_lock_irqsave(&hr_qp->rq.lock, rq_flag); 5237 hr_reg_write(context, QPC_RQ_PRODUCER_IDX, hr_qp->rq.head); 5238 hr_reg_clear(qpc_mask, QPC_RQ_PRODUCER_IDX); 5239 spin_unlock_irqrestore(&hr_qp->rq.lock, rq_flag); 5240 } 5241 5242 static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, 5243 const struct ib_qp_attr *attr, 5244 int attr_mask, enum ib_qp_state cur_state, 5245 enum ib_qp_state new_state, struct ib_udata *udata) 5246 { 5247 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 5248 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 5249 struct hns_roce_v2_qp_context ctx[2]; 5250 struct hns_roce_v2_qp_context *context = ctx; 5251 struct hns_roce_v2_qp_context *qpc_mask = ctx + 1; 5252 struct ib_device *ibdev = &hr_dev->ib_dev; 5253 int ret; 5254 5255 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 5256 return -EOPNOTSUPP; 5257 5258 /* 5259 * In v2 engine, software pass context and context mask to hardware 5260 * when modifying qp. If software need modify some fields in context, 5261 * we should set all bits of the relevant fields in context mask to 5262 * 0 at the same time, else set them to 0x1. 5263 */ 5264 memset(context, 0, hr_dev->caps.qpc_sz); 5265 memset(qpc_mask, 0xff, hr_dev->caps.qpc_sz); 5266 5267 ret = hns_roce_v2_set_abs_fields(ibqp, attr, attr_mask, cur_state, 5268 new_state, context, qpc_mask, udata); 5269 if (ret) 5270 goto out; 5271 5272 /* When QP state is err, SQ and RQ WQE should be flushed */ 5273 if (new_state == IB_QPS_ERR) 5274 v2_set_flushed_fields(ibqp, context, qpc_mask); 5275 5276 /* Configure the optional fields */ 5277 ret = hns_roce_v2_set_opt_fields(ibqp, attr, attr_mask, context, 5278 qpc_mask); 5279 if (ret) 5280 goto out; 5281 5282 hr_reg_write_bool(context, QPC_INV_CREDIT, 5283 to_hr_qp_type(hr_qp->ibqp.qp_type) == SERV_TYPE_XRC || 5284 ibqp->srq); 5285 hr_reg_clear(qpc_mask, QPC_INV_CREDIT); 5286 5287 /* Every status migrate must change state */ 5288 hr_reg_write(context, QPC_QP_ST, new_state); 5289 hr_reg_clear(qpc_mask, QPC_QP_ST); 5290 5291 /* SW pass context to HW */ 5292 ret = hns_roce_v2_qp_modify(hr_dev, context, qpc_mask, hr_qp); 5293 if (ret) { 5294 ibdev_err(ibdev, "failed to modify QP, ret = %d.\n", ret); 5295 goto out; 5296 } 5297 5298 hr_qp->state = new_state; 5299 5300 hns_roce_v2_record_opt_fields(ibqp, attr, attr_mask); 5301 5302 if (new_state == IB_QPS_RESET && !ibqp->uobject) 5303 clear_qp(hr_qp); 5304 5305 out: 5306 return ret; 5307 } 5308 5309 static int to_ib_qp_st(enum hns_roce_v2_qp_state state) 5310 { 5311 static const enum ib_qp_state map[] = { 5312 [HNS_ROCE_QP_ST_RST] = IB_QPS_RESET, 5313 [HNS_ROCE_QP_ST_INIT] = IB_QPS_INIT, 5314 [HNS_ROCE_QP_ST_RTR] = IB_QPS_RTR, 5315 [HNS_ROCE_QP_ST_RTS] = IB_QPS_RTS, 5316 [HNS_ROCE_QP_ST_SQD] = IB_QPS_SQD, 5317 [HNS_ROCE_QP_ST_SQER] = IB_QPS_SQE, 5318 [HNS_ROCE_QP_ST_ERR] = IB_QPS_ERR, 5319 [HNS_ROCE_QP_ST_SQ_DRAINING] = IB_QPS_SQD 5320 }; 5321 5322 return (state < ARRAY_SIZE(map)) ? map[state] : -1; 5323 } 5324 5325 static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev, u32 qpn, 5326 void *buffer) 5327 { 5328 struct hns_roce_cmd_mailbox *mailbox; 5329 int ret; 5330 5331 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 5332 if (IS_ERR(mailbox)) 5333 return PTR_ERR(mailbox); 5334 5335 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_QPC, 5336 qpn); 5337 if (ret) 5338 goto out; 5339 5340 memcpy(buffer, mailbox->buf, hr_dev->caps.qpc_sz); 5341 5342 out: 5343 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 5344 return ret; 5345 } 5346 5347 static int hns_roce_v2_query_srqc(struct hns_roce_dev *hr_dev, u32 srqn, 5348 void *buffer) 5349 { 5350 struct hns_roce_srq_context *context; 5351 struct hns_roce_cmd_mailbox *mailbox; 5352 int ret; 5353 5354 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 5355 if (IS_ERR(mailbox)) 5356 return PTR_ERR(mailbox); 5357 5358 context = mailbox->buf; 5359 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_SRQC, 5360 srqn); 5361 if (ret) 5362 goto out; 5363 5364 memcpy(buffer, context, sizeof(*context)); 5365 5366 out: 5367 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 5368 return ret; 5369 } 5370 5371 static int hns_roce_v2_query_sccc(struct hns_roce_dev *hr_dev, u32 qpn, 5372 void *buffer) 5373 { 5374 struct hns_roce_v2_scc_context *context; 5375 struct hns_roce_cmd_mailbox *mailbox; 5376 int ret; 5377 5378 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 5379 if (IS_ERR(mailbox)) 5380 return PTR_ERR(mailbox); 5381 5382 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_SCCC, 5383 qpn); 5384 if (ret) 5385 goto out; 5386 5387 context = mailbox->buf; 5388 memcpy(buffer, context, sizeof(*context)); 5389 5390 out: 5391 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 5392 return ret; 5393 } 5394 5395 static u8 get_qp_timeout_attr(struct hns_roce_dev *hr_dev, 5396 struct hns_roce_v2_qp_context *context) 5397 { 5398 u8 timeout; 5399 5400 timeout = (u8)hr_reg_read(context, QPC_AT); 5401 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) 5402 timeout -= HNS_ROCE_V2_QP_ACK_TIMEOUT_OFS_HIP08; 5403 5404 return timeout; 5405 } 5406 5407 static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, 5408 int qp_attr_mask, 5409 struct ib_qp_init_attr *qp_init_attr) 5410 { 5411 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 5412 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 5413 struct hns_roce_v2_qp_context context = {}; 5414 struct ib_device *ibdev = &hr_dev->ib_dev; 5415 int tmp_qp_state; 5416 int state; 5417 int ret; 5418 5419 memset(qp_attr, 0, sizeof(*qp_attr)); 5420 memset(qp_init_attr, 0, sizeof(*qp_init_attr)); 5421 5422 mutex_lock(&hr_qp->mutex); 5423 5424 if (hr_qp->state == IB_QPS_RESET) { 5425 qp_attr->qp_state = IB_QPS_RESET; 5426 ret = 0; 5427 goto done; 5428 } 5429 5430 ret = hns_roce_v2_query_qpc(hr_dev, hr_qp->qpn, &context); 5431 if (ret) { 5432 ibdev_err(ibdev, "failed to query QPC, ret = %d.\n", ret); 5433 ret = -EINVAL; 5434 goto out; 5435 } 5436 5437 state = hr_reg_read(&context, QPC_QP_ST); 5438 tmp_qp_state = to_ib_qp_st((enum hns_roce_v2_qp_state)state); 5439 if (tmp_qp_state == -1) { 5440 ibdev_err(ibdev, "Illegal ib_qp_state\n"); 5441 ret = -EINVAL; 5442 goto out; 5443 } 5444 hr_qp->state = (u8)tmp_qp_state; 5445 qp_attr->qp_state = (enum ib_qp_state)hr_qp->state; 5446 qp_attr->path_mtu = (enum ib_mtu)hr_reg_read(&context, QPC_MTU); 5447 qp_attr->path_mig_state = IB_MIG_ARMED; 5448 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 5449 if (hr_qp->ibqp.qp_type == IB_QPT_UD) 5450 qp_attr->qkey = le32_to_cpu(context.qkey_xrcd); 5451 5452 qp_attr->rq_psn = hr_reg_read(&context, QPC_RX_REQ_EPSN); 5453 qp_attr->sq_psn = (u32)hr_reg_read(&context, QPC_SQ_CUR_PSN); 5454 qp_attr->dest_qp_num = hr_reg_read(&context, QPC_DQPN); 5455 qp_attr->qp_access_flags = 5456 ((hr_reg_read(&context, QPC_RRE)) << V2_QP_RRE_S) | 5457 ((hr_reg_read(&context, QPC_RWE)) << V2_QP_RWE_S) | 5458 ((hr_reg_read(&context, QPC_ATE)) << V2_QP_ATE_S); 5459 5460 if (hr_qp->ibqp.qp_type == IB_QPT_RC || 5461 hr_qp->ibqp.qp_type == IB_QPT_XRC_INI || 5462 hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT) { 5463 struct ib_global_route *grh = 5464 rdma_ah_retrieve_grh(&qp_attr->ah_attr); 5465 5466 rdma_ah_set_sl(&qp_attr->ah_attr, 5467 hr_reg_read(&context, QPC_SL)); 5468 rdma_ah_set_port_num(&qp_attr->ah_attr, hr_qp->port + 1); 5469 rdma_ah_set_ah_flags(&qp_attr->ah_attr, IB_AH_GRH); 5470 grh->flow_label = hr_reg_read(&context, QPC_FL); 5471 grh->sgid_index = hr_reg_read(&context, QPC_GMV_IDX); 5472 grh->hop_limit = hr_reg_read(&context, QPC_HOPLIMIT); 5473 grh->traffic_class = hr_reg_read(&context, QPC_TC); 5474 5475 memcpy(grh->dgid.raw, context.dgid, sizeof(grh->dgid.raw)); 5476 } 5477 5478 qp_attr->port_num = hr_qp->port + 1; 5479 qp_attr->sq_draining = 0; 5480 qp_attr->max_rd_atomic = 1 << hr_reg_read(&context, QPC_SR_MAX); 5481 qp_attr->max_dest_rd_atomic = 1 << hr_reg_read(&context, QPC_RR_MAX); 5482 5483 qp_attr->min_rnr_timer = (u8)hr_reg_read(&context, QPC_MIN_RNR_TIME); 5484 qp_attr->timeout = get_qp_timeout_attr(hr_dev, &context); 5485 qp_attr->retry_cnt = hr_reg_read(&context, QPC_RETRY_NUM_INIT); 5486 qp_attr->rnr_retry = hr_reg_read(&context, QPC_RNR_NUM_INIT); 5487 5488 done: 5489 qp_attr->cur_qp_state = qp_attr->qp_state; 5490 qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt; 5491 qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge; 5492 qp_attr->cap.max_inline_data = hr_qp->max_inline_data; 5493 5494 qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt; 5495 qp_attr->cap.max_send_sge = hr_qp->sq.max_gs; 5496 5497 qp_init_attr->qp_context = ibqp->qp_context; 5498 qp_init_attr->qp_type = ibqp->qp_type; 5499 qp_init_attr->recv_cq = ibqp->recv_cq; 5500 qp_init_attr->send_cq = ibqp->send_cq; 5501 qp_init_attr->srq = ibqp->srq; 5502 qp_init_attr->cap = qp_attr->cap; 5503 qp_init_attr->sq_sig_type = hr_qp->sq_signal_bits; 5504 5505 out: 5506 mutex_unlock(&hr_qp->mutex); 5507 return ret; 5508 } 5509 5510 static inline int modify_qp_is_ok(struct hns_roce_qp *hr_qp) 5511 { 5512 return ((hr_qp->ibqp.qp_type == IB_QPT_RC || 5513 hr_qp->ibqp.qp_type == IB_QPT_UD || 5514 hr_qp->ibqp.qp_type == IB_QPT_XRC_INI || 5515 hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT) && 5516 hr_qp->state != IB_QPS_RESET); 5517 } 5518 5519 static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, 5520 struct hns_roce_qp *hr_qp, 5521 struct ib_udata *udata) 5522 { 5523 struct ib_device *ibdev = &hr_dev->ib_dev; 5524 struct hns_roce_cq *send_cq, *recv_cq; 5525 unsigned long flags; 5526 int ret = 0; 5527 5528 if (modify_qp_is_ok(hr_qp)) { 5529 /* Modify qp to reset before destroying qp */ 5530 ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0, 5531 hr_qp->state, IB_QPS_RESET, udata); 5532 if (ret) 5533 ibdev_err(ibdev, 5534 "failed to modify QP to RST, ret = %d.\n", 5535 ret); 5536 } 5537 5538 send_cq = hr_qp->ibqp.send_cq ? to_hr_cq(hr_qp->ibqp.send_cq) : NULL; 5539 recv_cq = hr_qp->ibqp.recv_cq ? to_hr_cq(hr_qp->ibqp.recv_cq) : NULL; 5540 5541 spin_lock_irqsave(&hr_dev->qp_list_lock, flags); 5542 hns_roce_lock_cqs(send_cq, recv_cq); 5543 5544 if (!udata) { 5545 if (recv_cq) 5546 __hns_roce_v2_cq_clean(recv_cq, hr_qp->qpn, 5547 (hr_qp->ibqp.srq ? 5548 to_hr_srq(hr_qp->ibqp.srq) : 5549 NULL)); 5550 5551 if (send_cq && send_cq != recv_cq) 5552 __hns_roce_v2_cq_clean(send_cq, hr_qp->qpn, NULL); 5553 } 5554 5555 hns_roce_qp_remove(hr_dev, hr_qp); 5556 5557 hns_roce_unlock_cqs(send_cq, recv_cq); 5558 spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags); 5559 5560 return ret; 5561 } 5562 5563 int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 5564 { 5565 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 5566 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 5567 int ret; 5568 5569 ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata); 5570 if (ret) 5571 ibdev_err(&hr_dev->ib_dev, 5572 "failed to destroy QP, QPN = 0x%06lx, ret = %d.\n", 5573 hr_qp->qpn, ret); 5574 5575 hns_roce_qp_destroy(hr_dev, hr_qp, udata); 5576 5577 return 0; 5578 } 5579 5580 static int hns_roce_v2_qp_flow_control_init(struct hns_roce_dev *hr_dev, 5581 struct hns_roce_qp *hr_qp) 5582 { 5583 struct ib_device *ibdev = &hr_dev->ib_dev; 5584 struct hns_roce_sccc_clr_done *resp; 5585 struct hns_roce_sccc_clr *clr; 5586 struct hns_roce_cmq_desc desc; 5587 int ret, i; 5588 5589 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) 5590 return 0; 5591 5592 mutex_lock(&hr_dev->qp_table.scc_mutex); 5593 5594 /* set scc ctx clear done flag */ 5595 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_RESET_SCCC, false); 5596 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 5597 if (ret) { 5598 ibdev_err(ibdev, "failed to reset SCC ctx, ret = %d.\n", ret); 5599 goto out; 5600 } 5601 5602 /* clear scc context */ 5603 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CLR_SCCC, false); 5604 clr = (struct hns_roce_sccc_clr *)desc.data; 5605 clr->qpn = cpu_to_le32(hr_qp->qpn); 5606 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 5607 if (ret) { 5608 ibdev_err(ibdev, "failed to clear SCC ctx, ret = %d.\n", ret); 5609 goto out; 5610 } 5611 5612 /* query scc context clear is done or not */ 5613 resp = (struct hns_roce_sccc_clr_done *)desc.data; 5614 for (i = 0; i <= HNS_ROCE_CMQ_SCC_CLR_DONE_CNT; i++) { 5615 hns_roce_cmq_setup_basic_desc(&desc, 5616 HNS_ROCE_OPC_QUERY_SCCC, true); 5617 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 5618 if (ret) { 5619 ibdev_err(ibdev, "failed to query clr cmq, ret = %d\n", 5620 ret); 5621 goto out; 5622 } 5623 5624 if (resp->clr_done) 5625 goto out; 5626 5627 msleep(20); 5628 } 5629 5630 ibdev_err(ibdev, "query SCC clr done flag overtime.\n"); 5631 ret = -ETIMEDOUT; 5632 5633 out: 5634 mutex_unlock(&hr_dev->qp_table.scc_mutex); 5635 return ret; 5636 } 5637 5638 #define DMA_IDX_SHIFT 3 5639 #define DMA_WQE_SHIFT 3 5640 5641 static int hns_roce_v2_write_srqc_index_queue(struct hns_roce_srq *srq, 5642 struct hns_roce_srq_context *ctx) 5643 { 5644 struct hns_roce_idx_que *idx_que = &srq->idx_que; 5645 struct ib_device *ibdev = srq->ibsrq.device; 5646 struct hns_roce_dev *hr_dev = to_hr_dev(ibdev); 5647 u64 mtts_idx[MTT_MIN_COUNT] = {}; 5648 dma_addr_t dma_handle_idx; 5649 int ret; 5650 5651 /* Get physical address of idx que buf */ 5652 ret = hns_roce_mtr_find(hr_dev, &idx_que->mtr, 0, mtts_idx, 5653 ARRAY_SIZE(mtts_idx)); 5654 if (ret) { 5655 ibdev_err(ibdev, "failed to find mtr for SRQ idx, ret = %d.\n", 5656 ret); 5657 return ret; 5658 } 5659 5660 dma_handle_idx = hns_roce_get_mtr_ba(&idx_que->mtr); 5661 5662 hr_reg_write(ctx, SRQC_IDX_HOP_NUM, 5663 to_hr_hem_hopnum(hr_dev->caps.idx_hop_num, srq->wqe_cnt)); 5664 5665 hr_reg_write(ctx, SRQC_IDX_BT_BA_L, dma_handle_idx >> DMA_IDX_SHIFT); 5666 hr_reg_write(ctx, SRQC_IDX_BT_BA_H, 5667 upper_32_bits(dma_handle_idx >> DMA_IDX_SHIFT)); 5668 5669 hr_reg_write(ctx, SRQC_IDX_BA_PG_SZ, 5670 to_hr_hw_page_shift(idx_que->mtr.hem_cfg.ba_pg_shift)); 5671 hr_reg_write(ctx, SRQC_IDX_BUF_PG_SZ, 5672 to_hr_hw_page_shift(idx_que->mtr.hem_cfg.buf_pg_shift)); 5673 5674 hr_reg_write(ctx, SRQC_IDX_CUR_BLK_ADDR_L, 5675 to_hr_hw_page_addr(mtts_idx[0])); 5676 hr_reg_write(ctx, SRQC_IDX_CUR_BLK_ADDR_H, 5677 upper_32_bits(to_hr_hw_page_addr(mtts_idx[0]))); 5678 5679 hr_reg_write(ctx, SRQC_IDX_NXT_BLK_ADDR_L, 5680 to_hr_hw_page_addr(mtts_idx[1])); 5681 hr_reg_write(ctx, SRQC_IDX_NXT_BLK_ADDR_H, 5682 upper_32_bits(to_hr_hw_page_addr(mtts_idx[1]))); 5683 5684 return 0; 5685 } 5686 5687 static int hns_roce_v2_write_srqc(struct hns_roce_srq *srq, void *mb_buf) 5688 { 5689 struct ib_device *ibdev = srq->ibsrq.device; 5690 struct hns_roce_dev *hr_dev = to_hr_dev(ibdev); 5691 struct hns_roce_srq_context *ctx = mb_buf; 5692 u64 mtts_wqe[MTT_MIN_COUNT] = {}; 5693 dma_addr_t dma_handle_wqe; 5694 int ret; 5695 5696 memset(ctx, 0, sizeof(*ctx)); 5697 5698 /* Get the physical address of srq buf */ 5699 ret = hns_roce_mtr_find(hr_dev, &srq->buf_mtr, 0, mtts_wqe, 5700 ARRAY_SIZE(mtts_wqe)); 5701 if (ret) { 5702 ibdev_err(ibdev, "failed to find mtr for SRQ WQE, ret = %d.\n", 5703 ret); 5704 return ret; 5705 } 5706 5707 dma_handle_wqe = hns_roce_get_mtr_ba(&srq->buf_mtr); 5708 5709 hr_reg_write(ctx, SRQC_SRQ_ST, 1); 5710 hr_reg_write_bool(ctx, SRQC_SRQ_TYPE, 5711 srq->ibsrq.srq_type == IB_SRQT_XRC); 5712 hr_reg_write(ctx, SRQC_PD, to_hr_pd(srq->ibsrq.pd)->pdn); 5713 hr_reg_write(ctx, SRQC_SRQN, srq->srqn); 5714 hr_reg_write(ctx, SRQC_XRCD, srq->xrcdn); 5715 hr_reg_write(ctx, SRQC_XRC_CQN, srq->cqn); 5716 hr_reg_write(ctx, SRQC_SHIFT, ilog2(srq->wqe_cnt)); 5717 hr_reg_write(ctx, SRQC_RQWS, 5718 srq->max_gs <= 0 ? 0 : fls(srq->max_gs - 1)); 5719 5720 hr_reg_write(ctx, SRQC_WQE_HOP_NUM, 5721 to_hr_hem_hopnum(hr_dev->caps.srqwqe_hop_num, 5722 srq->wqe_cnt)); 5723 5724 hr_reg_write(ctx, SRQC_WQE_BT_BA_L, dma_handle_wqe >> DMA_WQE_SHIFT); 5725 hr_reg_write(ctx, SRQC_WQE_BT_BA_H, 5726 upper_32_bits(dma_handle_wqe >> DMA_WQE_SHIFT)); 5727 5728 hr_reg_write(ctx, SRQC_WQE_BA_PG_SZ, 5729 to_hr_hw_page_shift(srq->buf_mtr.hem_cfg.ba_pg_shift)); 5730 hr_reg_write(ctx, SRQC_WQE_BUF_PG_SZ, 5731 to_hr_hw_page_shift(srq->buf_mtr.hem_cfg.buf_pg_shift)); 5732 5733 if (srq->cap_flags & HNS_ROCE_SRQ_CAP_RECORD_DB) { 5734 hr_reg_enable(ctx, SRQC_DB_RECORD_EN); 5735 hr_reg_write(ctx, SRQC_DB_RECORD_ADDR_L, 5736 lower_32_bits(srq->rdb.dma) >> 1); 5737 hr_reg_write(ctx, SRQC_DB_RECORD_ADDR_H, 5738 upper_32_bits(srq->rdb.dma)); 5739 } 5740 5741 return hns_roce_v2_write_srqc_index_queue(srq, ctx); 5742 } 5743 5744 static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq, 5745 struct ib_srq_attr *srq_attr, 5746 enum ib_srq_attr_mask srq_attr_mask, 5747 struct ib_udata *udata) 5748 { 5749 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device); 5750 struct hns_roce_srq *srq = to_hr_srq(ibsrq); 5751 struct hns_roce_srq_context *srq_context; 5752 struct hns_roce_srq_context *srqc_mask; 5753 struct hns_roce_cmd_mailbox *mailbox; 5754 int ret = 0; 5755 5756 /* Resizing SRQs is not supported yet */ 5757 if (srq_attr_mask & IB_SRQ_MAX_WR) { 5758 ret = -EOPNOTSUPP; 5759 goto out; 5760 } 5761 5762 if (srq_attr_mask & IB_SRQ_LIMIT) { 5763 if (srq_attr->srq_limit > srq->wqe_cnt) { 5764 ret = -EINVAL; 5765 goto out; 5766 } 5767 5768 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 5769 if (IS_ERR(mailbox)) { 5770 ret = PTR_ERR(mailbox); 5771 goto out; 5772 } 5773 5774 srq_context = mailbox->buf; 5775 srqc_mask = (struct hns_roce_srq_context *)mailbox->buf + 1; 5776 5777 memset(srqc_mask, 0xff, sizeof(*srqc_mask)); 5778 5779 hr_reg_write(srq_context, SRQC_LIMIT_WL, srq_attr->srq_limit); 5780 hr_reg_clear(srqc_mask, SRQC_LIMIT_WL); 5781 5782 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, 5783 HNS_ROCE_CMD_MODIFY_SRQC, srq->srqn); 5784 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 5785 if (ret) 5786 ibdev_err(&hr_dev->ib_dev, 5787 "failed to handle cmd of modifying SRQ, ret = %d.\n", 5788 ret); 5789 } 5790 5791 out: 5792 if (ret) 5793 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_SRQ_MODIFY_ERR_CNT]); 5794 5795 return ret; 5796 } 5797 5798 static int hns_roce_v2_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr) 5799 { 5800 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device); 5801 struct hns_roce_srq *srq = to_hr_srq(ibsrq); 5802 struct hns_roce_srq_context *srq_context; 5803 struct hns_roce_cmd_mailbox *mailbox; 5804 int ret; 5805 5806 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 5807 if (IS_ERR(mailbox)) 5808 return PTR_ERR(mailbox); 5809 5810 srq_context = mailbox->buf; 5811 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, 5812 HNS_ROCE_CMD_QUERY_SRQC, srq->srqn); 5813 if (ret) { 5814 ibdev_err(&hr_dev->ib_dev, 5815 "failed to process cmd of querying SRQ, ret = %d.\n", 5816 ret); 5817 goto out; 5818 } 5819 5820 attr->srq_limit = hr_reg_read(srq_context, SRQC_LIMIT_WL); 5821 attr->max_wr = srq->wqe_cnt; 5822 attr->max_sge = srq->max_gs - srq->rsv_sge; 5823 5824 out: 5825 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 5826 return ret; 5827 } 5828 5829 static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) 5830 { 5831 struct hns_roce_dev *hr_dev = to_hr_dev(cq->device); 5832 struct hns_roce_v2_cq_context *cq_context; 5833 struct hns_roce_cq *hr_cq = to_hr_cq(cq); 5834 struct hns_roce_v2_cq_context *cqc_mask; 5835 struct hns_roce_cmd_mailbox *mailbox; 5836 int ret; 5837 5838 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 5839 ret = PTR_ERR_OR_ZERO(mailbox); 5840 if (ret) 5841 goto err_out; 5842 5843 cq_context = mailbox->buf; 5844 cqc_mask = (struct hns_roce_v2_cq_context *)mailbox->buf + 1; 5845 5846 memset(cqc_mask, 0xff, sizeof(*cqc_mask)); 5847 5848 hr_reg_write(cq_context, CQC_CQ_MAX_CNT, cq_count); 5849 hr_reg_clear(cqc_mask, CQC_CQ_MAX_CNT); 5850 5851 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) { 5852 if (cq_period * HNS_ROCE_CLOCK_ADJUST > USHRT_MAX) { 5853 dev_info(hr_dev->dev, 5854 "cq_period(%u) reached the upper limit, adjusted to 65.\n", 5855 cq_period); 5856 cq_period = HNS_ROCE_MAX_CQ_PERIOD_HIP08; 5857 } 5858 cq_period *= HNS_ROCE_CLOCK_ADJUST; 5859 } 5860 hr_reg_write(cq_context, CQC_CQ_PERIOD, cq_period); 5861 hr_reg_clear(cqc_mask, CQC_CQ_PERIOD); 5862 5863 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, 5864 HNS_ROCE_CMD_MODIFY_CQC, hr_cq->cqn); 5865 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 5866 if (ret) 5867 ibdev_err(&hr_dev->ib_dev, 5868 "failed to process cmd when modifying CQ, ret = %d.\n", 5869 ret); 5870 5871 err_out: 5872 if (ret) 5873 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_CQ_MODIFY_ERR_CNT]); 5874 5875 return ret; 5876 } 5877 5878 static int hns_roce_v2_query_cqc(struct hns_roce_dev *hr_dev, u32 cqn, 5879 void *buffer) 5880 { 5881 struct hns_roce_v2_cq_context *context; 5882 struct hns_roce_cmd_mailbox *mailbox; 5883 int ret; 5884 5885 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 5886 if (IS_ERR(mailbox)) 5887 return PTR_ERR(mailbox); 5888 5889 context = mailbox->buf; 5890 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, 5891 HNS_ROCE_CMD_QUERY_CQC, cqn); 5892 if (ret) { 5893 ibdev_err(&hr_dev->ib_dev, 5894 "failed to process cmd when querying CQ, ret = %d.\n", 5895 ret); 5896 goto err_mailbox; 5897 } 5898 5899 memcpy(buffer, context, sizeof(*context)); 5900 5901 err_mailbox: 5902 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 5903 5904 return ret; 5905 } 5906 5907 static int hns_roce_v2_query_mpt(struct hns_roce_dev *hr_dev, u32 key, 5908 void *buffer) 5909 { 5910 struct hns_roce_v2_mpt_entry *context; 5911 struct hns_roce_cmd_mailbox *mailbox; 5912 int ret; 5913 5914 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 5915 if (IS_ERR(mailbox)) 5916 return PTR_ERR(mailbox); 5917 5918 context = mailbox->buf; 5919 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_MPT, 5920 key_to_hw_index(key)); 5921 if (ret) { 5922 ibdev_err(&hr_dev->ib_dev, 5923 "failed to process cmd when querying MPT, ret = %d.\n", 5924 ret); 5925 goto err_mailbox; 5926 } 5927 5928 memcpy(buffer, context, sizeof(*context)); 5929 5930 err_mailbox: 5931 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 5932 5933 return ret; 5934 } 5935 5936 static void hns_roce_irq_work_handle(struct work_struct *work) 5937 { 5938 struct hns_roce_work *irq_work = 5939 container_of(work, struct hns_roce_work, work); 5940 struct ib_device *ibdev = &irq_work->hr_dev->ib_dev; 5941 5942 switch (irq_work->event_type) { 5943 case HNS_ROCE_EVENT_TYPE_PATH_MIG: 5944 ibdev_info(ibdev, "path migrated succeeded.\n"); 5945 break; 5946 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED: 5947 ibdev_warn(ibdev, "path migration failed.\n"); 5948 break; 5949 case HNS_ROCE_EVENT_TYPE_COMM_EST: 5950 break; 5951 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: 5952 ibdev_dbg(ibdev, "send queue drained.\n"); 5953 break; 5954 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: 5955 ibdev_err(ibdev, "local work queue 0x%x catast error, sub_event type is: %d\n", 5956 irq_work->queue_num, irq_work->sub_type); 5957 break; 5958 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: 5959 ibdev_err(ibdev, "invalid request local work queue 0x%x error.\n", 5960 irq_work->queue_num); 5961 break; 5962 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: 5963 ibdev_err(ibdev, "local access violation work queue 0x%x error, sub_event type is: %d\n", 5964 irq_work->queue_num, irq_work->sub_type); 5965 break; 5966 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH: 5967 ibdev_dbg(ibdev, "SRQ limit reach.\n"); 5968 break; 5969 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH: 5970 ibdev_dbg(ibdev, "SRQ last wqe reach.\n"); 5971 break; 5972 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR: 5973 ibdev_err(ibdev, "SRQ catas error.\n"); 5974 break; 5975 case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR: 5976 ibdev_err(ibdev, "CQ 0x%x access err.\n", irq_work->queue_num); 5977 break; 5978 case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW: 5979 ibdev_warn(ibdev, "CQ 0x%x overflow\n", irq_work->queue_num); 5980 break; 5981 case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW: 5982 ibdev_warn(ibdev, "DB overflow.\n"); 5983 break; 5984 case HNS_ROCE_EVENT_TYPE_FLR: 5985 ibdev_warn(ibdev, "function level reset.\n"); 5986 break; 5987 case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION: 5988 ibdev_err(ibdev, "xrc domain violation error.\n"); 5989 break; 5990 case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH: 5991 ibdev_err(ibdev, "invalid xrceth error.\n"); 5992 break; 5993 default: 5994 break; 5995 } 5996 5997 kfree(irq_work); 5998 } 5999 6000 static void hns_roce_v2_init_irq_work(struct hns_roce_dev *hr_dev, 6001 struct hns_roce_eq *eq, u32 queue_num) 6002 { 6003 struct hns_roce_work *irq_work; 6004 6005 irq_work = kzalloc(sizeof(struct hns_roce_work), GFP_ATOMIC); 6006 if (!irq_work) 6007 return; 6008 6009 INIT_WORK(&irq_work->work, hns_roce_irq_work_handle); 6010 irq_work->hr_dev = hr_dev; 6011 irq_work->event_type = eq->event_type; 6012 irq_work->sub_type = eq->sub_type; 6013 irq_work->queue_num = queue_num; 6014 queue_work(hr_dev->irq_workq, &irq_work->work); 6015 } 6016 6017 static void update_eq_db(struct hns_roce_eq *eq) 6018 { 6019 struct hns_roce_dev *hr_dev = eq->hr_dev; 6020 struct hns_roce_v2_db eq_db = {}; 6021 6022 if (eq->type_flag == HNS_ROCE_AEQ) { 6023 hr_reg_write(&eq_db, EQ_DB_CMD, 6024 eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ? 6025 HNS_ROCE_EQ_DB_CMD_AEQ : 6026 HNS_ROCE_EQ_DB_CMD_AEQ_ARMED); 6027 } else { 6028 hr_reg_write(&eq_db, EQ_DB_TAG, eq->eqn); 6029 6030 hr_reg_write(&eq_db, EQ_DB_CMD, 6031 eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ? 6032 HNS_ROCE_EQ_DB_CMD_CEQ : 6033 HNS_ROCE_EQ_DB_CMD_CEQ_ARMED); 6034 } 6035 6036 hr_reg_write(&eq_db, EQ_DB_CI, eq->cons_index); 6037 6038 hns_roce_write64(hr_dev, (__le32 *)&eq_db, eq->db_reg); 6039 } 6040 6041 static struct hns_roce_aeqe *next_aeqe_sw_v2(struct hns_roce_eq *eq) 6042 { 6043 struct hns_roce_aeqe *aeqe; 6044 6045 aeqe = hns_roce_buf_offset(eq->mtr.kmem, 6046 (eq->cons_index & (eq->entries - 1)) * 6047 eq->eqe_size); 6048 6049 return (hr_reg_read(aeqe, AEQE_OWNER) ^ 6050 !!(eq->cons_index & eq->entries)) ? aeqe : NULL; 6051 } 6052 6053 static irqreturn_t hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev, 6054 struct hns_roce_eq *eq) 6055 { 6056 struct device *dev = hr_dev->dev; 6057 struct hns_roce_aeqe *aeqe = next_aeqe_sw_v2(eq); 6058 irqreturn_t aeqe_found = IRQ_NONE; 6059 int event_type; 6060 u32 queue_num; 6061 int sub_type; 6062 6063 while (aeqe) { 6064 /* Make sure we read AEQ entry after we have checked the 6065 * ownership bit 6066 */ 6067 dma_rmb(); 6068 6069 event_type = hr_reg_read(aeqe, AEQE_EVENT_TYPE); 6070 sub_type = hr_reg_read(aeqe, AEQE_SUB_TYPE); 6071 queue_num = hr_reg_read(aeqe, AEQE_EVENT_QUEUE_NUM); 6072 6073 switch (event_type) { 6074 case HNS_ROCE_EVENT_TYPE_PATH_MIG: 6075 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED: 6076 case HNS_ROCE_EVENT_TYPE_COMM_EST: 6077 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: 6078 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: 6079 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH: 6080 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: 6081 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: 6082 case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION: 6083 case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH: 6084 hns_roce_qp_event(hr_dev, queue_num, event_type); 6085 break; 6086 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH: 6087 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR: 6088 hns_roce_srq_event(hr_dev, queue_num, event_type); 6089 break; 6090 case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR: 6091 case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW: 6092 hns_roce_cq_event(hr_dev, queue_num, event_type); 6093 break; 6094 case HNS_ROCE_EVENT_TYPE_MB: 6095 hns_roce_cmd_event(hr_dev, 6096 le16_to_cpu(aeqe->event.cmd.token), 6097 aeqe->event.cmd.status, 6098 le64_to_cpu(aeqe->event.cmd.out_param)); 6099 break; 6100 case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW: 6101 case HNS_ROCE_EVENT_TYPE_FLR: 6102 break; 6103 default: 6104 dev_err(dev, "unhandled event %d on EQ %d at idx %u.\n", 6105 event_type, eq->eqn, eq->cons_index); 6106 break; 6107 } 6108 6109 eq->event_type = event_type; 6110 eq->sub_type = sub_type; 6111 ++eq->cons_index; 6112 aeqe_found = IRQ_HANDLED; 6113 6114 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_AEQE_CNT]); 6115 6116 hns_roce_v2_init_irq_work(hr_dev, eq, queue_num); 6117 6118 aeqe = next_aeqe_sw_v2(eq); 6119 } 6120 6121 update_eq_db(eq); 6122 6123 return IRQ_RETVAL(aeqe_found); 6124 } 6125 6126 static struct hns_roce_ceqe *next_ceqe_sw_v2(struct hns_roce_eq *eq) 6127 { 6128 struct hns_roce_ceqe *ceqe; 6129 6130 ceqe = hns_roce_buf_offset(eq->mtr.kmem, 6131 (eq->cons_index & (eq->entries - 1)) * 6132 eq->eqe_size); 6133 6134 return (hr_reg_read(ceqe, CEQE_OWNER) ^ 6135 !!(eq->cons_index & eq->entries)) ? ceqe : NULL; 6136 } 6137 6138 static irqreturn_t hns_roce_v2_ceq_int(struct hns_roce_dev *hr_dev, 6139 struct hns_roce_eq *eq) 6140 { 6141 struct hns_roce_ceqe *ceqe = next_ceqe_sw_v2(eq); 6142 irqreturn_t ceqe_found = IRQ_NONE; 6143 u32 cqn; 6144 6145 while (ceqe) { 6146 /* Make sure we read CEQ entry after we have checked the 6147 * ownership bit 6148 */ 6149 dma_rmb(); 6150 6151 cqn = hr_reg_read(ceqe, CEQE_CQN); 6152 6153 hns_roce_cq_completion(hr_dev, cqn); 6154 6155 ++eq->cons_index; 6156 ceqe_found = IRQ_HANDLED; 6157 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_CEQE_CNT]); 6158 6159 ceqe = next_ceqe_sw_v2(eq); 6160 } 6161 6162 update_eq_db(eq); 6163 6164 return IRQ_RETVAL(ceqe_found); 6165 } 6166 6167 static irqreturn_t hns_roce_v2_msix_interrupt_eq(int irq, void *eq_ptr) 6168 { 6169 struct hns_roce_eq *eq = eq_ptr; 6170 struct hns_roce_dev *hr_dev = eq->hr_dev; 6171 irqreturn_t int_work; 6172 6173 if (eq->type_flag == HNS_ROCE_CEQ) 6174 /* Completion event interrupt */ 6175 int_work = hns_roce_v2_ceq_int(hr_dev, eq); 6176 else 6177 /* Asynchronous event interrupt */ 6178 int_work = hns_roce_v2_aeq_int(hr_dev, eq); 6179 6180 return IRQ_RETVAL(int_work); 6181 } 6182 6183 static irqreturn_t abnormal_interrupt_basic(struct hns_roce_dev *hr_dev, 6184 u32 int_st) 6185 { 6186 struct pci_dev *pdev = hr_dev->pci_dev; 6187 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev); 6188 const struct hnae3_ae_ops *ops = ae_dev->ops; 6189 irqreturn_t int_work = IRQ_NONE; 6190 u32 int_en; 6191 6192 int_en = roce_read(hr_dev, ROCEE_VF_ABN_INT_EN_REG); 6193 6194 if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S)) { 6195 dev_err(hr_dev->dev, "AEQ overflow!\n"); 6196 6197 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, 6198 1 << HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S); 6199 6200 /* Set reset level for reset_event() */ 6201 if (ops->set_default_reset_request) 6202 ops->set_default_reset_request(ae_dev, 6203 HNAE3_FUNC_RESET); 6204 if (ops->reset_event) 6205 ops->reset_event(pdev, NULL); 6206 6207 int_en |= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S; 6208 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en); 6209 6210 int_work = IRQ_HANDLED; 6211 } else { 6212 dev_err(hr_dev->dev, "there is no basic abn irq found.\n"); 6213 } 6214 6215 return IRQ_RETVAL(int_work); 6216 } 6217 6218 static int fmea_ram_ecc_query(struct hns_roce_dev *hr_dev, 6219 struct fmea_ram_ecc *ecc_info) 6220 { 6221 struct hns_roce_cmq_desc desc; 6222 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data; 6223 int ret; 6224 6225 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_QUERY_RAM_ECC, true); 6226 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 6227 if (ret) 6228 return ret; 6229 6230 ecc_info->is_ecc_err = hr_reg_read(req, QUERY_RAM_ECC_1BIT_ERR); 6231 ecc_info->res_type = hr_reg_read(req, QUERY_RAM_ECC_RES_TYPE); 6232 ecc_info->index = hr_reg_read(req, QUERY_RAM_ECC_TAG); 6233 6234 return 0; 6235 } 6236 6237 static int fmea_recover_gmv(struct hns_roce_dev *hr_dev, u32 idx) 6238 { 6239 struct hns_roce_cmq_desc desc; 6240 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data; 6241 u32 addr_upper; 6242 u32 addr_low; 6243 int ret; 6244 6245 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT, true); 6246 hr_reg_write(req, CFG_GMV_BT_IDX, idx); 6247 6248 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 6249 if (ret) { 6250 dev_err(hr_dev->dev, 6251 "failed to execute cmd to read gmv, ret = %d.\n", ret); 6252 return ret; 6253 } 6254 6255 addr_low = hr_reg_read(req, CFG_GMV_BT_BA_L); 6256 addr_upper = hr_reg_read(req, CFG_GMV_BT_BA_H); 6257 6258 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT, false); 6259 hr_reg_write(req, CFG_GMV_BT_BA_L, addr_low); 6260 hr_reg_write(req, CFG_GMV_BT_BA_H, addr_upper); 6261 hr_reg_write(req, CFG_GMV_BT_IDX, idx); 6262 6263 return hns_roce_cmq_send(hr_dev, &desc, 1); 6264 } 6265 6266 static u64 fmea_get_ram_res_addr(u32 res_type, __le64 *data) 6267 { 6268 if (res_type == ECC_RESOURCE_QPC_TIMER || 6269 res_type == ECC_RESOURCE_CQC_TIMER || 6270 res_type == ECC_RESOURCE_SCCC) 6271 return le64_to_cpu(*data); 6272 6273 return le64_to_cpu(*data) << PAGE_SHIFT; 6274 } 6275 6276 static int fmea_recover_others(struct hns_roce_dev *hr_dev, u32 res_type, 6277 u32 index) 6278 { 6279 u8 write_bt0_op = fmea_ram_res[res_type].write_bt0_op; 6280 u8 read_bt0_op = fmea_ram_res[res_type].read_bt0_op; 6281 struct hns_roce_cmd_mailbox *mailbox; 6282 u64 addr; 6283 int ret; 6284 6285 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 6286 if (IS_ERR(mailbox)) 6287 return PTR_ERR(mailbox); 6288 6289 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, read_bt0_op, index); 6290 if (ret) { 6291 dev_err(hr_dev->dev, 6292 "failed to execute cmd to read fmea ram, ret = %d.\n", 6293 ret); 6294 goto out; 6295 } 6296 6297 addr = fmea_get_ram_res_addr(res_type, mailbox->buf); 6298 6299 ret = hns_roce_cmd_mbox(hr_dev, addr, 0, write_bt0_op, index); 6300 if (ret) 6301 dev_err(hr_dev->dev, 6302 "failed to execute cmd to write fmea ram, ret = %d.\n", 6303 ret); 6304 6305 out: 6306 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 6307 return ret; 6308 } 6309 6310 static void fmea_ram_ecc_recover(struct hns_roce_dev *hr_dev, 6311 struct fmea_ram_ecc *ecc_info) 6312 { 6313 u32 res_type = ecc_info->res_type; 6314 u32 index = ecc_info->index; 6315 int ret; 6316 6317 BUILD_BUG_ON(ARRAY_SIZE(fmea_ram_res) != ECC_RESOURCE_COUNT); 6318 6319 if (res_type >= ECC_RESOURCE_COUNT) { 6320 dev_err(hr_dev->dev, "unsupported fmea ram ecc type %u.\n", 6321 res_type); 6322 return; 6323 } 6324 6325 if (res_type == ECC_RESOURCE_GMV) 6326 ret = fmea_recover_gmv(hr_dev, index); 6327 else 6328 ret = fmea_recover_others(hr_dev, res_type, index); 6329 if (ret) 6330 dev_err(hr_dev->dev, 6331 "failed to recover %s, index = %u, ret = %d.\n", 6332 fmea_ram_res[res_type].name, index, ret); 6333 } 6334 6335 static void fmea_ram_ecc_work(struct work_struct *ecc_work) 6336 { 6337 struct hns_roce_dev *hr_dev = 6338 container_of(ecc_work, struct hns_roce_dev, ecc_work); 6339 struct fmea_ram_ecc ecc_info = {}; 6340 6341 if (fmea_ram_ecc_query(hr_dev, &ecc_info)) { 6342 dev_err(hr_dev->dev, "failed to query fmea ram ecc.\n"); 6343 return; 6344 } 6345 6346 if (!ecc_info.is_ecc_err) { 6347 dev_err(hr_dev->dev, "there is no fmea ram ecc err found.\n"); 6348 return; 6349 } 6350 6351 fmea_ram_ecc_recover(hr_dev, &ecc_info); 6352 } 6353 6354 static irqreturn_t hns_roce_v2_msix_interrupt_abn(int irq, void *dev_id) 6355 { 6356 struct hns_roce_dev *hr_dev = dev_id; 6357 irqreturn_t int_work = IRQ_NONE; 6358 u32 int_st; 6359 6360 int_st = roce_read(hr_dev, ROCEE_VF_ABN_INT_ST_REG); 6361 6362 if (int_st) { 6363 int_work = abnormal_interrupt_basic(hr_dev, int_st); 6364 } else if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) { 6365 queue_work(hr_dev->irq_workq, &hr_dev->ecc_work); 6366 int_work = IRQ_HANDLED; 6367 } else { 6368 dev_err(hr_dev->dev, "there is no abnormal irq found.\n"); 6369 } 6370 6371 return IRQ_RETVAL(int_work); 6372 } 6373 6374 static void hns_roce_v2_int_mask_enable(struct hns_roce_dev *hr_dev, 6375 int eq_num, u32 enable_flag) 6376 { 6377 int i; 6378 6379 for (i = 0; i < eq_num; i++) 6380 roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG + 6381 i * EQ_REG_OFFSET, enable_flag); 6382 6383 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, enable_flag); 6384 roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG, enable_flag); 6385 } 6386 6387 static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev, u32 eqn) 6388 { 6389 struct device *dev = hr_dev->dev; 6390 int ret; 6391 u8 cmd; 6392 6393 if (eqn < hr_dev->caps.num_comp_vectors) 6394 cmd = HNS_ROCE_CMD_DESTROY_CEQC; 6395 else 6396 cmd = HNS_ROCE_CMD_DESTROY_AEQC; 6397 6398 ret = hns_roce_destroy_hw_ctx(hr_dev, cmd, eqn & HNS_ROCE_V2_EQN_M); 6399 if (ret) 6400 dev_err(dev, "[mailbox cmd] destroy eqc(%u) failed.\n", eqn); 6401 } 6402 6403 static void free_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq) 6404 { 6405 hns_roce_mtr_destroy(hr_dev, &eq->mtr); 6406 } 6407 6408 static void init_eq_config(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq) 6409 { 6410 eq->db_reg = hr_dev->reg_base + ROCEE_VF_EQ_DB_CFG0_REG; 6411 eq->cons_index = 0; 6412 eq->over_ignore = HNS_ROCE_V2_EQ_OVER_IGNORE_0; 6413 eq->coalesce = HNS_ROCE_V2_EQ_COALESCE_0; 6414 eq->arm_st = HNS_ROCE_V2_EQ_ALWAYS_ARMED; 6415 eq->shift = ilog2((unsigned int)eq->entries); 6416 } 6417 6418 static int config_eqc(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq, 6419 void *mb_buf) 6420 { 6421 u64 eqe_ba[MTT_MIN_COUNT] = { 0 }; 6422 struct hns_roce_eq_context *eqc; 6423 u64 bt_ba = 0; 6424 int ret; 6425 6426 eqc = mb_buf; 6427 memset(eqc, 0, sizeof(struct hns_roce_eq_context)); 6428 6429 init_eq_config(hr_dev, eq); 6430 6431 /* if not multi-hop, eqe buffer only use one trunk */ 6432 ret = hns_roce_mtr_find(hr_dev, &eq->mtr, 0, eqe_ba, 6433 ARRAY_SIZE(eqe_ba)); 6434 if (ret) { 6435 dev_err(hr_dev->dev, "failed to find EQE mtr, ret = %d\n", ret); 6436 return ret; 6437 } 6438 6439 bt_ba = hns_roce_get_mtr_ba(&eq->mtr); 6440 6441 hr_reg_write(eqc, EQC_EQ_ST, HNS_ROCE_V2_EQ_STATE_VALID); 6442 hr_reg_write(eqc, EQC_EQE_HOP_NUM, eq->hop_num); 6443 hr_reg_write(eqc, EQC_OVER_IGNORE, eq->over_ignore); 6444 hr_reg_write(eqc, EQC_COALESCE, eq->coalesce); 6445 hr_reg_write(eqc, EQC_ARM_ST, eq->arm_st); 6446 hr_reg_write(eqc, EQC_EQN, eq->eqn); 6447 hr_reg_write(eqc, EQC_EQE_CNT, HNS_ROCE_EQ_INIT_EQE_CNT); 6448 hr_reg_write(eqc, EQC_EQE_BA_PG_SZ, 6449 to_hr_hw_page_shift(eq->mtr.hem_cfg.ba_pg_shift)); 6450 hr_reg_write(eqc, EQC_EQE_BUF_PG_SZ, 6451 to_hr_hw_page_shift(eq->mtr.hem_cfg.buf_pg_shift)); 6452 hr_reg_write(eqc, EQC_EQ_PROD_INDX, HNS_ROCE_EQ_INIT_PROD_IDX); 6453 hr_reg_write(eqc, EQC_EQ_MAX_CNT, eq->eq_max_cnt); 6454 6455 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) { 6456 if (eq->eq_period * HNS_ROCE_CLOCK_ADJUST > USHRT_MAX) { 6457 dev_info(hr_dev->dev, "eq_period(%u) reached the upper limit, adjusted to 65.\n", 6458 eq->eq_period); 6459 eq->eq_period = HNS_ROCE_MAX_EQ_PERIOD; 6460 } 6461 eq->eq_period *= HNS_ROCE_CLOCK_ADJUST; 6462 } 6463 6464 hr_reg_write(eqc, EQC_EQ_PERIOD, eq->eq_period); 6465 hr_reg_write(eqc, EQC_EQE_REPORT_TIMER, HNS_ROCE_EQ_INIT_REPORT_TIMER); 6466 hr_reg_write(eqc, EQC_EQE_BA_L, bt_ba >> 3); 6467 hr_reg_write(eqc, EQC_EQE_BA_H, bt_ba >> 35); 6468 hr_reg_write(eqc, EQC_SHIFT, eq->shift); 6469 hr_reg_write(eqc, EQC_MSI_INDX, HNS_ROCE_EQ_INIT_MSI_IDX); 6470 hr_reg_write(eqc, EQC_CUR_EQE_BA_L, eqe_ba[0] >> 12); 6471 hr_reg_write(eqc, EQC_CUR_EQE_BA_M, eqe_ba[0] >> 28); 6472 hr_reg_write(eqc, EQC_CUR_EQE_BA_H, eqe_ba[0] >> 60); 6473 hr_reg_write(eqc, EQC_EQ_CONS_INDX, HNS_ROCE_EQ_INIT_CONS_IDX); 6474 hr_reg_write(eqc, EQC_NEX_EQE_BA_L, eqe_ba[1] >> 12); 6475 hr_reg_write(eqc, EQC_NEX_EQE_BA_H, eqe_ba[1] >> 44); 6476 hr_reg_write(eqc, EQC_EQE_SIZE, eq->eqe_size == HNS_ROCE_V3_EQE_SIZE); 6477 6478 return 0; 6479 } 6480 6481 static int alloc_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq) 6482 { 6483 struct hns_roce_buf_attr buf_attr = {}; 6484 int err; 6485 6486 if (hr_dev->caps.eqe_hop_num == HNS_ROCE_HOP_NUM_0) 6487 eq->hop_num = 0; 6488 else 6489 eq->hop_num = hr_dev->caps.eqe_hop_num; 6490 6491 buf_attr.page_shift = hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT; 6492 buf_attr.region[0].size = eq->entries * eq->eqe_size; 6493 buf_attr.region[0].hopnum = eq->hop_num; 6494 buf_attr.region_count = 1; 6495 6496 err = hns_roce_mtr_create(hr_dev, &eq->mtr, &buf_attr, 6497 hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT, NULL, 6498 0); 6499 if (err) 6500 dev_err(hr_dev->dev, "failed to alloc EQE mtr, err %d\n", err); 6501 6502 return err; 6503 } 6504 6505 static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev, 6506 struct hns_roce_eq *eq, u8 eq_cmd) 6507 { 6508 struct hns_roce_cmd_mailbox *mailbox; 6509 int ret; 6510 6511 /* Allocate mailbox memory */ 6512 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 6513 if (IS_ERR(mailbox)) 6514 return PTR_ERR(mailbox); 6515 6516 ret = alloc_eq_buf(hr_dev, eq); 6517 if (ret) 6518 goto free_cmd_mbox; 6519 6520 ret = config_eqc(hr_dev, eq, mailbox->buf); 6521 if (ret) 6522 goto err_cmd_mbox; 6523 6524 ret = hns_roce_create_hw_ctx(hr_dev, mailbox, eq_cmd, eq->eqn); 6525 if (ret) { 6526 dev_err(hr_dev->dev, "[mailbox cmd] create eqc failed.\n"); 6527 goto err_cmd_mbox; 6528 } 6529 6530 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 6531 6532 return 0; 6533 6534 err_cmd_mbox: 6535 free_eq_buf(hr_dev, eq); 6536 6537 free_cmd_mbox: 6538 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 6539 6540 return ret; 6541 } 6542 6543 static int __hns_roce_request_irq(struct hns_roce_dev *hr_dev, int irq_num, 6544 int comp_num, int aeq_num, int other_num) 6545 { 6546 struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; 6547 int i, j; 6548 int ret; 6549 6550 for (i = 0; i < irq_num; i++) { 6551 hr_dev->irq_names[i] = kzalloc(HNS_ROCE_INT_NAME_LEN, 6552 GFP_KERNEL); 6553 if (!hr_dev->irq_names[i]) { 6554 ret = -ENOMEM; 6555 goto err_kzalloc_failed; 6556 } 6557 } 6558 6559 /* irq contains: abnormal + AEQ + CEQ */ 6560 for (j = 0; j < other_num; j++) 6561 snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN, 6562 "hns-%s-abn-%d", pci_name(hr_dev->pci_dev), j); 6563 6564 for (j = other_num; j < (other_num + aeq_num); j++) 6565 snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN, 6566 "hns-%s-aeq-%d", pci_name(hr_dev->pci_dev), j - other_num); 6567 6568 for (j = (other_num + aeq_num); j < irq_num; j++) 6569 snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN, 6570 "hns-%s-ceq-%d", pci_name(hr_dev->pci_dev), 6571 j - other_num - aeq_num); 6572 6573 for (j = 0; j < irq_num; j++) { 6574 if (j < other_num) 6575 ret = request_irq(hr_dev->irq[j], 6576 hns_roce_v2_msix_interrupt_abn, 6577 0, hr_dev->irq_names[j], hr_dev); 6578 6579 else if (j < (other_num + comp_num)) 6580 ret = request_irq(eq_table->eq[j - other_num].irq, 6581 hns_roce_v2_msix_interrupt_eq, 6582 0, hr_dev->irq_names[j + aeq_num], 6583 &eq_table->eq[j - other_num]); 6584 else 6585 ret = request_irq(eq_table->eq[j - other_num].irq, 6586 hns_roce_v2_msix_interrupt_eq, 6587 0, hr_dev->irq_names[j - comp_num], 6588 &eq_table->eq[j - other_num]); 6589 if (ret) { 6590 dev_err(hr_dev->dev, "request irq error!\n"); 6591 goto err_request_failed; 6592 } 6593 } 6594 6595 return 0; 6596 6597 err_request_failed: 6598 for (j -= 1; j >= 0; j--) 6599 if (j < other_num) 6600 free_irq(hr_dev->irq[j], hr_dev); 6601 else 6602 free_irq(eq_table->eq[j - other_num].irq, 6603 &eq_table->eq[j - other_num]); 6604 6605 err_kzalloc_failed: 6606 for (i -= 1; i >= 0; i--) 6607 kfree(hr_dev->irq_names[i]); 6608 6609 return ret; 6610 } 6611 6612 static void __hns_roce_free_irq(struct hns_roce_dev *hr_dev) 6613 { 6614 int irq_num; 6615 int eq_num; 6616 int i; 6617 6618 eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors; 6619 irq_num = eq_num + hr_dev->caps.num_other_vectors; 6620 6621 for (i = 0; i < hr_dev->caps.num_other_vectors; i++) 6622 free_irq(hr_dev->irq[i], hr_dev); 6623 6624 for (i = 0; i < eq_num; i++) 6625 free_irq(hr_dev->eq_table.eq[i].irq, &hr_dev->eq_table.eq[i]); 6626 6627 for (i = 0; i < irq_num; i++) 6628 kfree(hr_dev->irq_names[i]); 6629 } 6630 6631 static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev) 6632 { 6633 struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; 6634 struct device *dev = hr_dev->dev; 6635 struct hns_roce_eq *eq; 6636 int other_num; 6637 int comp_num; 6638 int aeq_num; 6639 int irq_num; 6640 int eq_num; 6641 u8 eq_cmd; 6642 int ret; 6643 int i; 6644 6645 other_num = hr_dev->caps.num_other_vectors; 6646 comp_num = hr_dev->caps.num_comp_vectors; 6647 aeq_num = hr_dev->caps.num_aeq_vectors; 6648 6649 eq_num = comp_num + aeq_num; 6650 irq_num = eq_num + other_num; 6651 6652 eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL); 6653 if (!eq_table->eq) 6654 return -ENOMEM; 6655 6656 /* create eq */ 6657 for (i = 0; i < eq_num; i++) { 6658 eq = &eq_table->eq[i]; 6659 eq->hr_dev = hr_dev; 6660 eq->eqn = i; 6661 if (i < comp_num) { 6662 /* CEQ */ 6663 eq_cmd = HNS_ROCE_CMD_CREATE_CEQC; 6664 eq->type_flag = HNS_ROCE_CEQ; 6665 eq->entries = hr_dev->caps.ceqe_depth; 6666 eq->eqe_size = hr_dev->caps.ceqe_size; 6667 eq->irq = hr_dev->irq[i + other_num + aeq_num]; 6668 eq->eq_max_cnt = HNS_ROCE_CEQ_DEFAULT_BURST_NUM; 6669 eq->eq_period = HNS_ROCE_CEQ_DEFAULT_INTERVAL; 6670 } else { 6671 /* AEQ */ 6672 eq_cmd = HNS_ROCE_CMD_CREATE_AEQC; 6673 eq->type_flag = HNS_ROCE_AEQ; 6674 eq->entries = hr_dev->caps.aeqe_depth; 6675 eq->eqe_size = hr_dev->caps.aeqe_size; 6676 eq->irq = hr_dev->irq[i - comp_num + other_num]; 6677 eq->eq_max_cnt = HNS_ROCE_AEQ_DEFAULT_BURST_NUM; 6678 eq->eq_period = HNS_ROCE_AEQ_DEFAULT_INTERVAL; 6679 } 6680 6681 ret = hns_roce_v2_create_eq(hr_dev, eq, eq_cmd); 6682 if (ret) { 6683 dev_err(dev, "failed to create eq.\n"); 6684 goto err_create_eq_fail; 6685 } 6686 } 6687 6688 INIT_WORK(&hr_dev->ecc_work, fmea_ram_ecc_work); 6689 6690 hr_dev->irq_workq = alloc_ordered_workqueue("hns_roce_irq_workq", 0); 6691 if (!hr_dev->irq_workq) { 6692 dev_err(dev, "failed to create irq workqueue.\n"); 6693 ret = -ENOMEM; 6694 goto err_create_eq_fail; 6695 } 6696 6697 ret = __hns_roce_request_irq(hr_dev, irq_num, comp_num, aeq_num, 6698 other_num); 6699 if (ret) { 6700 dev_err(dev, "failed to request irq.\n"); 6701 goto err_request_irq_fail; 6702 } 6703 6704 /* enable irq */ 6705 hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_ENABLE); 6706 6707 return 0; 6708 6709 err_request_irq_fail: 6710 destroy_workqueue(hr_dev->irq_workq); 6711 6712 err_create_eq_fail: 6713 for (i -= 1; i >= 0; i--) 6714 free_eq_buf(hr_dev, &eq_table->eq[i]); 6715 kfree(eq_table->eq); 6716 6717 return ret; 6718 } 6719 6720 static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev) 6721 { 6722 struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; 6723 int eq_num; 6724 int i; 6725 6726 eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors; 6727 6728 /* Disable irq */ 6729 hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE); 6730 6731 __hns_roce_free_irq(hr_dev); 6732 destroy_workqueue(hr_dev->irq_workq); 6733 6734 for (i = 0; i < eq_num; i++) { 6735 hns_roce_v2_destroy_eqc(hr_dev, i); 6736 6737 free_eq_buf(hr_dev, &eq_table->eq[i]); 6738 } 6739 6740 kfree(eq_table->eq); 6741 } 6742 6743 static const struct ib_device_ops hns_roce_v2_dev_ops = { 6744 .destroy_qp = hns_roce_v2_destroy_qp, 6745 .modify_cq = hns_roce_v2_modify_cq, 6746 .poll_cq = hns_roce_v2_poll_cq, 6747 .post_recv = hns_roce_v2_post_recv, 6748 .post_send = hns_roce_v2_post_send, 6749 .query_qp = hns_roce_v2_query_qp, 6750 .req_notify_cq = hns_roce_v2_req_notify_cq, 6751 }; 6752 6753 static const struct ib_device_ops hns_roce_v2_dev_srq_ops = { 6754 .modify_srq = hns_roce_v2_modify_srq, 6755 .post_srq_recv = hns_roce_v2_post_srq_recv, 6756 .query_srq = hns_roce_v2_query_srq, 6757 }; 6758 6759 static const struct hns_roce_hw hns_roce_hw_v2 = { 6760 .cmq_init = hns_roce_v2_cmq_init, 6761 .cmq_exit = hns_roce_v2_cmq_exit, 6762 .hw_profile = hns_roce_v2_profile, 6763 .hw_init = hns_roce_v2_init, 6764 .hw_exit = hns_roce_v2_exit, 6765 .post_mbox = v2_post_mbox, 6766 .poll_mbox_done = v2_poll_mbox_done, 6767 .chk_mbox_avail = v2_chk_mbox_is_avail, 6768 .set_gid = hns_roce_v2_set_gid, 6769 .set_mac = hns_roce_v2_set_mac, 6770 .write_mtpt = hns_roce_v2_write_mtpt, 6771 .rereg_write_mtpt = hns_roce_v2_rereg_write_mtpt, 6772 .frmr_write_mtpt = hns_roce_v2_frmr_write_mtpt, 6773 .mw_write_mtpt = hns_roce_v2_mw_write_mtpt, 6774 .write_cqc = hns_roce_v2_write_cqc, 6775 .set_hem = hns_roce_v2_set_hem, 6776 .clear_hem = hns_roce_v2_clear_hem, 6777 .modify_qp = hns_roce_v2_modify_qp, 6778 .dereg_mr = hns_roce_v2_dereg_mr, 6779 .qp_flow_control_init = hns_roce_v2_qp_flow_control_init, 6780 .init_eq = hns_roce_v2_init_eq_table, 6781 .cleanup_eq = hns_roce_v2_cleanup_eq_table, 6782 .write_srqc = hns_roce_v2_write_srqc, 6783 .query_cqc = hns_roce_v2_query_cqc, 6784 .query_qpc = hns_roce_v2_query_qpc, 6785 .query_mpt = hns_roce_v2_query_mpt, 6786 .query_srqc = hns_roce_v2_query_srqc, 6787 .query_sccc = hns_roce_v2_query_sccc, 6788 .query_hw_counter = hns_roce_hw_v2_query_counter, 6789 .get_dscp = hns_roce_hw_v2_get_dscp, 6790 .hns_roce_dev_ops = &hns_roce_v2_dev_ops, 6791 .hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops, 6792 }; 6793 6794 static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = { 6795 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 0}, 6796 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 0}, 6797 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0}, 6798 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0}, 6799 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0}, 6800 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA), 0}, 6801 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_RDMA_DCB_PFC_VF), 6802 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS}, 6803 /* required last entry */ 6804 {0, } 6805 }; 6806 6807 MODULE_DEVICE_TABLE(pci, hns_roce_hw_v2_pci_tbl); 6808 6809 static void hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev, 6810 struct hnae3_handle *handle) 6811 { 6812 struct hns_roce_v2_priv *priv = hr_dev->priv; 6813 const struct pci_device_id *id; 6814 int i; 6815 6816 hr_dev->pci_dev = handle->pdev; 6817 id = pci_match_id(hns_roce_hw_v2_pci_tbl, hr_dev->pci_dev); 6818 hr_dev->is_vf = id->driver_data; 6819 hr_dev->dev = &handle->pdev->dev; 6820 hr_dev->hw = &hns_roce_hw_v2; 6821 hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG; 6822 hr_dev->odb_offset = hr_dev->sdb_offset; 6823 6824 /* Get info from NIC driver. */ 6825 hr_dev->reg_base = handle->rinfo.roce_io_base; 6826 hr_dev->mem_base = handle->rinfo.roce_mem_base; 6827 hr_dev->caps.num_ports = 1; 6828 hr_dev->iboe.netdevs[0] = handle->rinfo.netdev; 6829 hr_dev->iboe.phy_port[0] = 0; 6830 6831 addrconf_addr_eui48((u8 *)&hr_dev->ib_dev.node_guid, 6832 hr_dev->iboe.netdevs[0]->dev_addr); 6833 6834 for (i = 0; i < handle->rinfo.num_vectors; i++) 6835 hr_dev->irq[i] = pci_irq_vector(handle->pdev, 6836 i + handle->rinfo.base_vector); 6837 6838 /* cmd issue mode: 0 is poll, 1 is event */ 6839 hr_dev->cmd_mod = 1; 6840 hr_dev->loop_idc = 0; 6841 6842 hr_dev->reset_cnt = handle->ae_algo->ops->ae_dev_reset_cnt(handle); 6843 priv->handle = handle; 6844 } 6845 6846 static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle) 6847 { 6848 struct hns_roce_dev *hr_dev; 6849 int ret; 6850 6851 hr_dev = ib_alloc_device(hns_roce_dev, ib_dev); 6852 if (!hr_dev) 6853 return -ENOMEM; 6854 6855 hr_dev->priv = kzalloc(sizeof(struct hns_roce_v2_priv), GFP_KERNEL); 6856 if (!hr_dev->priv) { 6857 ret = -ENOMEM; 6858 goto error_failed_kzalloc; 6859 } 6860 6861 hns_roce_hw_v2_get_cfg(hr_dev, handle); 6862 6863 ret = hns_roce_init(hr_dev); 6864 if (ret) { 6865 dev_err(hr_dev->dev, "RoCE Engine init failed!\n"); 6866 goto error_failed_roce_init; 6867 } 6868 6869 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) { 6870 ret = free_mr_init(hr_dev); 6871 if (ret) { 6872 dev_err(hr_dev->dev, "failed to init free mr!\n"); 6873 goto error_failed_free_mr_init; 6874 } 6875 } 6876 6877 handle->priv = hr_dev; 6878 6879 return 0; 6880 6881 error_failed_free_mr_init: 6882 hns_roce_exit(hr_dev); 6883 6884 error_failed_roce_init: 6885 kfree(hr_dev->priv); 6886 6887 error_failed_kzalloc: 6888 ib_dealloc_device(&hr_dev->ib_dev); 6889 6890 return ret; 6891 } 6892 6893 static void __hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle, 6894 bool reset) 6895 { 6896 struct hns_roce_dev *hr_dev = handle->priv; 6897 6898 if (!hr_dev) 6899 return; 6900 6901 handle->priv = NULL; 6902 6903 hr_dev->state = HNS_ROCE_DEVICE_STATE_UNINIT; 6904 hns_roce_handle_device_err(hr_dev); 6905 6906 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) 6907 free_mr_exit(hr_dev); 6908 6909 hns_roce_exit(hr_dev); 6910 kfree(hr_dev->priv); 6911 ib_dealloc_device(&hr_dev->ib_dev); 6912 } 6913 6914 static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle) 6915 { 6916 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 6917 const struct pci_device_id *id; 6918 struct device *dev = &handle->pdev->dev; 6919 int ret; 6920 6921 handle->rinfo.instance_state = HNS_ROCE_STATE_INIT; 6922 6923 if (ops->ae_dev_resetting(handle) || ops->get_hw_reset_stat(handle)) { 6924 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT; 6925 goto reset_chk_err; 6926 } 6927 6928 id = pci_match_id(hns_roce_hw_v2_pci_tbl, handle->pdev); 6929 if (!id) 6930 return 0; 6931 6932 if (id->driver_data && handle->pdev->revision == PCI_REVISION_ID_HIP08) 6933 return 0; 6934 6935 ret = __hns_roce_hw_v2_init_instance(handle); 6936 if (ret) { 6937 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT; 6938 dev_err(dev, "RoCE instance init failed! ret = %d\n", ret); 6939 if (ops->ae_dev_resetting(handle) || 6940 ops->get_hw_reset_stat(handle)) 6941 goto reset_chk_err; 6942 else 6943 return ret; 6944 } 6945 6946 handle->rinfo.instance_state = HNS_ROCE_STATE_INITED; 6947 6948 return 0; 6949 6950 reset_chk_err: 6951 dev_err(dev, "Device is busy in resetting state.\n" 6952 "please retry later.\n"); 6953 6954 return -EBUSY; 6955 } 6956 6957 static void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle, 6958 bool reset) 6959 { 6960 if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED) 6961 return; 6962 6963 handle->rinfo.instance_state = HNS_ROCE_STATE_UNINIT; 6964 6965 __hns_roce_hw_v2_uninit_instance(handle, reset); 6966 6967 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT; 6968 } 6969 static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle) 6970 { 6971 struct hns_roce_dev *hr_dev; 6972 6973 if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED) { 6974 set_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state); 6975 return 0; 6976 } 6977 6978 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_DOWN; 6979 clear_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state); 6980 6981 hr_dev = handle->priv; 6982 if (!hr_dev) 6983 return 0; 6984 6985 hr_dev->active = false; 6986 hr_dev->dis_db = true; 6987 hr_dev->state = HNS_ROCE_DEVICE_STATE_RST_DOWN; 6988 6989 return 0; 6990 } 6991 6992 static int hns_roce_hw_v2_reset_notify_init(struct hnae3_handle *handle) 6993 { 6994 struct device *dev = &handle->pdev->dev; 6995 int ret; 6996 6997 if (test_and_clear_bit(HNS_ROCE_RST_DIRECT_RETURN, 6998 &handle->rinfo.state)) { 6999 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED; 7000 return 0; 7001 } 7002 7003 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INIT; 7004 7005 dev_info(&handle->pdev->dev, "In reset process RoCE client reinit.\n"); 7006 ret = __hns_roce_hw_v2_init_instance(handle); 7007 if (ret) { 7008 /* when reset notify type is HNAE3_INIT_CLIENT In reset notify 7009 * callback function, RoCE Engine reinitialize. If RoCE reinit 7010 * failed, we should inform NIC driver. 7011 */ 7012 handle->priv = NULL; 7013 dev_err(dev, "In reset process RoCE reinit failed %d.\n", ret); 7014 } else { 7015 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED; 7016 dev_info(dev, "reset done, RoCE client reinit finished.\n"); 7017 } 7018 7019 return ret; 7020 } 7021 7022 static int hns_roce_hw_v2_reset_notify_uninit(struct hnae3_handle *handle) 7023 { 7024 if (test_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state)) 7025 return 0; 7026 7027 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_UNINIT; 7028 dev_info(&handle->pdev->dev, "In reset process RoCE client uninit.\n"); 7029 msleep(HNS_ROCE_V2_HW_RST_UNINT_DELAY); 7030 __hns_roce_hw_v2_uninit_instance(handle, false); 7031 7032 return 0; 7033 } 7034 7035 static int hns_roce_hw_v2_reset_notify(struct hnae3_handle *handle, 7036 enum hnae3_reset_notify_type type) 7037 { 7038 int ret = 0; 7039 7040 switch (type) { 7041 case HNAE3_DOWN_CLIENT: 7042 ret = hns_roce_hw_v2_reset_notify_down(handle); 7043 break; 7044 case HNAE3_INIT_CLIENT: 7045 ret = hns_roce_hw_v2_reset_notify_init(handle); 7046 break; 7047 case HNAE3_UNINIT_CLIENT: 7048 ret = hns_roce_hw_v2_reset_notify_uninit(handle); 7049 break; 7050 default: 7051 break; 7052 } 7053 7054 return ret; 7055 } 7056 7057 static const struct hnae3_client_ops hns_roce_hw_v2_ops = { 7058 .init_instance = hns_roce_hw_v2_init_instance, 7059 .uninit_instance = hns_roce_hw_v2_uninit_instance, 7060 .reset_notify = hns_roce_hw_v2_reset_notify, 7061 }; 7062 7063 static struct hnae3_client hns_roce_hw_v2_client = { 7064 .name = "hns_roce_hw_v2", 7065 .type = HNAE3_CLIENT_ROCE, 7066 .ops = &hns_roce_hw_v2_ops, 7067 }; 7068 7069 static int __init hns_roce_hw_v2_init(void) 7070 { 7071 hns_roce_init_debugfs(); 7072 return hnae3_register_client(&hns_roce_hw_v2_client); 7073 } 7074 7075 static void __exit hns_roce_hw_v2_exit(void) 7076 { 7077 hnae3_unregister_client(&hns_roce_hw_v2_client); 7078 hns_roce_cleanup_debugfs(); 7079 } 7080 7081 module_init(hns_roce_hw_v2_init); 7082 module_exit(hns_roce_hw_v2_exit); 7083 7084 MODULE_LICENSE("Dual BSD/GPL"); 7085 MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>"); 7086 MODULE_AUTHOR("Lijun Ou <oulijun@huawei.com>"); 7087 MODULE_AUTHOR("Shaobo Xu <xushaobo2@huawei.com>"); 7088 MODULE_DESCRIPTION("Hisilicon Hip08 Family RoCE Driver"); 7089