1 /* 2 * Copyright (c) 2016-2017 Hisilicon Limited. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/acpi.h> 34 #include <linux/etherdevice.h> 35 #include <linux/interrupt.h> 36 #include <linux/iopoll.h> 37 #include <linux/kernel.h> 38 #include <linux/types.h> 39 #include <net/addrconf.h> 40 #include <rdma/ib_addr.h> 41 #include <rdma/ib_cache.h> 42 #include <rdma/ib_umem.h> 43 #include <rdma/uverbs_ioctl.h> 44 45 #include "hnae3.h" 46 #include "hns_roce_common.h" 47 #include "hns_roce_device.h" 48 #include "hns_roce_cmd.h" 49 #include "hns_roce_hem.h" 50 #include "hns_roce_hw_v2.h" 51 52 enum { 53 CMD_RST_PRC_OTHERS, 54 CMD_RST_PRC_SUCCESS, 55 CMD_RST_PRC_EBUSY, 56 }; 57 58 enum ecc_resource_type { 59 ECC_RESOURCE_QPC, 60 ECC_RESOURCE_CQC, 61 ECC_RESOURCE_MPT, 62 ECC_RESOURCE_SRQC, 63 ECC_RESOURCE_GMV, 64 ECC_RESOURCE_QPC_TIMER, 65 ECC_RESOURCE_CQC_TIMER, 66 ECC_RESOURCE_SCCC, 67 ECC_RESOURCE_COUNT, 68 }; 69 70 static const struct { 71 const char *name; 72 u8 read_bt0_op; 73 u8 write_bt0_op; 74 } fmea_ram_res[] = { 75 { "ECC_RESOURCE_QPC", 76 HNS_ROCE_CMD_READ_QPC_BT0, HNS_ROCE_CMD_WRITE_QPC_BT0 }, 77 { "ECC_RESOURCE_CQC", 78 HNS_ROCE_CMD_READ_CQC_BT0, HNS_ROCE_CMD_WRITE_CQC_BT0 }, 79 { "ECC_RESOURCE_MPT", 80 HNS_ROCE_CMD_READ_MPT_BT0, HNS_ROCE_CMD_WRITE_MPT_BT0 }, 81 { "ECC_RESOURCE_SRQC", 82 HNS_ROCE_CMD_READ_SRQC_BT0, HNS_ROCE_CMD_WRITE_SRQC_BT0 }, 83 /* ECC_RESOURCE_GMV is handled by cmdq, not mailbox */ 84 { "ECC_RESOURCE_GMV", 85 0, 0 }, 86 { "ECC_RESOURCE_QPC_TIMER", 87 HNS_ROCE_CMD_READ_QPC_TIMER_BT0, HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0 }, 88 { "ECC_RESOURCE_CQC_TIMER", 89 HNS_ROCE_CMD_READ_CQC_TIMER_BT0, HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0 }, 90 { "ECC_RESOURCE_SCCC", 91 HNS_ROCE_CMD_READ_SCCC_BT0, HNS_ROCE_CMD_WRITE_SCCC_BT0 }, 92 }; 93 94 static inline void set_data_seg_v2(struct hns_roce_v2_wqe_data_seg *dseg, 95 struct ib_sge *sg) 96 { 97 dseg->lkey = cpu_to_le32(sg->lkey); 98 dseg->addr = cpu_to_le64(sg->addr); 99 dseg->len = cpu_to_le32(sg->length); 100 } 101 102 /* 103 * mapped-value = 1 + real-value 104 * The hns wr opcode real value is start from 0, In order to distinguish between 105 * initialized and uninitialized map values, we plus 1 to the actual value when 106 * defining the mapping, so that the validity can be identified by checking the 107 * mapped value is greater than 0. 108 */ 109 #define HR_OPC_MAP(ib_key, hr_key) \ 110 [IB_WR_ ## ib_key] = 1 + HNS_ROCE_V2_WQE_OP_ ## hr_key 111 112 static const u32 hns_roce_op_code[] = { 113 HR_OPC_MAP(RDMA_WRITE, RDMA_WRITE), 114 HR_OPC_MAP(RDMA_WRITE_WITH_IMM, RDMA_WRITE_WITH_IMM), 115 HR_OPC_MAP(SEND, SEND), 116 HR_OPC_MAP(SEND_WITH_IMM, SEND_WITH_IMM), 117 HR_OPC_MAP(RDMA_READ, RDMA_READ), 118 HR_OPC_MAP(ATOMIC_CMP_AND_SWP, ATOM_CMP_AND_SWAP), 119 HR_OPC_MAP(ATOMIC_FETCH_AND_ADD, ATOM_FETCH_AND_ADD), 120 HR_OPC_MAP(SEND_WITH_INV, SEND_WITH_INV), 121 HR_OPC_MAP(MASKED_ATOMIC_CMP_AND_SWP, ATOM_MSK_CMP_AND_SWAP), 122 HR_OPC_MAP(MASKED_ATOMIC_FETCH_AND_ADD, ATOM_MSK_FETCH_AND_ADD), 123 HR_OPC_MAP(REG_MR, FAST_REG_PMR), 124 }; 125 126 static u32 to_hr_opcode(u32 ib_opcode) 127 { 128 if (ib_opcode >= ARRAY_SIZE(hns_roce_op_code)) 129 return HNS_ROCE_V2_WQE_OP_MASK; 130 131 return hns_roce_op_code[ib_opcode] ? hns_roce_op_code[ib_opcode] - 1 : 132 HNS_ROCE_V2_WQE_OP_MASK; 133 } 134 135 static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, 136 const struct ib_reg_wr *wr) 137 { 138 struct hns_roce_wqe_frmr_seg *fseg = 139 (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe); 140 struct hns_roce_mr *mr = to_hr_mr(wr->mr); 141 u64 pbl_ba; 142 143 /* use ib_access_flags */ 144 hr_reg_write_bool(fseg, FRMR_BIND_EN, wr->access & IB_ACCESS_MW_BIND); 145 hr_reg_write_bool(fseg, FRMR_ATOMIC, 146 wr->access & IB_ACCESS_REMOTE_ATOMIC); 147 hr_reg_write_bool(fseg, FRMR_RR, wr->access & IB_ACCESS_REMOTE_READ); 148 hr_reg_write_bool(fseg, FRMR_RW, wr->access & IB_ACCESS_REMOTE_WRITE); 149 hr_reg_write_bool(fseg, FRMR_LW, wr->access & IB_ACCESS_LOCAL_WRITE); 150 151 /* Data structure reuse may lead to confusion */ 152 pbl_ba = mr->pbl_mtr.hem_cfg.root_ba; 153 rc_sq_wqe->msg_len = cpu_to_le32(lower_32_bits(pbl_ba)); 154 rc_sq_wqe->inv_key = cpu_to_le32(upper_32_bits(pbl_ba)); 155 156 rc_sq_wqe->byte_16 = cpu_to_le32(wr->mr->length & 0xffffffff); 157 rc_sq_wqe->byte_20 = cpu_to_le32(wr->mr->length >> 32); 158 rc_sq_wqe->rkey = cpu_to_le32(wr->key); 159 rc_sq_wqe->va = cpu_to_le64(wr->mr->iova); 160 161 hr_reg_write(fseg, FRMR_PBL_SIZE, mr->npages); 162 hr_reg_write(fseg, FRMR_PBL_BUF_PG_SZ, 163 to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift)); 164 hr_reg_clear(fseg, FRMR_BLK_MODE); 165 } 166 167 static void set_atomic_seg(const struct ib_send_wr *wr, 168 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, 169 unsigned int valid_num_sge) 170 { 171 struct hns_roce_v2_wqe_data_seg *dseg = 172 (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe); 173 struct hns_roce_wqe_atomic_seg *aseg = 174 (void *)dseg + sizeof(struct hns_roce_v2_wqe_data_seg); 175 176 set_data_seg_v2(dseg, wr->sg_list); 177 178 if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) { 179 aseg->fetchadd_swap_data = cpu_to_le64(atomic_wr(wr)->swap); 180 aseg->cmp_data = cpu_to_le64(atomic_wr(wr)->compare_add); 181 } else { 182 aseg->fetchadd_swap_data = 183 cpu_to_le64(atomic_wr(wr)->compare_add); 184 aseg->cmp_data = 0; 185 } 186 187 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_SGE_NUM, valid_num_sge); 188 } 189 190 static int fill_ext_sge_inl_data(struct hns_roce_qp *qp, 191 const struct ib_send_wr *wr, 192 unsigned int *sge_idx, u32 msg_len) 193 { 194 struct ib_device *ibdev = &(to_hr_dev(qp->ibqp.device))->ib_dev; 195 unsigned int left_len_in_pg; 196 unsigned int idx = *sge_idx; 197 unsigned int i = 0; 198 unsigned int len; 199 void *addr; 200 void *dseg; 201 202 if (msg_len > qp->sq.ext_sge_cnt * HNS_ROCE_SGE_SIZE) { 203 ibdev_err(ibdev, 204 "no enough extended sge space for inline data.\n"); 205 return -EINVAL; 206 } 207 208 dseg = hns_roce_get_extend_sge(qp, idx & (qp->sge.sge_cnt - 1)); 209 left_len_in_pg = hr_hw_page_align((uintptr_t)dseg) - (uintptr_t)dseg; 210 len = wr->sg_list[0].length; 211 addr = (void *)(unsigned long)(wr->sg_list[0].addr); 212 213 /* When copying data to extended sge space, the left length in page may 214 * not long enough for current user's sge. So the data should be 215 * splited into several parts, one in the first page, and the others in 216 * the subsequent pages. 217 */ 218 while (1) { 219 if (len <= left_len_in_pg) { 220 memcpy(dseg, addr, len); 221 222 idx += len / HNS_ROCE_SGE_SIZE; 223 224 i++; 225 if (i >= wr->num_sge) 226 break; 227 228 left_len_in_pg -= len; 229 len = wr->sg_list[i].length; 230 addr = (void *)(unsigned long)(wr->sg_list[i].addr); 231 dseg += len; 232 } else { 233 memcpy(dseg, addr, left_len_in_pg); 234 235 len -= left_len_in_pg; 236 addr += left_len_in_pg; 237 idx += left_len_in_pg / HNS_ROCE_SGE_SIZE; 238 dseg = hns_roce_get_extend_sge(qp, 239 idx & (qp->sge.sge_cnt - 1)); 240 left_len_in_pg = 1 << HNS_HW_PAGE_SHIFT; 241 } 242 } 243 244 *sge_idx = idx; 245 246 return 0; 247 } 248 249 static void set_extend_sge(struct hns_roce_qp *qp, struct ib_sge *sge, 250 unsigned int *sge_ind, unsigned int cnt) 251 { 252 struct hns_roce_v2_wqe_data_seg *dseg; 253 unsigned int idx = *sge_ind; 254 255 while (cnt > 0) { 256 dseg = hns_roce_get_extend_sge(qp, idx & (qp->sge.sge_cnt - 1)); 257 if (likely(sge->length)) { 258 set_data_seg_v2(dseg, sge); 259 idx++; 260 cnt--; 261 } 262 sge++; 263 } 264 265 *sge_ind = idx; 266 } 267 268 static bool check_inl_data_len(struct hns_roce_qp *qp, unsigned int len) 269 { 270 struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device); 271 int mtu = ib_mtu_enum_to_int(qp->path_mtu); 272 273 if (len > qp->max_inline_data || len > mtu) { 274 ibdev_err(&hr_dev->ib_dev, 275 "invalid length of data, data len = %u, max inline len = %u, path mtu = %d.\n", 276 len, qp->max_inline_data, mtu); 277 return false; 278 } 279 280 return true; 281 } 282 283 static int set_rc_inl(struct hns_roce_qp *qp, const struct ib_send_wr *wr, 284 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, 285 unsigned int *sge_idx) 286 { 287 struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device); 288 u32 msg_len = le32_to_cpu(rc_sq_wqe->msg_len); 289 struct ib_device *ibdev = &hr_dev->ib_dev; 290 unsigned int curr_idx = *sge_idx; 291 void *dseg = rc_sq_wqe; 292 unsigned int i; 293 int ret; 294 295 if (unlikely(wr->opcode == IB_WR_RDMA_READ)) { 296 ibdev_err(ibdev, "invalid inline parameters!\n"); 297 return -EINVAL; 298 } 299 300 if (!check_inl_data_len(qp, msg_len)) 301 return -EINVAL; 302 303 dseg += sizeof(struct hns_roce_v2_rc_send_wqe); 304 305 if (msg_len <= HNS_ROCE_V2_MAX_RC_INL_INN_SZ) { 306 hr_reg_clear(rc_sq_wqe, RC_SEND_WQE_INL_TYPE); 307 308 for (i = 0; i < wr->num_sge; i++) { 309 memcpy(dseg, ((void *)wr->sg_list[i].addr), 310 wr->sg_list[i].length); 311 dseg += wr->sg_list[i].length; 312 } 313 } else { 314 hr_reg_enable(rc_sq_wqe, RC_SEND_WQE_INL_TYPE); 315 316 ret = fill_ext_sge_inl_data(qp, wr, &curr_idx, msg_len); 317 if (ret) 318 return ret; 319 320 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_SGE_NUM, curr_idx - *sge_idx); 321 } 322 323 *sge_idx = curr_idx; 324 325 return 0; 326 } 327 328 static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr, 329 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, 330 unsigned int *sge_ind, 331 unsigned int valid_num_sge) 332 { 333 struct hns_roce_v2_wqe_data_seg *dseg = 334 (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe); 335 struct hns_roce_qp *qp = to_hr_qp(ibqp); 336 int j = 0; 337 int i; 338 339 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_MSG_START_SGE_IDX, 340 (*sge_ind) & (qp->sge.sge_cnt - 1)); 341 342 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_INLINE, 343 !!(wr->send_flags & IB_SEND_INLINE)); 344 if (wr->send_flags & IB_SEND_INLINE) 345 return set_rc_inl(qp, wr, rc_sq_wqe, sge_ind); 346 347 if (valid_num_sge <= HNS_ROCE_SGE_IN_WQE) { 348 for (i = 0; i < wr->num_sge; i++) { 349 if (likely(wr->sg_list[i].length)) { 350 set_data_seg_v2(dseg, wr->sg_list + i); 351 dseg++; 352 } 353 } 354 } else { 355 for (i = 0; i < wr->num_sge && j < HNS_ROCE_SGE_IN_WQE; i++) { 356 if (likely(wr->sg_list[i].length)) { 357 set_data_seg_v2(dseg, wr->sg_list + i); 358 dseg++; 359 j++; 360 } 361 } 362 363 set_extend_sge(qp, wr->sg_list + i, sge_ind, 364 valid_num_sge - HNS_ROCE_SGE_IN_WQE); 365 } 366 367 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_SGE_NUM, valid_num_sge); 368 369 return 0; 370 } 371 372 static int check_send_valid(struct hns_roce_dev *hr_dev, 373 struct hns_roce_qp *hr_qp) 374 { 375 struct ib_device *ibdev = &hr_dev->ib_dev; 376 struct ib_qp *ibqp = &hr_qp->ibqp; 377 378 if (unlikely(ibqp->qp_type != IB_QPT_RC && 379 ibqp->qp_type != IB_QPT_GSI && 380 ibqp->qp_type != IB_QPT_UD)) { 381 ibdev_err(ibdev, "not supported QP(0x%x)type!\n", 382 ibqp->qp_type); 383 return -EOPNOTSUPP; 384 } else if (unlikely(hr_qp->state == IB_QPS_RESET || 385 hr_qp->state == IB_QPS_INIT || 386 hr_qp->state == IB_QPS_RTR)) { 387 ibdev_err(ibdev, "failed to post WQE, QP state %u!\n", 388 hr_qp->state); 389 return -EINVAL; 390 } else if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN)) { 391 ibdev_err(ibdev, "failed to post WQE, dev state %d!\n", 392 hr_dev->state); 393 return -EIO; 394 } 395 396 return 0; 397 } 398 399 static unsigned int calc_wr_sge_num(const struct ib_send_wr *wr, 400 unsigned int *sge_len) 401 { 402 unsigned int valid_num = 0; 403 unsigned int len = 0; 404 int i; 405 406 for (i = 0; i < wr->num_sge; i++) { 407 if (likely(wr->sg_list[i].length)) { 408 len += wr->sg_list[i].length; 409 valid_num++; 410 } 411 } 412 413 *sge_len = len; 414 return valid_num; 415 } 416 417 static __le32 get_immtdata(const struct ib_send_wr *wr) 418 { 419 switch (wr->opcode) { 420 case IB_WR_SEND_WITH_IMM: 421 case IB_WR_RDMA_WRITE_WITH_IMM: 422 return cpu_to_le32(be32_to_cpu(wr->ex.imm_data)); 423 default: 424 return 0; 425 } 426 } 427 428 static int set_ud_opcode(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe, 429 const struct ib_send_wr *wr) 430 { 431 u32 ib_op = wr->opcode; 432 433 if (ib_op != IB_WR_SEND && ib_op != IB_WR_SEND_WITH_IMM) 434 return -EINVAL; 435 436 ud_sq_wqe->immtdata = get_immtdata(wr); 437 438 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_OPCODE, to_hr_opcode(ib_op)); 439 440 return 0; 441 } 442 443 static int fill_ud_av(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe, 444 struct hns_roce_ah *ah) 445 { 446 struct ib_device *ib_dev = ah->ibah.device; 447 struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev); 448 449 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_UDPSPN, ah->av.udp_sport); 450 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_HOPLIMIT, ah->av.hop_limit); 451 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_TCLASS, ah->av.tclass); 452 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_FLOW_LABEL, ah->av.flowlabel); 453 454 if (WARN_ON(ah->av.sl > MAX_SERVICE_LEVEL)) 455 return -EINVAL; 456 457 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SL, ah->av.sl); 458 459 ud_sq_wqe->sgid_index = ah->av.gid_index; 460 461 memcpy(ud_sq_wqe->dmac, ah->av.mac, ETH_ALEN); 462 memcpy(ud_sq_wqe->dgid, ah->av.dgid, GID_LEN_V2); 463 464 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) 465 return 0; 466 467 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_VLAN_EN, ah->av.vlan_en); 468 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_VLAN, ah->av.vlan_id); 469 470 return 0; 471 } 472 473 static inline int set_ud_wqe(struct hns_roce_qp *qp, 474 const struct ib_send_wr *wr, 475 void *wqe, unsigned int *sge_idx, 476 unsigned int owner_bit) 477 { 478 struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah); 479 struct hns_roce_v2_ud_send_wqe *ud_sq_wqe = wqe; 480 unsigned int curr_idx = *sge_idx; 481 unsigned int valid_num_sge; 482 u32 msg_len = 0; 483 int ret; 484 485 valid_num_sge = calc_wr_sge_num(wr, &msg_len); 486 487 ret = set_ud_opcode(ud_sq_wqe, wr); 488 if (WARN_ON(ret)) 489 return ret; 490 491 ud_sq_wqe->msg_len = cpu_to_le32(msg_len); 492 493 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_CQE, 494 !!(wr->send_flags & IB_SEND_SIGNALED)); 495 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SE, 496 !!(wr->send_flags & IB_SEND_SOLICITED)); 497 498 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_PD, to_hr_pd(qp->ibqp.pd)->pdn); 499 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SGE_NUM, valid_num_sge); 500 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_MSG_START_SGE_IDX, 501 curr_idx & (qp->sge.sge_cnt - 1)); 502 503 ud_sq_wqe->qkey = cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ? 504 qp->qkey : ud_wr(wr)->remote_qkey); 505 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_DQPN, ud_wr(wr)->remote_qpn); 506 507 ret = fill_ud_av(ud_sq_wqe, ah); 508 if (ret) 509 return ret; 510 511 qp->sl = to_hr_ah(ud_wr(wr)->ah)->av.sl; 512 513 set_extend_sge(qp, wr->sg_list, &curr_idx, valid_num_sge); 514 515 /* 516 * The pipeline can sequentially post all valid WQEs into WQ buffer, 517 * including new WQEs waiting for the doorbell to update the PI again. 518 * Therefore, the owner bit of WQE MUST be updated after all fields 519 * and extSGEs have been written into DDR instead of cache. 520 */ 521 if (qp->en_flags & HNS_ROCE_QP_CAP_OWNER_DB) 522 dma_wmb(); 523 524 *sge_idx = curr_idx; 525 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_OWNER, owner_bit); 526 527 return 0; 528 } 529 530 static int set_rc_opcode(struct hns_roce_dev *hr_dev, 531 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, 532 const struct ib_send_wr *wr) 533 { 534 u32 ib_op = wr->opcode; 535 int ret = 0; 536 537 rc_sq_wqe->immtdata = get_immtdata(wr); 538 539 switch (ib_op) { 540 case IB_WR_RDMA_READ: 541 case IB_WR_RDMA_WRITE: 542 case IB_WR_RDMA_WRITE_WITH_IMM: 543 rc_sq_wqe->rkey = cpu_to_le32(rdma_wr(wr)->rkey); 544 rc_sq_wqe->va = cpu_to_le64(rdma_wr(wr)->remote_addr); 545 break; 546 case IB_WR_SEND: 547 case IB_WR_SEND_WITH_IMM: 548 break; 549 case IB_WR_ATOMIC_CMP_AND_SWP: 550 case IB_WR_ATOMIC_FETCH_AND_ADD: 551 rc_sq_wqe->rkey = cpu_to_le32(atomic_wr(wr)->rkey); 552 rc_sq_wqe->va = cpu_to_le64(atomic_wr(wr)->remote_addr); 553 break; 554 case IB_WR_REG_MR: 555 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) 556 set_frmr_seg(rc_sq_wqe, reg_wr(wr)); 557 else 558 ret = -EOPNOTSUPP; 559 break; 560 case IB_WR_SEND_WITH_INV: 561 rc_sq_wqe->inv_key = cpu_to_le32(wr->ex.invalidate_rkey); 562 break; 563 default: 564 ret = -EINVAL; 565 } 566 567 if (unlikely(ret)) 568 return ret; 569 570 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_OPCODE, to_hr_opcode(ib_op)); 571 572 return ret; 573 } 574 575 static inline int set_rc_wqe(struct hns_roce_qp *qp, 576 const struct ib_send_wr *wr, 577 void *wqe, unsigned int *sge_idx, 578 unsigned int owner_bit) 579 { 580 struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device); 581 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe = wqe; 582 unsigned int curr_idx = *sge_idx; 583 unsigned int valid_num_sge; 584 u32 msg_len = 0; 585 int ret; 586 587 valid_num_sge = calc_wr_sge_num(wr, &msg_len); 588 589 rc_sq_wqe->msg_len = cpu_to_le32(msg_len); 590 591 ret = set_rc_opcode(hr_dev, rc_sq_wqe, wr); 592 if (WARN_ON(ret)) 593 return ret; 594 595 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_FENCE, 596 (wr->send_flags & IB_SEND_FENCE) ? 1 : 0); 597 598 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_SE, 599 (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0); 600 601 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_CQE, 602 (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0); 603 604 if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP || 605 wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) 606 set_atomic_seg(wr, rc_sq_wqe, valid_num_sge); 607 else if (wr->opcode != IB_WR_REG_MR) 608 ret = set_rwqe_data_seg(&qp->ibqp, wr, rc_sq_wqe, 609 &curr_idx, valid_num_sge); 610 611 /* 612 * The pipeline can sequentially post all valid WQEs into WQ buffer, 613 * including new WQEs waiting for the doorbell to update the PI again. 614 * Therefore, the owner bit of WQE MUST be updated after all fields 615 * and extSGEs have been written into DDR instead of cache. 616 */ 617 if (qp->en_flags & HNS_ROCE_QP_CAP_OWNER_DB) 618 dma_wmb(); 619 620 *sge_idx = curr_idx; 621 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_OWNER, owner_bit); 622 623 return ret; 624 } 625 626 static inline void update_sq_db(struct hns_roce_dev *hr_dev, 627 struct hns_roce_qp *qp) 628 { 629 if (unlikely(qp->state == IB_QPS_ERR)) { 630 flush_cqe(hr_dev, qp); 631 } else { 632 struct hns_roce_v2_db sq_db = {}; 633 634 hr_reg_write(&sq_db, DB_TAG, qp->qpn); 635 hr_reg_write(&sq_db, DB_CMD, HNS_ROCE_V2_SQ_DB); 636 hr_reg_write(&sq_db, DB_PI, qp->sq.head); 637 hr_reg_write(&sq_db, DB_SL, qp->sl); 638 639 hns_roce_write64(hr_dev, (__le32 *)&sq_db, qp->sq.db_reg); 640 } 641 } 642 643 static inline void update_rq_db(struct hns_roce_dev *hr_dev, 644 struct hns_roce_qp *qp) 645 { 646 if (unlikely(qp->state == IB_QPS_ERR)) { 647 flush_cqe(hr_dev, qp); 648 } else { 649 if (likely(qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)) { 650 *qp->rdb.db_record = 651 qp->rq.head & V2_DB_PRODUCER_IDX_M; 652 } else { 653 struct hns_roce_v2_db rq_db = {}; 654 655 hr_reg_write(&rq_db, DB_TAG, qp->qpn); 656 hr_reg_write(&rq_db, DB_CMD, HNS_ROCE_V2_RQ_DB); 657 hr_reg_write(&rq_db, DB_PI, qp->rq.head); 658 659 hns_roce_write64(hr_dev, (__le32 *)&rq_db, 660 qp->rq.db_reg); 661 } 662 } 663 } 664 665 static void hns_roce_write512(struct hns_roce_dev *hr_dev, u64 *val, 666 u64 __iomem *dest) 667 { 668 #define HNS_ROCE_WRITE_TIMES 8 669 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv; 670 struct hnae3_handle *handle = priv->handle; 671 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 672 int i; 673 674 if (!hr_dev->dis_db && !ops->get_hw_reset_stat(handle)) 675 for (i = 0; i < HNS_ROCE_WRITE_TIMES; i++) 676 writeq_relaxed(*(val + i), dest + i); 677 } 678 679 static void write_dwqe(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp, 680 void *wqe) 681 { 682 #define HNS_ROCE_SL_SHIFT 2 683 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe = wqe; 684 685 /* All kinds of DirectWQE have the same header field layout */ 686 hr_reg_enable(rc_sq_wqe, RC_SEND_WQE_FLAG); 687 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_DB_SL_L, qp->sl); 688 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_DB_SL_H, 689 qp->sl >> HNS_ROCE_SL_SHIFT); 690 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_WQE_INDEX, qp->sq.head); 691 692 hns_roce_write512(hr_dev, wqe, qp->sq.db_reg); 693 } 694 695 static int hns_roce_v2_post_send(struct ib_qp *ibqp, 696 const struct ib_send_wr *wr, 697 const struct ib_send_wr **bad_wr) 698 { 699 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 700 struct ib_device *ibdev = &hr_dev->ib_dev; 701 struct hns_roce_qp *qp = to_hr_qp(ibqp); 702 unsigned long flags = 0; 703 unsigned int owner_bit; 704 unsigned int sge_idx; 705 unsigned int wqe_idx; 706 void *wqe = NULL; 707 u32 nreq; 708 int ret; 709 710 spin_lock_irqsave(&qp->sq.lock, flags); 711 712 ret = check_send_valid(hr_dev, qp); 713 if (unlikely(ret)) { 714 *bad_wr = wr; 715 nreq = 0; 716 goto out; 717 } 718 719 sge_idx = qp->next_sge; 720 721 for (nreq = 0; wr; ++nreq, wr = wr->next) { 722 if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) { 723 ret = -ENOMEM; 724 *bad_wr = wr; 725 goto out; 726 } 727 728 wqe_idx = (qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1); 729 730 if (unlikely(wr->num_sge > qp->sq.max_gs)) { 731 ibdev_err(ibdev, "num_sge = %d > qp->sq.max_gs = %u.\n", 732 wr->num_sge, qp->sq.max_gs); 733 ret = -EINVAL; 734 *bad_wr = wr; 735 goto out; 736 } 737 738 wqe = hns_roce_get_send_wqe(qp, wqe_idx); 739 qp->sq.wrid[wqe_idx] = wr->wr_id; 740 owner_bit = 741 ~(((qp->sq.head + nreq) >> ilog2(qp->sq.wqe_cnt)) & 0x1); 742 743 /* Corresponding to the QP type, wqe process separately */ 744 if (ibqp->qp_type == IB_QPT_RC) 745 ret = set_rc_wqe(qp, wr, wqe, &sge_idx, owner_bit); 746 else 747 ret = set_ud_wqe(qp, wr, wqe, &sge_idx, owner_bit); 748 749 if (unlikely(ret)) { 750 *bad_wr = wr; 751 goto out; 752 } 753 } 754 755 out: 756 if (likely(nreq)) { 757 qp->sq.head += nreq; 758 qp->next_sge = sge_idx; 759 760 if (nreq == 1 && (qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE)) 761 write_dwqe(hr_dev, qp, wqe); 762 else 763 update_sq_db(hr_dev, qp); 764 } 765 766 spin_unlock_irqrestore(&qp->sq.lock, flags); 767 768 return ret; 769 } 770 771 static int check_recv_valid(struct hns_roce_dev *hr_dev, 772 struct hns_roce_qp *hr_qp) 773 { 774 struct ib_device *ibdev = &hr_dev->ib_dev; 775 struct ib_qp *ibqp = &hr_qp->ibqp; 776 777 if (unlikely(ibqp->qp_type != IB_QPT_RC && 778 ibqp->qp_type != IB_QPT_GSI && 779 ibqp->qp_type != IB_QPT_UD)) { 780 ibdev_err(ibdev, "unsupported qp type, qp_type = %d.\n", 781 ibqp->qp_type); 782 return -EOPNOTSUPP; 783 } 784 785 if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN)) 786 return -EIO; 787 788 if (hr_qp->state == IB_QPS_RESET) 789 return -EINVAL; 790 791 return 0; 792 } 793 794 static void fill_recv_sge_to_wqe(const struct ib_recv_wr *wr, void *wqe, 795 u32 max_sge, bool rsv) 796 { 797 struct hns_roce_v2_wqe_data_seg *dseg = wqe; 798 u32 i, cnt; 799 800 for (i = 0, cnt = 0; i < wr->num_sge; i++) { 801 /* Skip zero-length sge */ 802 if (!wr->sg_list[i].length) 803 continue; 804 set_data_seg_v2(dseg + cnt, wr->sg_list + i); 805 cnt++; 806 } 807 808 /* Fill a reserved sge to make hw stop reading remaining segments */ 809 if (rsv) { 810 dseg[cnt].lkey = cpu_to_le32(HNS_ROCE_INVALID_LKEY); 811 dseg[cnt].addr = 0; 812 dseg[cnt].len = cpu_to_le32(HNS_ROCE_INVALID_SGE_LENGTH); 813 } else { 814 /* Clear remaining segments to make ROCEE ignore sges */ 815 if (cnt < max_sge) 816 memset(dseg + cnt, 0, 817 (max_sge - cnt) * HNS_ROCE_SGE_SIZE); 818 } 819 } 820 821 static void fill_rq_wqe(struct hns_roce_qp *hr_qp, const struct ib_recv_wr *wr, 822 u32 wqe_idx, u32 max_sge) 823 { 824 struct hns_roce_rinl_sge *sge_list; 825 void *wqe = NULL; 826 u32 i; 827 828 wqe = hns_roce_get_recv_wqe(hr_qp, wqe_idx); 829 fill_recv_sge_to_wqe(wr, wqe, max_sge, hr_qp->rq.rsv_sge); 830 831 /* rq support inline data */ 832 if (hr_qp->rq_inl_buf.wqe_cnt) { 833 sge_list = hr_qp->rq_inl_buf.wqe_list[wqe_idx].sg_list; 834 hr_qp->rq_inl_buf.wqe_list[wqe_idx].sge_cnt = (u32)wr->num_sge; 835 for (i = 0; i < wr->num_sge; i++) { 836 sge_list[i].addr = (void *)(u64)wr->sg_list[i].addr; 837 sge_list[i].len = wr->sg_list[i].length; 838 } 839 } 840 } 841 842 static int hns_roce_v2_post_recv(struct ib_qp *ibqp, 843 const struct ib_recv_wr *wr, 844 const struct ib_recv_wr **bad_wr) 845 { 846 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 847 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 848 struct ib_device *ibdev = &hr_dev->ib_dev; 849 u32 wqe_idx, nreq, max_sge; 850 unsigned long flags; 851 int ret; 852 853 spin_lock_irqsave(&hr_qp->rq.lock, flags); 854 855 ret = check_recv_valid(hr_dev, hr_qp); 856 if (unlikely(ret)) { 857 *bad_wr = wr; 858 nreq = 0; 859 goto out; 860 } 861 862 max_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge; 863 for (nreq = 0; wr; ++nreq, wr = wr->next) { 864 if (unlikely(hns_roce_wq_overflow(&hr_qp->rq, nreq, 865 hr_qp->ibqp.recv_cq))) { 866 ret = -ENOMEM; 867 *bad_wr = wr; 868 goto out; 869 } 870 871 if (unlikely(wr->num_sge > max_sge)) { 872 ibdev_err(ibdev, "num_sge = %d >= max_sge = %u.\n", 873 wr->num_sge, max_sge); 874 ret = -EINVAL; 875 *bad_wr = wr; 876 goto out; 877 } 878 879 wqe_idx = (hr_qp->rq.head + nreq) & (hr_qp->rq.wqe_cnt - 1); 880 fill_rq_wqe(hr_qp, wr, wqe_idx, max_sge); 881 hr_qp->rq.wrid[wqe_idx] = wr->wr_id; 882 } 883 884 out: 885 if (likely(nreq)) { 886 hr_qp->rq.head += nreq; 887 888 update_rq_db(hr_dev, hr_qp); 889 } 890 spin_unlock_irqrestore(&hr_qp->rq.lock, flags); 891 892 return ret; 893 } 894 895 static void *get_srq_wqe_buf(struct hns_roce_srq *srq, u32 n) 896 { 897 return hns_roce_buf_offset(srq->buf_mtr.kmem, n << srq->wqe_shift); 898 } 899 900 static void *get_idx_buf(struct hns_roce_idx_que *idx_que, u32 n) 901 { 902 return hns_roce_buf_offset(idx_que->mtr.kmem, 903 n << idx_que->entry_shift); 904 } 905 906 static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, u32 wqe_index) 907 { 908 /* always called with interrupts disabled. */ 909 spin_lock(&srq->lock); 910 911 bitmap_clear(srq->idx_que.bitmap, wqe_index, 1); 912 srq->idx_que.tail++; 913 914 spin_unlock(&srq->lock); 915 } 916 917 static int hns_roce_srqwq_overflow(struct hns_roce_srq *srq) 918 { 919 struct hns_roce_idx_que *idx_que = &srq->idx_que; 920 921 return idx_que->head - idx_que->tail >= srq->wqe_cnt; 922 } 923 924 static int check_post_srq_valid(struct hns_roce_srq *srq, u32 max_sge, 925 const struct ib_recv_wr *wr) 926 { 927 struct ib_device *ib_dev = srq->ibsrq.device; 928 929 if (unlikely(wr->num_sge > max_sge)) { 930 ibdev_err(ib_dev, 931 "failed to check sge, wr->num_sge = %d, max_sge = %u.\n", 932 wr->num_sge, max_sge); 933 return -EINVAL; 934 } 935 936 if (unlikely(hns_roce_srqwq_overflow(srq))) { 937 ibdev_err(ib_dev, 938 "failed to check srqwq status, srqwq is full.\n"); 939 return -ENOMEM; 940 } 941 942 return 0; 943 } 944 945 static int get_srq_wqe_idx(struct hns_roce_srq *srq, u32 *wqe_idx) 946 { 947 struct hns_roce_idx_que *idx_que = &srq->idx_que; 948 u32 pos; 949 950 pos = find_first_zero_bit(idx_que->bitmap, srq->wqe_cnt); 951 if (unlikely(pos == srq->wqe_cnt)) 952 return -ENOSPC; 953 954 bitmap_set(idx_que->bitmap, pos, 1); 955 *wqe_idx = pos; 956 return 0; 957 } 958 959 static void fill_wqe_idx(struct hns_roce_srq *srq, unsigned int wqe_idx) 960 { 961 struct hns_roce_idx_que *idx_que = &srq->idx_que; 962 unsigned int head; 963 __le32 *buf; 964 965 head = idx_que->head & (srq->wqe_cnt - 1); 966 967 buf = get_idx_buf(idx_que, head); 968 *buf = cpu_to_le32(wqe_idx); 969 970 idx_que->head++; 971 } 972 973 static void update_srq_db(struct hns_roce_v2_db *db, struct hns_roce_srq *srq) 974 { 975 hr_reg_write(db, DB_TAG, srq->srqn); 976 hr_reg_write(db, DB_CMD, HNS_ROCE_V2_SRQ_DB); 977 hr_reg_write(db, DB_PI, srq->idx_que.head); 978 } 979 980 static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq, 981 const struct ib_recv_wr *wr, 982 const struct ib_recv_wr **bad_wr) 983 { 984 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device); 985 struct hns_roce_srq *srq = to_hr_srq(ibsrq); 986 struct hns_roce_v2_db srq_db; 987 unsigned long flags; 988 int ret = 0; 989 u32 max_sge; 990 u32 wqe_idx; 991 void *wqe; 992 u32 nreq; 993 994 spin_lock_irqsave(&srq->lock, flags); 995 996 max_sge = srq->max_gs - srq->rsv_sge; 997 for (nreq = 0; wr; ++nreq, wr = wr->next) { 998 ret = check_post_srq_valid(srq, max_sge, wr); 999 if (ret) { 1000 *bad_wr = wr; 1001 break; 1002 } 1003 1004 ret = get_srq_wqe_idx(srq, &wqe_idx); 1005 if (unlikely(ret)) { 1006 *bad_wr = wr; 1007 break; 1008 } 1009 1010 wqe = get_srq_wqe_buf(srq, wqe_idx); 1011 fill_recv_sge_to_wqe(wr, wqe, max_sge, srq->rsv_sge); 1012 fill_wqe_idx(srq, wqe_idx); 1013 srq->wrid[wqe_idx] = wr->wr_id; 1014 } 1015 1016 if (likely(nreq)) { 1017 update_srq_db(&srq_db, srq); 1018 1019 hns_roce_write64(hr_dev, (__le32 *)&srq_db, srq->db_reg); 1020 } 1021 1022 spin_unlock_irqrestore(&srq->lock, flags); 1023 1024 return ret; 1025 } 1026 1027 static u32 hns_roce_v2_cmd_hw_reseted(struct hns_roce_dev *hr_dev, 1028 unsigned long instance_stage, 1029 unsigned long reset_stage) 1030 { 1031 /* When hardware reset has been completed once or more, we should stop 1032 * sending mailbox&cmq&doorbell to hardware. If now in .init_instance() 1033 * function, we should exit with error. If now at HNAE3_INIT_CLIENT 1034 * stage of soft reset process, we should exit with error, and then 1035 * HNAE3_INIT_CLIENT related process can rollback the operation like 1036 * notifing hardware to free resources, HNAE3_INIT_CLIENT related 1037 * process will exit with error to notify NIC driver to reschedule soft 1038 * reset process once again. 1039 */ 1040 hr_dev->is_reset = true; 1041 hr_dev->dis_db = true; 1042 1043 if (reset_stage == HNS_ROCE_STATE_RST_INIT || 1044 instance_stage == HNS_ROCE_STATE_INIT) 1045 return CMD_RST_PRC_EBUSY; 1046 1047 return CMD_RST_PRC_SUCCESS; 1048 } 1049 1050 static u32 hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev, 1051 unsigned long instance_stage, 1052 unsigned long reset_stage) 1053 { 1054 #define HW_RESET_TIMEOUT_US 1000000 1055 #define HW_RESET_SLEEP_US 1000 1056 1057 struct hns_roce_v2_priv *priv = hr_dev->priv; 1058 struct hnae3_handle *handle = priv->handle; 1059 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 1060 unsigned long val; 1061 int ret; 1062 1063 /* When hardware reset is detected, we should stop sending mailbox&cmq& 1064 * doorbell to hardware. If now in .init_instance() function, we should 1065 * exit with error. If now at HNAE3_INIT_CLIENT stage of soft reset 1066 * process, we should exit with error, and then HNAE3_INIT_CLIENT 1067 * related process can rollback the operation like notifing hardware to 1068 * free resources, HNAE3_INIT_CLIENT related process will exit with 1069 * error to notify NIC driver to reschedule soft reset process once 1070 * again. 1071 */ 1072 hr_dev->dis_db = true; 1073 1074 ret = read_poll_timeout(ops->ae_dev_reset_cnt, val, 1075 val > hr_dev->reset_cnt, HW_RESET_SLEEP_US, 1076 HW_RESET_TIMEOUT_US, false, handle); 1077 if (!ret) 1078 hr_dev->is_reset = true; 1079 1080 if (!hr_dev->is_reset || reset_stage == HNS_ROCE_STATE_RST_INIT || 1081 instance_stage == HNS_ROCE_STATE_INIT) 1082 return CMD_RST_PRC_EBUSY; 1083 1084 return CMD_RST_PRC_SUCCESS; 1085 } 1086 1087 static u32 hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev *hr_dev) 1088 { 1089 struct hns_roce_v2_priv *priv = hr_dev->priv; 1090 struct hnae3_handle *handle = priv->handle; 1091 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 1092 1093 /* When software reset is detected at .init_instance() function, we 1094 * should stop sending mailbox&cmq&doorbell to hardware, and exit 1095 * with error. 1096 */ 1097 hr_dev->dis_db = true; 1098 if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt) 1099 hr_dev->is_reset = true; 1100 1101 return CMD_RST_PRC_EBUSY; 1102 } 1103 1104 static u32 check_aedev_reset_status(struct hns_roce_dev *hr_dev, 1105 struct hnae3_handle *handle) 1106 { 1107 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 1108 unsigned long instance_stage; /* the current instance stage */ 1109 unsigned long reset_stage; /* the current reset stage */ 1110 unsigned long reset_cnt; 1111 bool sw_resetting; 1112 bool hw_resetting; 1113 1114 /* Get information about reset from NIC driver or RoCE driver itself, 1115 * the meaning of the following variables from NIC driver are described 1116 * as below: 1117 * reset_cnt -- The count value of completed hardware reset. 1118 * hw_resetting -- Whether hardware device is resetting now. 1119 * sw_resetting -- Whether NIC's software reset process is running now. 1120 */ 1121 instance_stage = handle->rinfo.instance_state; 1122 reset_stage = handle->rinfo.reset_state; 1123 reset_cnt = ops->ae_dev_reset_cnt(handle); 1124 if (reset_cnt != hr_dev->reset_cnt) 1125 return hns_roce_v2_cmd_hw_reseted(hr_dev, instance_stage, 1126 reset_stage); 1127 1128 hw_resetting = ops->get_cmdq_stat(handle); 1129 if (hw_resetting) 1130 return hns_roce_v2_cmd_hw_resetting(hr_dev, instance_stage, 1131 reset_stage); 1132 1133 sw_resetting = ops->ae_dev_resetting(handle); 1134 if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT) 1135 return hns_roce_v2_cmd_sw_resetting(hr_dev); 1136 1137 return CMD_RST_PRC_OTHERS; 1138 } 1139 1140 static bool check_device_is_in_reset(struct hns_roce_dev *hr_dev) 1141 { 1142 struct hns_roce_v2_priv *priv = hr_dev->priv; 1143 struct hnae3_handle *handle = priv->handle; 1144 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 1145 1146 if (hr_dev->reset_cnt != ops->ae_dev_reset_cnt(handle)) 1147 return true; 1148 1149 if (ops->get_hw_reset_stat(handle)) 1150 return true; 1151 1152 if (ops->ae_dev_resetting(handle)) 1153 return true; 1154 1155 return false; 1156 } 1157 1158 static bool v2_chk_mbox_is_avail(struct hns_roce_dev *hr_dev, bool *busy) 1159 { 1160 struct hns_roce_v2_priv *priv = hr_dev->priv; 1161 u32 status; 1162 1163 if (hr_dev->is_reset) 1164 status = CMD_RST_PRC_SUCCESS; 1165 else 1166 status = check_aedev_reset_status(hr_dev, priv->handle); 1167 1168 *busy = (status == CMD_RST_PRC_EBUSY); 1169 1170 return status == CMD_RST_PRC_OTHERS; 1171 } 1172 1173 static int hns_roce_alloc_cmq_desc(struct hns_roce_dev *hr_dev, 1174 struct hns_roce_v2_cmq_ring *ring) 1175 { 1176 int size = ring->desc_num * sizeof(struct hns_roce_cmq_desc); 1177 1178 ring->desc = dma_alloc_coherent(hr_dev->dev, size, 1179 &ring->desc_dma_addr, GFP_KERNEL); 1180 if (!ring->desc) 1181 return -ENOMEM; 1182 1183 return 0; 1184 } 1185 1186 static void hns_roce_free_cmq_desc(struct hns_roce_dev *hr_dev, 1187 struct hns_roce_v2_cmq_ring *ring) 1188 { 1189 dma_free_coherent(hr_dev->dev, 1190 ring->desc_num * sizeof(struct hns_roce_cmq_desc), 1191 ring->desc, ring->desc_dma_addr); 1192 1193 ring->desc_dma_addr = 0; 1194 } 1195 1196 static int init_csq(struct hns_roce_dev *hr_dev, 1197 struct hns_roce_v2_cmq_ring *csq) 1198 { 1199 dma_addr_t dma; 1200 int ret; 1201 1202 csq->desc_num = CMD_CSQ_DESC_NUM; 1203 spin_lock_init(&csq->lock); 1204 csq->flag = TYPE_CSQ; 1205 csq->head = 0; 1206 1207 ret = hns_roce_alloc_cmq_desc(hr_dev, csq); 1208 if (ret) 1209 return ret; 1210 1211 dma = csq->desc_dma_addr; 1212 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_L_REG, lower_32_bits(dma)); 1213 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_H_REG, upper_32_bits(dma)); 1214 roce_write(hr_dev, ROCEE_TX_CMQ_DEPTH_REG, 1215 (u32)csq->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S); 1216 1217 /* Make sure to write CI first and then PI */ 1218 roce_write(hr_dev, ROCEE_TX_CMQ_CI_REG, 0); 1219 roce_write(hr_dev, ROCEE_TX_CMQ_PI_REG, 0); 1220 1221 return 0; 1222 } 1223 1224 static int hns_roce_v2_cmq_init(struct hns_roce_dev *hr_dev) 1225 { 1226 struct hns_roce_v2_priv *priv = hr_dev->priv; 1227 int ret; 1228 1229 priv->cmq.tx_timeout = HNS_ROCE_CMQ_TX_TIMEOUT; 1230 1231 ret = init_csq(hr_dev, &priv->cmq.csq); 1232 if (ret) 1233 dev_err(hr_dev->dev, "failed to init CSQ, ret = %d.\n", ret); 1234 1235 return ret; 1236 } 1237 1238 static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev) 1239 { 1240 struct hns_roce_v2_priv *priv = hr_dev->priv; 1241 1242 hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq); 1243 } 1244 1245 static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc, 1246 enum hns_roce_opcode_type opcode, 1247 bool is_read) 1248 { 1249 memset((void *)desc, 0, sizeof(struct hns_roce_cmq_desc)); 1250 desc->opcode = cpu_to_le16(opcode); 1251 desc->flag = cpu_to_le16(HNS_ROCE_CMD_FLAG_IN); 1252 if (is_read) 1253 desc->flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_WR); 1254 else 1255 desc->flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR); 1256 } 1257 1258 static int hns_roce_cmq_csq_done(struct hns_roce_dev *hr_dev) 1259 { 1260 u32 tail = roce_read(hr_dev, ROCEE_TX_CMQ_CI_REG); 1261 struct hns_roce_v2_priv *priv = hr_dev->priv; 1262 1263 return tail == priv->cmq.csq.head; 1264 } 1265 1266 static void update_cmdq_status(struct hns_roce_dev *hr_dev) 1267 { 1268 struct hns_roce_v2_priv *priv = hr_dev->priv; 1269 struct hnae3_handle *handle = priv->handle; 1270 1271 if (handle->rinfo.reset_state == HNS_ROCE_STATE_RST_INIT || 1272 handle->rinfo.instance_state == HNS_ROCE_STATE_INIT) 1273 hr_dev->cmd.state = HNS_ROCE_CMDQ_STATE_FATAL_ERR; 1274 } 1275 1276 static int hns_roce_cmd_err_convert_errno(u16 desc_ret) 1277 { 1278 struct hns_roce_cmd_errcode errcode_table[] = { 1279 {CMD_EXEC_SUCCESS, 0}, 1280 {CMD_NO_AUTH, -EPERM}, 1281 {CMD_NOT_EXIST, -EOPNOTSUPP}, 1282 {CMD_CRQ_FULL, -EXFULL}, 1283 {CMD_NEXT_ERR, -ENOSR}, 1284 {CMD_NOT_EXEC, -ENOTBLK}, 1285 {CMD_PARA_ERR, -EINVAL}, 1286 {CMD_RESULT_ERR, -ERANGE}, 1287 {CMD_TIMEOUT, -ETIME}, 1288 {CMD_HILINK_ERR, -ENOLINK}, 1289 {CMD_INFO_ILLEGAL, -ENXIO}, 1290 {CMD_INVALID, -EBADR}, 1291 }; 1292 u16 i; 1293 1294 for (i = 0; i < ARRAY_SIZE(errcode_table); i++) 1295 if (desc_ret == errcode_table[i].return_status) 1296 return errcode_table[i].errno; 1297 return -EIO; 1298 } 1299 1300 static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev, 1301 struct hns_roce_cmq_desc *desc, int num) 1302 { 1303 struct hns_roce_v2_priv *priv = hr_dev->priv; 1304 struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq; 1305 u32 timeout = 0; 1306 u16 desc_ret; 1307 u32 tail; 1308 int ret; 1309 int i; 1310 1311 spin_lock_bh(&csq->lock); 1312 1313 tail = csq->head; 1314 1315 for (i = 0; i < num; i++) { 1316 csq->desc[csq->head++] = desc[i]; 1317 if (csq->head == csq->desc_num) 1318 csq->head = 0; 1319 } 1320 1321 /* Write to hardware */ 1322 roce_write(hr_dev, ROCEE_TX_CMQ_PI_REG, csq->head); 1323 1324 do { 1325 if (hns_roce_cmq_csq_done(hr_dev)) 1326 break; 1327 udelay(1); 1328 } while (++timeout < priv->cmq.tx_timeout); 1329 1330 if (hns_roce_cmq_csq_done(hr_dev)) { 1331 ret = 0; 1332 for (i = 0; i < num; i++) { 1333 /* check the result of hardware write back */ 1334 desc[i] = csq->desc[tail++]; 1335 if (tail == csq->desc_num) 1336 tail = 0; 1337 1338 desc_ret = le16_to_cpu(desc[i].retval); 1339 if (likely(desc_ret == CMD_EXEC_SUCCESS)) 1340 continue; 1341 1342 dev_err_ratelimited(hr_dev->dev, 1343 "Cmdq IO error, opcode = 0x%x, return = 0x%x.\n", 1344 desc->opcode, desc_ret); 1345 ret = hns_roce_cmd_err_convert_errno(desc_ret); 1346 } 1347 } else { 1348 /* FW/HW reset or incorrect number of desc */ 1349 tail = roce_read(hr_dev, ROCEE_TX_CMQ_CI_REG); 1350 dev_warn(hr_dev->dev, "CMDQ move tail from %u to %u.\n", 1351 csq->head, tail); 1352 csq->head = tail; 1353 1354 update_cmdq_status(hr_dev); 1355 1356 ret = -EAGAIN; 1357 } 1358 1359 spin_unlock_bh(&csq->lock); 1360 1361 return ret; 1362 } 1363 1364 static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev, 1365 struct hns_roce_cmq_desc *desc, int num) 1366 { 1367 bool busy; 1368 int ret; 1369 1370 if (hr_dev->cmd.state == HNS_ROCE_CMDQ_STATE_FATAL_ERR) 1371 return -EIO; 1372 1373 if (!v2_chk_mbox_is_avail(hr_dev, &busy)) 1374 return busy ? -EBUSY : 0; 1375 1376 ret = __hns_roce_cmq_send(hr_dev, desc, num); 1377 if (ret) { 1378 if (!v2_chk_mbox_is_avail(hr_dev, &busy)) 1379 return busy ? -EBUSY : 0; 1380 } 1381 1382 return ret; 1383 } 1384 1385 static int config_hem_ba_to_hw(struct hns_roce_dev *hr_dev, 1386 dma_addr_t base_addr, u8 cmd, unsigned long tag) 1387 { 1388 struct hns_roce_cmd_mailbox *mbox; 1389 int ret; 1390 1391 mbox = hns_roce_alloc_cmd_mailbox(hr_dev); 1392 if (IS_ERR(mbox)) 1393 return PTR_ERR(mbox); 1394 1395 ret = hns_roce_cmd_mbox(hr_dev, base_addr, mbox->dma, cmd, tag); 1396 hns_roce_free_cmd_mailbox(hr_dev, mbox); 1397 return ret; 1398 } 1399 1400 static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev) 1401 { 1402 struct hns_roce_query_version *resp; 1403 struct hns_roce_cmq_desc desc; 1404 int ret; 1405 1406 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_HW_VER, true); 1407 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 1408 if (ret) 1409 return ret; 1410 1411 resp = (struct hns_roce_query_version *)desc.data; 1412 hr_dev->hw_rev = le16_to_cpu(resp->rocee_hw_version); 1413 hr_dev->vendor_id = hr_dev->pci_dev->vendor; 1414 1415 return 0; 1416 } 1417 1418 static void func_clr_hw_resetting_state(struct hns_roce_dev *hr_dev, 1419 struct hnae3_handle *handle) 1420 { 1421 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 1422 unsigned long end; 1423 1424 hr_dev->dis_db = true; 1425 1426 dev_warn(hr_dev->dev, 1427 "func clear is pending, device in resetting state.\n"); 1428 end = HNS_ROCE_V2_HW_RST_TIMEOUT; 1429 while (end) { 1430 if (!ops->get_hw_reset_stat(handle)) { 1431 hr_dev->is_reset = true; 1432 dev_info(hr_dev->dev, 1433 "func clear success after reset.\n"); 1434 return; 1435 } 1436 msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT); 1437 end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT; 1438 } 1439 1440 dev_warn(hr_dev->dev, "func clear failed.\n"); 1441 } 1442 1443 static void func_clr_sw_resetting_state(struct hns_roce_dev *hr_dev, 1444 struct hnae3_handle *handle) 1445 { 1446 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 1447 unsigned long end; 1448 1449 hr_dev->dis_db = true; 1450 1451 dev_warn(hr_dev->dev, 1452 "func clear is pending, device in resetting state.\n"); 1453 end = HNS_ROCE_V2_HW_RST_TIMEOUT; 1454 while (end) { 1455 if (ops->ae_dev_reset_cnt(handle) != 1456 hr_dev->reset_cnt) { 1457 hr_dev->is_reset = true; 1458 dev_info(hr_dev->dev, 1459 "func clear success after sw reset\n"); 1460 return; 1461 } 1462 msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT); 1463 end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT; 1464 } 1465 1466 dev_warn(hr_dev->dev, "func clear failed because of unfinished sw reset\n"); 1467 } 1468 1469 static void hns_roce_func_clr_rst_proc(struct hns_roce_dev *hr_dev, int retval, 1470 int flag) 1471 { 1472 struct hns_roce_v2_priv *priv = hr_dev->priv; 1473 struct hnae3_handle *handle = priv->handle; 1474 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 1475 1476 if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt) { 1477 hr_dev->dis_db = true; 1478 hr_dev->is_reset = true; 1479 dev_info(hr_dev->dev, "func clear success after reset.\n"); 1480 return; 1481 } 1482 1483 if (ops->get_hw_reset_stat(handle)) { 1484 func_clr_hw_resetting_state(hr_dev, handle); 1485 return; 1486 } 1487 1488 if (ops->ae_dev_resetting(handle) && 1489 handle->rinfo.instance_state == HNS_ROCE_STATE_INIT) { 1490 func_clr_sw_resetting_state(hr_dev, handle); 1491 return; 1492 } 1493 1494 if (retval && !flag) 1495 dev_warn(hr_dev->dev, 1496 "func clear read failed, ret = %d.\n", retval); 1497 1498 dev_warn(hr_dev->dev, "func clear failed.\n"); 1499 } 1500 1501 static void __hns_roce_function_clear(struct hns_roce_dev *hr_dev, int vf_id) 1502 { 1503 bool fclr_write_fail_flag = false; 1504 struct hns_roce_func_clear *resp; 1505 struct hns_roce_cmq_desc desc; 1506 unsigned long end; 1507 int ret = 0; 1508 1509 if (check_device_is_in_reset(hr_dev)) 1510 goto out; 1511 1512 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, false); 1513 resp = (struct hns_roce_func_clear *)desc.data; 1514 resp->rst_funcid_en = cpu_to_le32(vf_id); 1515 1516 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 1517 if (ret) { 1518 fclr_write_fail_flag = true; 1519 dev_err(hr_dev->dev, "func clear write failed, ret = %d.\n", 1520 ret); 1521 goto out; 1522 } 1523 1524 msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL); 1525 end = HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS; 1526 while (end) { 1527 if (check_device_is_in_reset(hr_dev)) 1528 goto out; 1529 msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT); 1530 end -= HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT; 1531 1532 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, 1533 true); 1534 1535 resp->rst_funcid_en = cpu_to_le32(vf_id); 1536 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 1537 if (ret) 1538 continue; 1539 1540 if (hr_reg_read(resp, FUNC_CLEAR_RST_FUN_DONE)) { 1541 if (vf_id == 0) 1542 hr_dev->is_reset = true; 1543 return; 1544 } 1545 } 1546 1547 out: 1548 hns_roce_func_clr_rst_proc(hr_dev, ret, fclr_write_fail_flag); 1549 } 1550 1551 static int hns_roce_free_vf_resource(struct hns_roce_dev *hr_dev, int vf_id) 1552 { 1553 enum hns_roce_opcode_type opcode = HNS_ROCE_OPC_ALLOC_VF_RES; 1554 struct hns_roce_cmq_desc desc[2]; 1555 struct hns_roce_cmq_req *req_a; 1556 1557 req_a = (struct hns_roce_cmq_req *)desc[0].data; 1558 hns_roce_cmq_setup_basic_desc(&desc[0], opcode, false); 1559 desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 1560 hns_roce_cmq_setup_basic_desc(&desc[1], opcode, false); 1561 hr_reg_write(req_a, FUNC_RES_A_VF_ID, vf_id); 1562 1563 return hns_roce_cmq_send(hr_dev, desc, 2); 1564 } 1565 1566 static void hns_roce_function_clear(struct hns_roce_dev *hr_dev) 1567 { 1568 int ret; 1569 int i; 1570 1571 if (hr_dev->cmd.state == HNS_ROCE_CMDQ_STATE_FATAL_ERR) 1572 return; 1573 1574 for (i = hr_dev->func_num - 1; i >= 0; i--) { 1575 __hns_roce_function_clear(hr_dev, i); 1576 1577 if (i == 0) 1578 continue; 1579 1580 ret = hns_roce_free_vf_resource(hr_dev, i); 1581 if (ret) 1582 ibdev_err(&hr_dev->ib_dev, 1583 "failed to free vf resource, vf_id = %d, ret = %d.\n", 1584 i, ret); 1585 } 1586 } 1587 1588 static int hns_roce_clear_extdb_list_info(struct hns_roce_dev *hr_dev) 1589 { 1590 struct hns_roce_cmq_desc desc; 1591 int ret; 1592 1593 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CLEAR_EXTDB_LIST_INFO, 1594 false); 1595 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 1596 if (ret) 1597 ibdev_err(&hr_dev->ib_dev, 1598 "failed to clear extended doorbell info, ret = %d.\n", 1599 ret); 1600 1601 return ret; 1602 } 1603 1604 static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev) 1605 { 1606 struct hns_roce_query_fw_info *resp; 1607 struct hns_roce_cmq_desc desc; 1608 int ret; 1609 1610 hns_roce_cmq_setup_basic_desc(&desc, HNS_QUERY_FW_VER, true); 1611 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 1612 if (ret) 1613 return ret; 1614 1615 resp = (struct hns_roce_query_fw_info *)desc.data; 1616 hr_dev->caps.fw_ver = (u64)(le32_to_cpu(resp->fw_ver)); 1617 1618 return 0; 1619 } 1620 1621 static int hns_roce_query_func_info(struct hns_roce_dev *hr_dev) 1622 { 1623 struct hns_roce_cmq_desc desc; 1624 int ret; 1625 1626 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) { 1627 hr_dev->func_num = 1; 1628 return 0; 1629 } 1630 1631 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_FUNC_INFO, 1632 true); 1633 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 1634 if (ret) { 1635 hr_dev->func_num = 1; 1636 return ret; 1637 } 1638 1639 hr_dev->func_num = le32_to_cpu(desc.func_info.own_func_num); 1640 hr_dev->cong_algo_tmpl_id = le32_to_cpu(desc.func_info.own_mac_id); 1641 1642 return 0; 1643 } 1644 1645 static int hns_roce_config_global_param(struct hns_roce_dev *hr_dev) 1646 { 1647 struct hns_roce_cmq_desc desc; 1648 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data; 1649 u32 clock_cycles_of_1us; 1650 1651 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GLOBAL_PARAM, 1652 false); 1653 1654 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) 1655 clock_cycles_of_1us = HNS_ROCE_1NS_CFG; 1656 else 1657 clock_cycles_of_1us = HNS_ROCE_1US_CFG; 1658 1659 hr_reg_write(req, CFG_GLOBAL_PARAM_1US_CYCLES, clock_cycles_of_1us); 1660 hr_reg_write(req, CFG_GLOBAL_PARAM_UDP_PORT, ROCE_V2_UDP_DPORT); 1661 1662 return hns_roce_cmq_send(hr_dev, &desc, 1); 1663 } 1664 1665 static int load_func_res_caps(struct hns_roce_dev *hr_dev, bool is_vf) 1666 { 1667 struct hns_roce_cmq_desc desc[2]; 1668 struct hns_roce_cmq_req *r_a = (struct hns_roce_cmq_req *)desc[0].data; 1669 struct hns_roce_cmq_req *r_b = (struct hns_roce_cmq_req *)desc[1].data; 1670 struct hns_roce_caps *caps = &hr_dev->caps; 1671 enum hns_roce_opcode_type opcode; 1672 u32 func_num; 1673 int ret; 1674 1675 if (is_vf) { 1676 opcode = HNS_ROCE_OPC_QUERY_VF_RES; 1677 func_num = 1; 1678 } else { 1679 opcode = HNS_ROCE_OPC_QUERY_PF_RES; 1680 func_num = hr_dev->func_num; 1681 } 1682 1683 hns_roce_cmq_setup_basic_desc(&desc[0], opcode, true); 1684 desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 1685 hns_roce_cmq_setup_basic_desc(&desc[1], opcode, true); 1686 1687 ret = hns_roce_cmq_send(hr_dev, desc, 2); 1688 if (ret) 1689 return ret; 1690 1691 caps->qpc_bt_num = hr_reg_read(r_a, FUNC_RES_A_QPC_BT_NUM) / func_num; 1692 caps->srqc_bt_num = hr_reg_read(r_a, FUNC_RES_A_SRQC_BT_NUM) / func_num; 1693 caps->cqc_bt_num = hr_reg_read(r_a, FUNC_RES_A_CQC_BT_NUM) / func_num; 1694 caps->mpt_bt_num = hr_reg_read(r_a, FUNC_RES_A_MPT_BT_NUM) / func_num; 1695 caps->eqc_bt_num = hr_reg_read(r_a, FUNC_RES_A_EQC_BT_NUM) / func_num; 1696 caps->smac_bt_num = hr_reg_read(r_b, FUNC_RES_B_SMAC_NUM) / func_num; 1697 caps->sgid_bt_num = hr_reg_read(r_b, FUNC_RES_B_SGID_NUM) / func_num; 1698 caps->sccc_bt_num = hr_reg_read(r_b, FUNC_RES_B_SCCC_BT_NUM) / func_num; 1699 1700 if (is_vf) { 1701 caps->sl_num = hr_reg_read(r_b, FUNC_RES_V_QID_NUM) / func_num; 1702 caps->gmv_bt_num = hr_reg_read(r_b, FUNC_RES_V_GMV_BT_NUM) / 1703 func_num; 1704 } else { 1705 caps->sl_num = hr_reg_read(r_b, FUNC_RES_B_QID_NUM) / func_num; 1706 caps->gmv_bt_num = hr_reg_read(r_b, FUNC_RES_B_GMV_BT_NUM) / 1707 func_num; 1708 } 1709 1710 return 0; 1711 } 1712 1713 static int load_ext_cfg_caps(struct hns_roce_dev *hr_dev, bool is_vf) 1714 { 1715 struct hns_roce_cmq_desc desc; 1716 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data; 1717 struct hns_roce_caps *caps = &hr_dev->caps; 1718 u32 func_num, qp_num; 1719 int ret; 1720 1721 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_EXT_CFG, true); 1722 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 1723 if (ret) 1724 return ret; 1725 1726 func_num = is_vf ? 1 : max_t(u32, 1, hr_dev->func_num); 1727 qp_num = hr_reg_read(req, EXT_CFG_QP_PI_NUM) / func_num; 1728 caps->num_pi_qps = round_down(qp_num, HNS_ROCE_QP_BANK_NUM); 1729 1730 qp_num = hr_reg_read(req, EXT_CFG_QP_NUM) / func_num; 1731 caps->num_qps = round_down(qp_num, HNS_ROCE_QP_BANK_NUM); 1732 1733 return 0; 1734 } 1735 1736 static int load_pf_timer_res_caps(struct hns_roce_dev *hr_dev) 1737 { 1738 struct hns_roce_cmq_desc desc; 1739 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data; 1740 struct hns_roce_caps *caps = &hr_dev->caps; 1741 int ret; 1742 1743 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_PF_TIMER_RES, 1744 true); 1745 1746 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 1747 if (ret) 1748 return ret; 1749 1750 caps->qpc_timer_bt_num = hr_reg_read(req, PF_TIMER_RES_QPC_ITEM_NUM); 1751 caps->cqc_timer_bt_num = hr_reg_read(req, PF_TIMER_RES_CQC_ITEM_NUM); 1752 1753 return 0; 1754 } 1755 1756 static int query_func_resource_caps(struct hns_roce_dev *hr_dev, bool is_vf) 1757 { 1758 struct device *dev = hr_dev->dev; 1759 int ret; 1760 1761 ret = load_func_res_caps(hr_dev, is_vf); 1762 if (ret) { 1763 dev_err(dev, "failed to load res caps, ret = %d (%s).\n", ret, 1764 is_vf ? "vf" : "pf"); 1765 return ret; 1766 } 1767 1768 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) { 1769 ret = load_ext_cfg_caps(hr_dev, is_vf); 1770 if (ret) 1771 dev_err(dev, "failed to load ext cfg, ret = %d (%s).\n", 1772 ret, is_vf ? "vf" : "pf"); 1773 } 1774 1775 return ret; 1776 } 1777 1778 static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev) 1779 { 1780 struct device *dev = hr_dev->dev; 1781 int ret; 1782 1783 ret = query_func_resource_caps(hr_dev, false); 1784 if (ret) 1785 return ret; 1786 1787 ret = load_pf_timer_res_caps(hr_dev); 1788 if (ret) 1789 dev_err(dev, "failed to load pf timer resource, ret = %d.\n", 1790 ret); 1791 1792 return ret; 1793 } 1794 1795 static int hns_roce_query_vf_resource(struct hns_roce_dev *hr_dev) 1796 { 1797 return query_func_resource_caps(hr_dev, true); 1798 } 1799 1800 static int __hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev, 1801 u32 vf_id) 1802 { 1803 struct hns_roce_vf_switch *swt; 1804 struct hns_roce_cmq_desc desc; 1805 int ret; 1806 1807 swt = (struct hns_roce_vf_switch *)desc.data; 1808 hns_roce_cmq_setup_basic_desc(&desc, HNS_SWITCH_PARAMETER_CFG, true); 1809 swt->rocee_sel |= cpu_to_le32(HNS_ICL_SWITCH_CMD_ROCEE_SEL); 1810 hr_reg_write(swt, VF_SWITCH_VF_ID, vf_id); 1811 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 1812 if (ret) 1813 return ret; 1814 1815 desc.flag = cpu_to_le16(HNS_ROCE_CMD_FLAG_IN); 1816 desc.flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR); 1817 hr_reg_enable(swt, VF_SWITCH_ALW_LPBK); 1818 hr_reg_clear(swt, VF_SWITCH_ALW_LCL_LPBK); 1819 hr_reg_enable(swt, VF_SWITCH_ALW_DST_OVRD); 1820 1821 return hns_roce_cmq_send(hr_dev, &desc, 1); 1822 } 1823 1824 static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev) 1825 { 1826 u32 vf_id; 1827 int ret; 1828 1829 for (vf_id = 0; vf_id < hr_dev->func_num; vf_id++) { 1830 ret = __hns_roce_set_vf_switch_param(hr_dev, vf_id); 1831 if (ret) 1832 return ret; 1833 } 1834 return 0; 1835 } 1836 1837 static int config_vf_hem_resource(struct hns_roce_dev *hr_dev, int vf_id) 1838 { 1839 struct hns_roce_cmq_desc desc[2]; 1840 struct hns_roce_cmq_req *r_a = (struct hns_roce_cmq_req *)desc[0].data; 1841 struct hns_roce_cmq_req *r_b = (struct hns_roce_cmq_req *)desc[1].data; 1842 enum hns_roce_opcode_type opcode = HNS_ROCE_OPC_ALLOC_VF_RES; 1843 struct hns_roce_caps *caps = &hr_dev->caps; 1844 1845 hns_roce_cmq_setup_basic_desc(&desc[0], opcode, false); 1846 desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 1847 hns_roce_cmq_setup_basic_desc(&desc[1], opcode, false); 1848 1849 hr_reg_write(r_a, FUNC_RES_A_VF_ID, vf_id); 1850 1851 hr_reg_write(r_a, FUNC_RES_A_QPC_BT_NUM, caps->qpc_bt_num); 1852 hr_reg_write(r_a, FUNC_RES_A_QPC_BT_IDX, vf_id * caps->qpc_bt_num); 1853 hr_reg_write(r_a, FUNC_RES_A_SRQC_BT_NUM, caps->srqc_bt_num); 1854 hr_reg_write(r_a, FUNC_RES_A_SRQC_BT_IDX, vf_id * caps->srqc_bt_num); 1855 hr_reg_write(r_a, FUNC_RES_A_CQC_BT_NUM, caps->cqc_bt_num); 1856 hr_reg_write(r_a, FUNC_RES_A_CQC_BT_IDX, vf_id * caps->cqc_bt_num); 1857 hr_reg_write(r_a, FUNC_RES_A_MPT_BT_NUM, caps->mpt_bt_num); 1858 hr_reg_write(r_a, FUNC_RES_A_MPT_BT_IDX, vf_id * caps->mpt_bt_num); 1859 hr_reg_write(r_a, FUNC_RES_A_EQC_BT_NUM, caps->eqc_bt_num); 1860 hr_reg_write(r_a, FUNC_RES_A_EQC_BT_IDX, vf_id * caps->eqc_bt_num); 1861 hr_reg_write(r_b, FUNC_RES_V_QID_NUM, caps->sl_num); 1862 hr_reg_write(r_b, FUNC_RES_B_QID_IDX, vf_id * caps->sl_num); 1863 hr_reg_write(r_b, FUNC_RES_B_SCCC_BT_NUM, caps->sccc_bt_num); 1864 hr_reg_write(r_b, FUNC_RES_B_SCCC_BT_IDX, vf_id * caps->sccc_bt_num); 1865 1866 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) { 1867 hr_reg_write(r_b, FUNC_RES_V_GMV_BT_NUM, caps->gmv_bt_num); 1868 hr_reg_write(r_b, FUNC_RES_B_GMV_BT_IDX, 1869 vf_id * caps->gmv_bt_num); 1870 } else { 1871 hr_reg_write(r_b, FUNC_RES_B_SGID_NUM, caps->sgid_bt_num); 1872 hr_reg_write(r_b, FUNC_RES_B_SGID_IDX, 1873 vf_id * caps->sgid_bt_num); 1874 hr_reg_write(r_b, FUNC_RES_B_SMAC_NUM, caps->smac_bt_num); 1875 hr_reg_write(r_b, FUNC_RES_B_SMAC_IDX, 1876 vf_id * caps->smac_bt_num); 1877 } 1878 1879 return hns_roce_cmq_send(hr_dev, desc, 2); 1880 } 1881 1882 static int config_vf_ext_resource(struct hns_roce_dev *hr_dev, u32 vf_id) 1883 { 1884 struct hns_roce_cmq_desc desc; 1885 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data; 1886 struct hns_roce_caps *caps = &hr_dev->caps; 1887 1888 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_EXT_CFG, false); 1889 1890 hr_reg_write(req, EXT_CFG_VF_ID, vf_id); 1891 1892 hr_reg_write(req, EXT_CFG_QP_PI_NUM, caps->num_pi_qps); 1893 hr_reg_write(req, EXT_CFG_QP_PI_IDX, vf_id * caps->num_pi_qps); 1894 hr_reg_write(req, EXT_CFG_QP_NUM, caps->num_qps); 1895 hr_reg_write(req, EXT_CFG_QP_IDX, vf_id * caps->num_qps); 1896 1897 return hns_roce_cmq_send(hr_dev, &desc, 1); 1898 } 1899 1900 static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev) 1901 { 1902 u32 func_num = max_t(u32, 1, hr_dev->func_num); 1903 u32 vf_id; 1904 int ret; 1905 1906 for (vf_id = 0; vf_id < func_num; vf_id++) { 1907 ret = config_vf_hem_resource(hr_dev, vf_id); 1908 if (ret) { 1909 dev_err(hr_dev->dev, 1910 "failed to config vf-%u hem res, ret = %d.\n", 1911 vf_id, ret); 1912 return ret; 1913 } 1914 1915 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) { 1916 ret = config_vf_ext_resource(hr_dev, vf_id); 1917 if (ret) { 1918 dev_err(hr_dev->dev, 1919 "failed to config vf-%u ext res, ret = %d.\n", 1920 vf_id, ret); 1921 return ret; 1922 } 1923 } 1924 } 1925 1926 return 0; 1927 } 1928 1929 static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev) 1930 { 1931 struct hns_roce_cmq_desc desc; 1932 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data; 1933 struct hns_roce_caps *caps = &hr_dev->caps; 1934 1935 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_BT_ATTR, false); 1936 1937 hr_reg_write(req, CFG_BT_ATTR_QPC_BA_PGSZ, 1938 caps->qpc_ba_pg_sz + PG_SHIFT_OFFSET); 1939 hr_reg_write(req, CFG_BT_ATTR_QPC_BUF_PGSZ, 1940 caps->qpc_buf_pg_sz + PG_SHIFT_OFFSET); 1941 hr_reg_write(req, CFG_BT_ATTR_QPC_HOPNUM, 1942 to_hr_hem_hopnum(caps->qpc_hop_num, caps->num_qps)); 1943 1944 hr_reg_write(req, CFG_BT_ATTR_SRQC_BA_PGSZ, 1945 caps->srqc_ba_pg_sz + PG_SHIFT_OFFSET); 1946 hr_reg_write(req, CFG_BT_ATTR_SRQC_BUF_PGSZ, 1947 caps->srqc_buf_pg_sz + PG_SHIFT_OFFSET); 1948 hr_reg_write(req, CFG_BT_ATTR_SRQC_HOPNUM, 1949 to_hr_hem_hopnum(caps->srqc_hop_num, caps->num_srqs)); 1950 1951 hr_reg_write(req, CFG_BT_ATTR_CQC_BA_PGSZ, 1952 caps->cqc_ba_pg_sz + PG_SHIFT_OFFSET); 1953 hr_reg_write(req, CFG_BT_ATTR_CQC_BUF_PGSZ, 1954 caps->cqc_buf_pg_sz + PG_SHIFT_OFFSET); 1955 hr_reg_write(req, CFG_BT_ATTR_CQC_HOPNUM, 1956 to_hr_hem_hopnum(caps->cqc_hop_num, caps->num_cqs)); 1957 1958 hr_reg_write(req, CFG_BT_ATTR_MPT_BA_PGSZ, 1959 caps->mpt_ba_pg_sz + PG_SHIFT_OFFSET); 1960 hr_reg_write(req, CFG_BT_ATTR_MPT_BUF_PGSZ, 1961 caps->mpt_buf_pg_sz + PG_SHIFT_OFFSET); 1962 hr_reg_write(req, CFG_BT_ATTR_MPT_HOPNUM, 1963 to_hr_hem_hopnum(caps->mpt_hop_num, caps->num_mtpts)); 1964 1965 hr_reg_write(req, CFG_BT_ATTR_SCCC_BA_PGSZ, 1966 caps->sccc_ba_pg_sz + PG_SHIFT_OFFSET); 1967 hr_reg_write(req, CFG_BT_ATTR_SCCC_BUF_PGSZ, 1968 caps->sccc_buf_pg_sz + PG_SHIFT_OFFSET); 1969 hr_reg_write(req, CFG_BT_ATTR_SCCC_HOPNUM, 1970 to_hr_hem_hopnum(caps->sccc_hop_num, caps->num_qps)); 1971 1972 return hns_roce_cmq_send(hr_dev, &desc, 1); 1973 } 1974 1975 /* Use default caps when hns_roce_query_pf_caps() failed or init VF profile */ 1976 static void set_default_caps(struct hns_roce_dev *hr_dev) 1977 { 1978 struct hns_roce_caps *caps = &hr_dev->caps; 1979 1980 caps->num_qps = HNS_ROCE_V2_MAX_QP_NUM; 1981 caps->max_wqes = HNS_ROCE_V2_MAX_WQE_NUM; 1982 caps->num_cqs = HNS_ROCE_V2_MAX_CQ_NUM; 1983 caps->num_srqs = HNS_ROCE_V2_MAX_SRQ_NUM; 1984 caps->min_cqes = HNS_ROCE_MIN_CQE_NUM; 1985 caps->max_cqes = HNS_ROCE_V2_MAX_CQE_NUM; 1986 caps->max_sq_sg = HNS_ROCE_V2_MAX_SQ_SGE_NUM; 1987 caps->max_rq_sg = HNS_ROCE_V2_MAX_RQ_SGE_NUM; 1988 1989 caps->num_uars = HNS_ROCE_V2_UAR_NUM; 1990 caps->phy_num_uars = HNS_ROCE_V2_PHY_UAR_NUM; 1991 caps->num_aeq_vectors = HNS_ROCE_V2_AEQE_VEC_NUM; 1992 caps->num_other_vectors = HNS_ROCE_V2_ABNORMAL_VEC_NUM; 1993 caps->num_comp_vectors = 0; 1994 1995 caps->num_mtpts = HNS_ROCE_V2_MAX_MTPT_NUM; 1996 caps->num_pds = HNS_ROCE_V2_MAX_PD_NUM; 1997 caps->qpc_timer_bt_num = HNS_ROCE_V2_MAX_QPC_TIMER_BT_NUM; 1998 caps->cqc_timer_bt_num = HNS_ROCE_V2_MAX_CQC_TIMER_BT_NUM; 1999 2000 caps->max_qp_init_rdma = HNS_ROCE_V2_MAX_QP_INIT_RDMA; 2001 caps->max_qp_dest_rdma = HNS_ROCE_V2_MAX_QP_DEST_RDMA; 2002 caps->max_sq_desc_sz = HNS_ROCE_V2_MAX_SQ_DESC_SZ; 2003 caps->max_rq_desc_sz = HNS_ROCE_V2_MAX_RQ_DESC_SZ; 2004 caps->irrl_entry_sz = HNS_ROCE_V2_IRRL_ENTRY_SZ; 2005 caps->trrl_entry_sz = HNS_ROCE_V2_EXT_ATOMIC_TRRL_ENTRY_SZ; 2006 caps->cqc_entry_sz = HNS_ROCE_V2_CQC_ENTRY_SZ; 2007 caps->srqc_entry_sz = HNS_ROCE_V2_SRQC_ENTRY_SZ; 2008 caps->mtpt_entry_sz = HNS_ROCE_V2_MTPT_ENTRY_SZ; 2009 caps->idx_entry_sz = HNS_ROCE_V2_IDX_ENTRY_SZ; 2010 caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED; 2011 caps->reserved_lkey = 0; 2012 caps->reserved_pds = 0; 2013 caps->reserved_mrws = 1; 2014 caps->reserved_uars = 0; 2015 caps->reserved_cqs = 0; 2016 caps->reserved_srqs = 0; 2017 caps->reserved_qps = HNS_ROCE_V2_RSV_QPS; 2018 2019 caps->qpc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; 2020 caps->srqc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; 2021 caps->cqc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; 2022 caps->mpt_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; 2023 caps->sccc_hop_num = HNS_ROCE_SCCC_HOP_NUM; 2024 2025 caps->mtt_hop_num = HNS_ROCE_MTT_HOP_NUM; 2026 caps->wqe_sq_hop_num = HNS_ROCE_SQWQE_HOP_NUM; 2027 caps->wqe_sge_hop_num = HNS_ROCE_EXT_SGE_HOP_NUM; 2028 caps->wqe_rq_hop_num = HNS_ROCE_RQWQE_HOP_NUM; 2029 caps->cqe_hop_num = HNS_ROCE_CQE_HOP_NUM; 2030 caps->srqwqe_hop_num = HNS_ROCE_SRQWQE_HOP_NUM; 2031 caps->idx_hop_num = HNS_ROCE_IDX_HOP_NUM; 2032 caps->chunk_sz = HNS_ROCE_V2_TABLE_CHUNK_SIZE; 2033 2034 caps->flags = HNS_ROCE_CAP_FLAG_REREG_MR | 2035 HNS_ROCE_CAP_FLAG_ROCE_V1_V2 | 2036 HNS_ROCE_CAP_FLAG_CQ_RECORD_DB | 2037 HNS_ROCE_CAP_FLAG_QP_RECORD_DB; 2038 2039 caps->pkey_table_len[0] = 1; 2040 caps->ceqe_depth = HNS_ROCE_V2_COMP_EQE_NUM; 2041 caps->aeqe_depth = HNS_ROCE_V2_ASYNC_EQE_NUM; 2042 caps->local_ca_ack_delay = 0; 2043 caps->max_mtu = IB_MTU_4096; 2044 2045 caps->max_srq_wrs = HNS_ROCE_V2_MAX_SRQ_WR; 2046 caps->max_srq_sges = HNS_ROCE_V2_MAX_SRQ_SGE; 2047 2048 caps->flags |= HNS_ROCE_CAP_FLAG_ATOMIC | HNS_ROCE_CAP_FLAG_MW | 2049 HNS_ROCE_CAP_FLAG_SRQ | HNS_ROCE_CAP_FLAG_FRMR | 2050 HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL; 2051 2052 caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM; 2053 2054 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) { 2055 caps->flags |= HNS_ROCE_CAP_FLAG_STASH | 2056 HNS_ROCE_CAP_FLAG_DIRECT_WQE | 2057 HNS_ROCE_CAP_FLAG_XRC; 2058 caps->max_sq_inline = HNS_ROCE_V3_MAX_SQ_INLINE; 2059 } else { 2060 caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INLINE; 2061 2062 /* The following configuration are only valid for HIP08 */ 2063 caps->qpc_sz = HNS_ROCE_V2_QPC_SZ; 2064 caps->sccc_sz = HNS_ROCE_V2_SCCC_SZ; 2065 caps->cqe_sz = HNS_ROCE_V2_CQE_SIZE; 2066 } 2067 } 2068 2069 static void calc_pg_sz(u32 obj_num, u32 obj_size, u32 hop_num, u32 ctx_bt_num, 2070 u32 *buf_page_size, u32 *bt_page_size, u32 hem_type) 2071 { 2072 u64 obj_per_chunk; 2073 u64 bt_chunk_size = PAGE_SIZE; 2074 u64 buf_chunk_size = PAGE_SIZE; 2075 u64 obj_per_chunk_default = buf_chunk_size / obj_size; 2076 2077 *buf_page_size = 0; 2078 *bt_page_size = 0; 2079 2080 switch (hop_num) { 2081 case 3: 2082 obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) * 2083 (bt_chunk_size / BA_BYTE_LEN) * 2084 (bt_chunk_size / BA_BYTE_LEN) * 2085 obj_per_chunk_default; 2086 break; 2087 case 2: 2088 obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) * 2089 (bt_chunk_size / BA_BYTE_LEN) * 2090 obj_per_chunk_default; 2091 break; 2092 case 1: 2093 obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) * 2094 obj_per_chunk_default; 2095 break; 2096 case HNS_ROCE_HOP_NUM_0: 2097 obj_per_chunk = ctx_bt_num * obj_per_chunk_default; 2098 break; 2099 default: 2100 pr_err("table %u not support hop_num = %u!\n", hem_type, 2101 hop_num); 2102 return; 2103 } 2104 2105 if (hem_type >= HEM_TYPE_MTT) 2106 *bt_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk)); 2107 else 2108 *buf_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk)); 2109 } 2110 2111 static void set_hem_page_size(struct hns_roce_dev *hr_dev) 2112 { 2113 struct hns_roce_caps *caps = &hr_dev->caps; 2114 2115 /* EQ */ 2116 caps->eqe_ba_pg_sz = 0; 2117 caps->eqe_buf_pg_sz = 0; 2118 2119 /* Link Table */ 2120 caps->llm_buf_pg_sz = 0; 2121 2122 /* MR */ 2123 caps->mpt_ba_pg_sz = 0; 2124 caps->mpt_buf_pg_sz = 0; 2125 caps->pbl_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_16K; 2126 caps->pbl_buf_pg_sz = 0; 2127 calc_pg_sz(caps->num_mtpts, caps->mtpt_entry_sz, caps->mpt_hop_num, 2128 caps->mpt_bt_num, &caps->mpt_buf_pg_sz, &caps->mpt_ba_pg_sz, 2129 HEM_TYPE_MTPT); 2130 2131 /* QP */ 2132 caps->qpc_ba_pg_sz = 0; 2133 caps->qpc_buf_pg_sz = 0; 2134 caps->qpc_timer_ba_pg_sz = 0; 2135 caps->qpc_timer_buf_pg_sz = 0; 2136 caps->sccc_ba_pg_sz = 0; 2137 caps->sccc_buf_pg_sz = 0; 2138 caps->mtt_ba_pg_sz = 0; 2139 caps->mtt_buf_pg_sz = 0; 2140 calc_pg_sz(caps->num_qps, caps->qpc_sz, caps->qpc_hop_num, 2141 caps->qpc_bt_num, &caps->qpc_buf_pg_sz, &caps->qpc_ba_pg_sz, 2142 HEM_TYPE_QPC); 2143 2144 if (caps->flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) 2145 calc_pg_sz(caps->num_qps, caps->sccc_sz, caps->sccc_hop_num, 2146 caps->sccc_bt_num, &caps->sccc_buf_pg_sz, 2147 &caps->sccc_ba_pg_sz, HEM_TYPE_SCCC); 2148 2149 /* CQ */ 2150 caps->cqc_ba_pg_sz = 0; 2151 caps->cqc_buf_pg_sz = 0; 2152 caps->cqc_timer_ba_pg_sz = 0; 2153 caps->cqc_timer_buf_pg_sz = 0; 2154 caps->cqe_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_256K; 2155 caps->cqe_buf_pg_sz = 0; 2156 calc_pg_sz(caps->num_cqs, caps->cqc_entry_sz, caps->cqc_hop_num, 2157 caps->cqc_bt_num, &caps->cqc_buf_pg_sz, &caps->cqc_ba_pg_sz, 2158 HEM_TYPE_CQC); 2159 calc_pg_sz(caps->max_cqes, caps->cqe_sz, caps->cqe_hop_num, 2160 1, &caps->cqe_buf_pg_sz, &caps->cqe_ba_pg_sz, HEM_TYPE_CQE); 2161 2162 /* SRQ */ 2163 if (caps->flags & HNS_ROCE_CAP_FLAG_SRQ) { 2164 caps->srqc_ba_pg_sz = 0; 2165 caps->srqc_buf_pg_sz = 0; 2166 caps->srqwqe_ba_pg_sz = 0; 2167 caps->srqwqe_buf_pg_sz = 0; 2168 caps->idx_ba_pg_sz = 0; 2169 caps->idx_buf_pg_sz = 0; 2170 calc_pg_sz(caps->num_srqs, caps->srqc_entry_sz, 2171 caps->srqc_hop_num, caps->srqc_bt_num, 2172 &caps->srqc_buf_pg_sz, &caps->srqc_ba_pg_sz, 2173 HEM_TYPE_SRQC); 2174 calc_pg_sz(caps->num_srqwqe_segs, caps->mtt_entry_sz, 2175 caps->srqwqe_hop_num, 1, &caps->srqwqe_buf_pg_sz, 2176 &caps->srqwqe_ba_pg_sz, HEM_TYPE_SRQWQE); 2177 calc_pg_sz(caps->num_idx_segs, caps->idx_entry_sz, 2178 caps->idx_hop_num, 1, &caps->idx_buf_pg_sz, 2179 &caps->idx_ba_pg_sz, HEM_TYPE_IDX); 2180 } 2181 2182 /* GMV */ 2183 caps->gmv_ba_pg_sz = 0; 2184 caps->gmv_buf_pg_sz = 0; 2185 } 2186 2187 /* Apply all loaded caps before setting to hardware */ 2188 static void apply_func_caps(struct hns_roce_dev *hr_dev) 2189 { 2190 struct hns_roce_caps *caps = &hr_dev->caps; 2191 struct hns_roce_v2_priv *priv = hr_dev->priv; 2192 2193 /* The following configurations don't need to be got from firmware. */ 2194 caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ; 2195 caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ; 2196 caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ; 2197 2198 caps->pbl_hop_num = HNS_ROCE_PBL_HOP_NUM; 2199 caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0; 2200 caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0; 2201 2202 caps->num_xrcds = HNS_ROCE_V2_MAX_XRCD_NUM; 2203 caps->reserved_xrcds = HNS_ROCE_V2_RSV_XRCD_NUM; 2204 2205 caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS; 2206 caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS; 2207 2208 if (!caps->num_comp_vectors) 2209 caps->num_comp_vectors = 2210 min_t(u32, caps->eqc_bt_num - HNS_ROCE_V2_AEQE_VEC_NUM, 2211 (u32)priv->handle->rinfo.num_vectors - 2212 (HNS_ROCE_V2_AEQE_VEC_NUM + HNS_ROCE_V2_ABNORMAL_VEC_NUM)); 2213 2214 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) { 2215 caps->eqe_hop_num = HNS_ROCE_V3_EQE_HOP_NUM; 2216 caps->ceqe_size = HNS_ROCE_V3_EQE_SIZE; 2217 caps->aeqe_size = HNS_ROCE_V3_EQE_SIZE; 2218 2219 /* The following configurations will be overwritten */ 2220 caps->qpc_sz = HNS_ROCE_V3_QPC_SZ; 2221 caps->cqe_sz = HNS_ROCE_V3_CQE_SIZE; 2222 caps->sccc_sz = HNS_ROCE_V3_SCCC_SZ; 2223 2224 /* The following configurations are not got from firmware */ 2225 caps->gmv_entry_sz = HNS_ROCE_V3_GMV_ENTRY_SZ; 2226 2227 caps->gmv_hop_num = HNS_ROCE_HOP_NUM_0; 2228 caps->gid_table_len[0] = caps->gmv_bt_num * 2229 (HNS_HW_PAGE_SIZE / caps->gmv_entry_sz); 2230 2231 caps->gmv_entry_num = caps->gmv_bt_num * (PAGE_SIZE / 2232 caps->gmv_entry_sz); 2233 } else { 2234 u32 func_num = max_t(u32, 1, hr_dev->func_num); 2235 2236 caps->eqe_hop_num = HNS_ROCE_V2_EQE_HOP_NUM; 2237 caps->ceqe_size = HNS_ROCE_CEQE_SIZE; 2238 caps->aeqe_size = HNS_ROCE_AEQE_SIZE; 2239 caps->gid_table_len[0] /= func_num; 2240 } 2241 2242 if (hr_dev->is_vf) { 2243 caps->default_aeq_arm_st = 0x3; 2244 caps->default_ceq_arm_st = 0x3; 2245 caps->default_ceq_max_cnt = 0x1; 2246 caps->default_ceq_period = 0x10; 2247 caps->default_aeq_max_cnt = 0x1; 2248 caps->default_aeq_period = 0x10; 2249 } 2250 2251 set_hem_page_size(hr_dev); 2252 } 2253 2254 static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev) 2255 { 2256 struct hns_roce_cmq_desc desc[HNS_ROCE_QUERY_PF_CAPS_CMD_NUM]; 2257 struct hns_roce_caps *caps = &hr_dev->caps; 2258 struct hns_roce_query_pf_caps_a *resp_a; 2259 struct hns_roce_query_pf_caps_b *resp_b; 2260 struct hns_roce_query_pf_caps_c *resp_c; 2261 struct hns_roce_query_pf_caps_d *resp_d; 2262 struct hns_roce_query_pf_caps_e *resp_e; 2263 int ctx_hop_num; 2264 int pbl_hop_num; 2265 int ret; 2266 int i; 2267 2268 for (i = 0; i < HNS_ROCE_QUERY_PF_CAPS_CMD_NUM; i++) { 2269 hns_roce_cmq_setup_basic_desc(&desc[i], 2270 HNS_ROCE_OPC_QUERY_PF_CAPS_NUM, 2271 true); 2272 if (i < (HNS_ROCE_QUERY_PF_CAPS_CMD_NUM - 1)) 2273 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 2274 else 2275 desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 2276 } 2277 2278 ret = hns_roce_cmq_send(hr_dev, desc, HNS_ROCE_QUERY_PF_CAPS_CMD_NUM); 2279 if (ret) 2280 return ret; 2281 2282 resp_a = (struct hns_roce_query_pf_caps_a *)desc[0].data; 2283 resp_b = (struct hns_roce_query_pf_caps_b *)desc[1].data; 2284 resp_c = (struct hns_roce_query_pf_caps_c *)desc[2].data; 2285 resp_d = (struct hns_roce_query_pf_caps_d *)desc[3].data; 2286 resp_e = (struct hns_roce_query_pf_caps_e *)desc[4].data; 2287 2288 caps->local_ca_ack_delay = resp_a->local_ca_ack_delay; 2289 caps->max_sq_sg = le16_to_cpu(resp_a->max_sq_sg); 2290 caps->max_sq_inline = le16_to_cpu(resp_a->max_sq_inline); 2291 caps->max_rq_sg = le16_to_cpu(resp_a->max_rq_sg); 2292 caps->max_rq_sg = roundup_pow_of_two(caps->max_rq_sg); 2293 caps->max_srq_sges = le16_to_cpu(resp_a->max_srq_sges); 2294 caps->max_srq_sges = roundup_pow_of_two(caps->max_srq_sges); 2295 caps->num_aeq_vectors = resp_a->num_aeq_vectors; 2296 caps->num_other_vectors = resp_a->num_other_vectors; 2297 caps->max_sq_desc_sz = resp_a->max_sq_desc_sz; 2298 caps->max_rq_desc_sz = resp_a->max_rq_desc_sz; 2299 caps->cqe_sz = resp_a->cqe_sz; 2300 2301 caps->mtpt_entry_sz = resp_b->mtpt_entry_sz; 2302 caps->irrl_entry_sz = resp_b->irrl_entry_sz; 2303 caps->trrl_entry_sz = resp_b->trrl_entry_sz; 2304 caps->cqc_entry_sz = resp_b->cqc_entry_sz; 2305 caps->srqc_entry_sz = resp_b->srqc_entry_sz; 2306 caps->idx_entry_sz = resp_b->idx_entry_sz; 2307 caps->sccc_sz = resp_b->sccc_sz; 2308 caps->max_mtu = resp_b->max_mtu; 2309 caps->qpc_sz = le16_to_cpu(resp_b->qpc_sz); 2310 caps->min_cqes = resp_b->min_cqes; 2311 caps->min_wqes = resp_b->min_wqes; 2312 caps->page_size_cap = le32_to_cpu(resp_b->page_size_cap); 2313 caps->pkey_table_len[0] = resp_b->pkey_table_len; 2314 caps->phy_num_uars = resp_b->phy_num_uars; 2315 ctx_hop_num = resp_b->ctx_hop_num; 2316 pbl_hop_num = resp_b->pbl_hop_num; 2317 2318 caps->num_pds = 1 << hr_reg_read(resp_c, PF_CAPS_C_NUM_PDS); 2319 2320 caps->flags = hr_reg_read(resp_c, PF_CAPS_C_CAP_FLAGS); 2321 caps->flags |= le16_to_cpu(resp_d->cap_flags_ex) << 2322 HNS_ROCE_CAP_FLAGS_EX_SHIFT; 2323 2324 caps->num_cqs = 1 << hr_reg_read(resp_c, PF_CAPS_C_NUM_CQS); 2325 caps->gid_table_len[0] = hr_reg_read(resp_c, PF_CAPS_C_MAX_GID); 2326 caps->max_cqes = 1 << hr_reg_read(resp_c, PF_CAPS_C_CQ_DEPTH); 2327 caps->num_mtpts = 1 << hr_reg_read(resp_c, PF_CAPS_C_NUM_MRWS); 2328 caps->num_qps = 1 << hr_reg_read(resp_c, PF_CAPS_C_NUM_QPS); 2329 caps->max_qp_init_rdma = hr_reg_read(resp_c, PF_CAPS_C_MAX_ORD); 2330 caps->max_qp_dest_rdma = caps->max_qp_init_rdma; 2331 caps->max_wqes = 1 << le16_to_cpu(resp_c->sq_depth); 2332 2333 caps->num_srqs = 1 << hr_reg_read(resp_d, PF_CAPS_D_NUM_SRQS); 2334 caps->cong_type = hr_reg_read(resp_d, PF_CAPS_D_CONG_TYPE); 2335 caps->max_srq_wrs = 1 << le16_to_cpu(resp_d->srq_depth); 2336 caps->ceqe_depth = 1 << hr_reg_read(resp_d, PF_CAPS_D_CEQ_DEPTH); 2337 caps->num_comp_vectors = hr_reg_read(resp_d, PF_CAPS_D_NUM_CEQS); 2338 caps->aeqe_depth = 1 << hr_reg_read(resp_d, PF_CAPS_D_AEQ_DEPTH); 2339 caps->default_aeq_arm_st = hr_reg_read(resp_d, PF_CAPS_D_AEQ_ARM_ST); 2340 caps->default_ceq_arm_st = hr_reg_read(resp_d, PF_CAPS_D_CEQ_ARM_ST); 2341 caps->reserved_pds = hr_reg_read(resp_d, PF_CAPS_D_RSV_PDS); 2342 caps->num_uars = 1 << hr_reg_read(resp_d, PF_CAPS_D_NUM_UARS); 2343 caps->reserved_qps = hr_reg_read(resp_d, PF_CAPS_D_RSV_QPS); 2344 caps->reserved_uars = hr_reg_read(resp_d, PF_CAPS_D_RSV_UARS); 2345 2346 caps->reserved_mrws = hr_reg_read(resp_e, PF_CAPS_E_RSV_MRWS); 2347 caps->chunk_sz = 1 << hr_reg_read(resp_e, PF_CAPS_E_CHUNK_SIZE_SHIFT); 2348 caps->reserved_cqs = hr_reg_read(resp_e, PF_CAPS_E_RSV_CQS); 2349 caps->reserved_srqs = hr_reg_read(resp_e, PF_CAPS_E_RSV_SRQS); 2350 caps->reserved_lkey = hr_reg_read(resp_e, PF_CAPS_E_RSV_LKEYS); 2351 caps->default_ceq_max_cnt = le16_to_cpu(resp_e->ceq_max_cnt); 2352 caps->default_ceq_period = le16_to_cpu(resp_e->ceq_period); 2353 caps->default_aeq_max_cnt = le16_to_cpu(resp_e->aeq_max_cnt); 2354 caps->default_aeq_period = le16_to_cpu(resp_e->aeq_period); 2355 2356 caps->qpc_hop_num = ctx_hop_num; 2357 caps->sccc_hop_num = ctx_hop_num; 2358 caps->srqc_hop_num = ctx_hop_num; 2359 caps->cqc_hop_num = ctx_hop_num; 2360 caps->mpt_hop_num = ctx_hop_num; 2361 caps->mtt_hop_num = pbl_hop_num; 2362 caps->cqe_hop_num = pbl_hop_num; 2363 caps->srqwqe_hop_num = pbl_hop_num; 2364 caps->idx_hop_num = pbl_hop_num; 2365 caps->wqe_sq_hop_num = hr_reg_read(resp_d, PF_CAPS_D_SQWQE_HOP_NUM); 2366 caps->wqe_sge_hop_num = hr_reg_read(resp_d, PF_CAPS_D_EX_SGE_HOP_NUM); 2367 caps->wqe_rq_hop_num = hr_reg_read(resp_d, PF_CAPS_D_RQWQE_HOP_NUM); 2368 2369 if (!(caps->page_size_cap & PAGE_SIZE)) 2370 caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED; 2371 2372 return 0; 2373 } 2374 2375 static int config_hem_entry_size(struct hns_roce_dev *hr_dev, u32 type, u32 val) 2376 { 2377 struct hns_roce_cmq_desc desc; 2378 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data; 2379 2380 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_ENTRY_SIZE, 2381 false); 2382 2383 hr_reg_write(req, CFG_HEM_ENTRY_SIZE_TYPE, type); 2384 hr_reg_write(req, CFG_HEM_ENTRY_SIZE_VALUE, val); 2385 2386 return hns_roce_cmq_send(hr_dev, &desc, 1); 2387 } 2388 2389 static int hns_roce_config_entry_size(struct hns_roce_dev *hr_dev) 2390 { 2391 struct hns_roce_caps *caps = &hr_dev->caps; 2392 int ret; 2393 2394 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) 2395 return 0; 2396 2397 ret = config_hem_entry_size(hr_dev, HNS_ROCE_CFG_QPC_SIZE, 2398 caps->qpc_sz); 2399 if (ret) { 2400 dev_err(hr_dev->dev, "failed to cfg qpc sz, ret = %d.\n", ret); 2401 return ret; 2402 } 2403 2404 ret = config_hem_entry_size(hr_dev, HNS_ROCE_CFG_SCCC_SIZE, 2405 caps->sccc_sz); 2406 if (ret) 2407 dev_err(hr_dev->dev, "failed to cfg sccc sz, ret = %d.\n", ret); 2408 2409 return ret; 2410 } 2411 2412 static int hns_roce_v2_vf_profile(struct hns_roce_dev *hr_dev) 2413 { 2414 struct device *dev = hr_dev->dev; 2415 int ret; 2416 2417 hr_dev->func_num = 1; 2418 2419 set_default_caps(hr_dev); 2420 2421 ret = hns_roce_query_vf_resource(hr_dev); 2422 if (ret) { 2423 dev_err(dev, "failed to query VF resource, ret = %d.\n", ret); 2424 return ret; 2425 } 2426 2427 apply_func_caps(hr_dev); 2428 2429 ret = hns_roce_v2_set_bt(hr_dev); 2430 if (ret) 2431 dev_err(dev, "failed to config VF BA table, ret = %d.\n", ret); 2432 2433 return ret; 2434 } 2435 2436 static int hns_roce_v2_pf_profile(struct hns_roce_dev *hr_dev) 2437 { 2438 struct device *dev = hr_dev->dev; 2439 int ret; 2440 2441 ret = hns_roce_query_func_info(hr_dev); 2442 if (ret) { 2443 dev_err(dev, "failed to query func info, ret = %d.\n", ret); 2444 return ret; 2445 } 2446 2447 ret = hns_roce_config_global_param(hr_dev); 2448 if (ret) { 2449 dev_err(dev, "failed to config global param, ret = %d.\n", ret); 2450 return ret; 2451 } 2452 2453 ret = hns_roce_set_vf_switch_param(hr_dev); 2454 if (ret) { 2455 dev_err(dev, "failed to set switch param, ret = %d.\n", ret); 2456 return ret; 2457 } 2458 2459 ret = hns_roce_query_pf_caps(hr_dev); 2460 if (ret) 2461 set_default_caps(hr_dev); 2462 2463 ret = hns_roce_query_pf_resource(hr_dev); 2464 if (ret) { 2465 dev_err(dev, "failed to query pf resource, ret = %d.\n", ret); 2466 return ret; 2467 } 2468 2469 apply_func_caps(hr_dev); 2470 2471 ret = hns_roce_alloc_vf_resource(hr_dev); 2472 if (ret) { 2473 dev_err(dev, "failed to alloc vf resource, ret = %d.\n", ret); 2474 return ret; 2475 } 2476 2477 ret = hns_roce_v2_set_bt(hr_dev); 2478 if (ret) { 2479 dev_err(dev, "failed to config BA table, ret = %d.\n", ret); 2480 return ret; 2481 } 2482 2483 /* Configure the size of QPC, SCCC, etc. */ 2484 return hns_roce_config_entry_size(hr_dev); 2485 } 2486 2487 static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) 2488 { 2489 struct device *dev = hr_dev->dev; 2490 int ret; 2491 2492 ret = hns_roce_cmq_query_hw_info(hr_dev); 2493 if (ret) { 2494 dev_err(dev, "failed to query hardware info, ret = %d.\n", ret); 2495 return ret; 2496 } 2497 2498 ret = hns_roce_query_fw_ver(hr_dev); 2499 if (ret) { 2500 dev_err(dev, "failed to query firmware info, ret = %d.\n", ret); 2501 return ret; 2502 } 2503 2504 hr_dev->vendor_part_id = hr_dev->pci_dev->device; 2505 hr_dev->sys_image_guid = be64_to_cpu(hr_dev->ib_dev.node_guid); 2506 2507 if (hr_dev->is_vf) 2508 return hns_roce_v2_vf_profile(hr_dev); 2509 else 2510 return hns_roce_v2_pf_profile(hr_dev); 2511 } 2512 2513 static void config_llm_table(struct hns_roce_buf *data_buf, void *cfg_buf) 2514 { 2515 u32 i, next_ptr, page_num; 2516 __le64 *entry = cfg_buf; 2517 dma_addr_t addr; 2518 u64 val; 2519 2520 page_num = data_buf->npages; 2521 for (i = 0; i < page_num; i++) { 2522 addr = hns_roce_buf_page(data_buf, i); 2523 if (i == (page_num - 1)) 2524 next_ptr = 0; 2525 else 2526 next_ptr = i + 1; 2527 2528 val = HNS_ROCE_EXT_LLM_ENTRY(addr, (u64)next_ptr); 2529 entry[i] = cpu_to_le64(val); 2530 } 2531 } 2532 2533 static int set_llm_cfg_to_hw(struct hns_roce_dev *hr_dev, 2534 struct hns_roce_link_table *table) 2535 { 2536 struct hns_roce_cmq_desc desc[2]; 2537 struct hns_roce_cmq_req *r_a = (struct hns_roce_cmq_req *)desc[0].data; 2538 struct hns_roce_cmq_req *r_b = (struct hns_roce_cmq_req *)desc[1].data; 2539 struct hns_roce_buf *buf = table->buf; 2540 enum hns_roce_opcode_type opcode; 2541 dma_addr_t addr; 2542 2543 opcode = HNS_ROCE_OPC_CFG_EXT_LLM; 2544 hns_roce_cmq_setup_basic_desc(&desc[0], opcode, false); 2545 desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 2546 hns_roce_cmq_setup_basic_desc(&desc[1], opcode, false); 2547 2548 hr_reg_write(r_a, CFG_LLM_A_BA_L, lower_32_bits(table->table.map)); 2549 hr_reg_write(r_a, CFG_LLM_A_BA_H, upper_32_bits(table->table.map)); 2550 hr_reg_write(r_a, CFG_LLM_A_DEPTH, buf->npages); 2551 hr_reg_write(r_a, CFG_LLM_A_PGSZ, to_hr_hw_page_shift(buf->page_shift)); 2552 hr_reg_enable(r_a, CFG_LLM_A_INIT_EN); 2553 2554 addr = to_hr_hw_page_addr(hns_roce_buf_page(buf, 0)); 2555 hr_reg_write(r_a, CFG_LLM_A_HEAD_BA_L, lower_32_bits(addr)); 2556 hr_reg_write(r_a, CFG_LLM_A_HEAD_BA_H, upper_32_bits(addr)); 2557 hr_reg_write(r_a, CFG_LLM_A_HEAD_NXTPTR, 1); 2558 hr_reg_write(r_a, CFG_LLM_A_HEAD_PTR, 0); 2559 2560 addr = to_hr_hw_page_addr(hns_roce_buf_page(buf, buf->npages - 1)); 2561 hr_reg_write(r_b, CFG_LLM_B_TAIL_BA_L, lower_32_bits(addr)); 2562 hr_reg_write(r_b, CFG_LLM_B_TAIL_BA_H, upper_32_bits(addr)); 2563 hr_reg_write(r_b, CFG_LLM_B_TAIL_PTR, buf->npages - 1); 2564 2565 return hns_roce_cmq_send(hr_dev, desc, 2); 2566 } 2567 2568 static struct hns_roce_link_table * 2569 alloc_link_table_buf(struct hns_roce_dev *hr_dev) 2570 { 2571 struct hns_roce_v2_priv *priv = hr_dev->priv; 2572 struct hns_roce_link_table *link_tbl; 2573 u32 pg_shift, size, min_size; 2574 2575 link_tbl = &priv->ext_llm; 2576 pg_shift = hr_dev->caps.llm_buf_pg_sz + PAGE_SHIFT; 2577 size = hr_dev->caps.num_qps * HNS_ROCE_V2_EXT_LLM_ENTRY_SZ; 2578 min_size = HNS_ROCE_EXT_LLM_MIN_PAGES(hr_dev->caps.sl_num) << pg_shift; 2579 2580 /* Alloc data table */ 2581 size = max(size, min_size); 2582 link_tbl->buf = hns_roce_buf_alloc(hr_dev, size, pg_shift, 0); 2583 if (IS_ERR(link_tbl->buf)) 2584 return ERR_PTR(-ENOMEM); 2585 2586 /* Alloc config table */ 2587 size = link_tbl->buf->npages * sizeof(u64); 2588 link_tbl->table.buf = dma_alloc_coherent(hr_dev->dev, size, 2589 &link_tbl->table.map, 2590 GFP_KERNEL); 2591 if (!link_tbl->table.buf) { 2592 hns_roce_buf_free(hr_dev, link_tbl->buf); 2593 return ERR_PTR(-ENOMEM); 2594 } 2595 2596 return link_tbl; 2597 } 2598 2599 static void free_link_table_buf(struct hns_roce_dev *hr_dev, 2600 struct hns_roce_link_table *tbl) 2601 { 2602 if (tbl->buf) { 2603 u32 size = tbl->buf->npages * sizeof(u64); 2604 2605 dma_free_coherent(hr_dev->dev, size, tbl->table.buf, 2606 tbl->table.map); 2607 } 2608 2609 hns_roce_buf_free(hr_dev, tbl->buf); 2610 } 2611 2612 static int hns_roce_init_link_table(struct hns_roce_dev *hr_dev) 2613 { 2614 struct hns_roce_link_table *link_tbl; 2615 int ret; 2616 2617 link_tbl = alloc_link_table_buf(hr_dev); 2618 if (IS_ERR(link_tbl)) 2619 return -ENOMEM; 2620 2621 if (WARN_ON(link_tbl->buf->npages > HNS_ROCE_V2_EXT_LLM_MAX_DEPTH)) { 2622 ret = -EINVAL; 2623 goto err_alloc; 2624 } 2625 2626 config_llm_table(link_tbl->buf, link_tbl->table.buf); 2627 ret = set_llm_cfg_to_hw(hr_dev, link_tbl); 2628 if (ret) 2629 goto err_alloc; 2630 2631 return 0; 2632 2633 err_alloc: 2634 free_link_table_buf(hr_dev, link_tbl); 2635 return ret; 2636 } 2637 2638 static void hns_roce_free_link_table(struct hns_roce_dev *hr_dev) 2639 { 2640 struct hns_roce_v2_priv *priv = hr_dev->priv; 2641 2642 free_link_table_buf(hr_dev, &priv->ext_llm); 2643 } 2644 2645 static void free_dip_list(struct hns_roce_dev *hr_dev) 2646 { 2647 struct hns_roce_dip *hr_dip; 2648 struct hns_roce_dip *tmp; 2649 unsigned long flags; 2650 2651 spin_lock_irqsave(&hr_dev->dip_list_lock, flags); 2652 2653 list_for_each_entry_safe(hr_dip, tmp, &hr_dev->dip_list, node) { 2654 list_del(&hr_dip->node); 2655 kfree(hr_dip); 2656 } 2657 2658 spin_unlock_irqrestore(&hr_dev->dip_list_lock, flags); 2659 } 2660 2661 static struct ib_pd *free_mr_init_pd(struct hns_roce_dev *hr_dev) 2662 { 2663 struct hns_roce_v2_priv *priv = hr_dev->priv; 2664 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr; 2665 struct ib_device *ibdev = &hr_dev->ib_dev; 2666 struct hns_roce_pd *hr_pd; 2667 struct ib_pd *pd; 2668 2669 hr_pd = kzalloc(sizeof(*hr_pd), GFP_KERNEL); 2670 if (ZERO_OR_NULL_PTR(hr_pd)) 2671 return NULL; 2672 pd = &hr_pd->ibpd; 2673 pd->device = ibdev; 2674 2675 if (hns_roce_alloc_pd(pd, NULL)) { 2676 ibdev_err(ibdev, "failed to create pd for free mr.\n"); 2677 kfree(hr_pd); 2678 return NULL; 2679 } 2680 free_mr->rsv_pd = to_hr_pd(pd); 2681 free_mr->rsv_pd->ibpd.device = &hr_dev->ib_dev; 2682 free_mr->rsv_pd->ibpd.uobject = NULL; 2683 free_mr->rsv_pd->ibpd.__internal_mr = NULL; 2684 atomic_set(&free_mr->rsv_pd->ibpd.usecnt, 0); 2685 2686 return pd; 2687 } 2688 2689 static struct ib_cq *free_mr_init_cq(struct hns_roce_dev *hr_dev) 2690 { 2691 struct hns_roce_v2_priv *priv = hr_dev->priv; 2692 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr; 2693 struct ib_device *ibdev = &hr_dev->ib_dev; 2694 struct ib_cq_init_attr cq_init_attr = {}; 2695 struct hns_roce_cq *hr_cq; 2696 struct ib_cq *cq; 2697 2698 cq_init_attr.cqe = HNS_ROCE_FREE_MR_USED_CQE_NUM; 2699 2700 hr_cq = kzalloc(sizeof(*hr_cq), GFP_KERNEL); 2701 if (ZERO_OR_NULL_PTR(hr_cq)) 2702 return NULL; 2703 2704 cq = &hr_cq->ib_cq; 2705 cq->device = ibdev; 2706 2707 if (hns_roce_create_cq(cq, &cq_init_attr, NULL)) { 2708 ibdev_err(ibdev, "failed to create cq for free mr.\n"); 2709 kfree(hr_cq); 2710 return NULL; 2711 } 2712 free_mr->rsv_cq = to_hr_cq(cq); 2713 free_mr->rsv_cq->ib_cq.device = &hr_dev->ib_dev; 2714 free_mr->rsv_cq->ib_cq.uobject = NULL; 2715 free_mr->rsv_cq->ib_cq.comp_handler = NULL; 2716 free_mr->rsv_cq->ib_cq.event_handler = NULL; 2717 free_mr->rsv_cq->ib_cq.cq_context = NULL; 2718 atomic_set(&free_mr->rsv_cq->ib_cq.usecnt, 0); 2719 2720 return cq; 2721 } 2722 2723 static int free_mr_init_qp(struct hns_roce_dev *hr_dev, struct ib_cq *cq, 2724 struct ib_qp_init_attr *init_attr, int i) 2725 { 2726 struct hns_roce_v2_priv *priv = hr_dev->priv; 2727 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr; 2728 struct ib_device *ibdev = &hr_dev->ib_dev; 2729 struct hns_roce_qp *hr_qp; 2730 struct ib_qp *qp; 2731 int ret; 2732 2733 hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL); 2734 if (ZERO_OR_NULL_PTR(hr_qp)) 2735 return -ENOMEM; 2736 2737 qp = &hr_qp->ibqp; 2738 qp->device = ibdev; 2739 2740 ret = hns_roce_create_qp(qp, init_attr, NULL); 2741 if (ret) { 2742 ibdev_err(ibdev, "failed to create qp for free mr.\n"); 2743 kfree(hr_qp); 2744 return ret; 2745 } 2746 2747 free_mr->rsv_qp[i] = hr_qp; 2748 free_mr->rsv_qp[i]->ibqp.recv_cq = cq; 2749 free_mr->rsv_qp[i]->ibqp.send_cq = cq; 2750 2751 return 0; 2752 } 2753 2754 static void free_mr_exit(struct hns_roce_dev *hr_dev) 2755 { 2756 struct hns_roce_v2_priv *priv = hr_dev->priv; 2757 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr; 2758 struct ib_qp *qp; 2759 int i; 2760 2761 for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) { 2762 if (free_mr->rsv_qp[i]) { 2763 qp = &free_mr->rsv_qp[i]->ibqp; 2764 hns_roce_v2_destroy_qp(qp, NULL); 2765 kfree(free_mr->rsv_qp[i]); 2766 free_mr->rsv_qp[i] = NULL; 2767 } 2768 } 2769 2770 if (free_mr->rsv_cq) { 2771 hns_roce_destroy_cq(&free_mr->rsv_cq->ib_cq, NULL); 2772 kfree(free_mr->rsv_cq); 2773 free_mr->rsv_cq = NULL; 2774 } 2775 2776 if (free_mr->rsv_pd) { 2777 hns_roce_dealloc_pd(&free_mr->rsv_pd->ibpd, NULL); 2778 kfree(free_mr->rsv_pd); 2779 free_mr->rsv_pd = NULL; 2780 } 2781 } 2782 2783 static int free_mr_alloc_res(struct hns_roce_dev *hr_dev) 2784 { 2785 struct hns_roce_v2_priv *priv = hr_dev->priv; 2786 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr; 2787 struct ib_qp_init_attr qp_init_attr = {}; 2788 struct ib_pd *pd; 2789 struct ib_cq *cq; 2790 int ret; 2791 int i; 2792 2793 pd = free_mr_init_pd(hr_dev); 2794 if (!pd) 2795 return -ENOMEM; 2796 2797 cq = free_mr_init_cq(hr_dev); 2798 if (!cq) { 2799 ret = -ENOMEM; 2800 goto create_failed_cq; 2801 } 2802 2803 qp_init_attr.qp_type = IB_QPT_RC; 2804 qp_init_attr.sq_sig_type = IB_SIGNAL_ALL_WR; 2805 qp_init_attr.send_cq = cq; 2806 qp_init_attr.recv_cq = cq; 2807 for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) { 2808 qp_init_attr.cap.max_send_wr = HNS_ROCE_FREE_MR_USED_SQWQE_NUM; 2809 qp_init_attr.cap.max_send_sge = HNS_ROCE_FREE_MR_USED_SQSGE_NUM; 2810 qp_init_attr.cap.max_recv_wr = HNS_ROCE_FREE_MR_USED_RQWQE_NUM; 2811 qp_init_attr.cap.max_recv_sge = HNS_ROCE_FREE_MR_USED_RQSGE_NUM; 2812 2813 ret = free_mr_init_qp(hr_dev, cq, &qp_init_attr, i); 2814 if (ret) 2815 goto create_failed_qp; 2816 } 2817 2818 return 0; 2819 2820 create_failed_qp: 2821 hns_roce_destroy_cq(cq, NULL); 2822 kfree(cq); 2823 2824 create_failed_cq: 2825 hns_roce_dealloc_pd(pd, NULL); 2826 kfree(pd); 2827 2828 return ret; 2829 } 2830 2831 static int free_mr_modify_rsv_qp(struct hns_roce_dev *hr_dev, 2832 struct ib_qp_attr *attr, int sl_num) 2833 { 2834 struct hns_roce_v2_priv *priv = hr_dev->priv; 2835 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr; 2836 struct ib_device *ibdev = &hr_dev->ib_dev; 2837 struct hns_roce_qp *hr_qp; 2838 int loopback; 2839 int mask; 2840 int ret; 2841 2842 hr_qp = to_hr_qp(&free_mr->rsv_qp[sl_num]->ibqp); 2843 hr_qp->free_mr_en = 1; 2844 hr_qp->ibqp.device = ibdev; 2845 hr_qp->ibqp.qp_type = IB_QPT_RC; 2846 2847 mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS; 2848 attr->qp_state = IB_QPS_INIT; 2849 attr->port_num = 1; 2850 attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE; 2851 ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_INIT, 2852 IB_QPS_INIT); 2853 if (ret) { 2854 ibdev_err(ibdev, "failed to modify qp to init, ret = %d.\n", 2855 ret); 2856 return ret; 2857 } 2858 2859 loopback = hr_dev->loop_idc; 2860 /* Set qpc lbi = 1 incidate loopback IO */ 2861 hr_dev->loop_idc = 1; 2862 2863 mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU | IB_QP_DEST_QPN | 2864 IB_QP_RQ_PSN | IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER; 2865 attr->qp_state = IB_QPS_RTR; 2866 attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 2867 attr->path_mtu = IB_MTU_256; 2868 attr->dest_qp_num = hr_qp->qpn; 2869 attr->rq_psn = HNS_ROCE_FREE_MR_USED_PSN; 2870 2871 rdma_ah_set_sl(&attr->ah_attr, (u8)sl_num); 2872 2873 ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_INIT, 2874 IB_QPS_RTR); 2875 hr_dev->loop_idc = loopback; 2876 if (ret) { 2877 ibdev_err(ibdev, "failed to modify qp to rtr, ret = %d.\n", 2878 ret); 2879 return ret; 2880 } 2881 2882 mask = IB_QP_STATE | IB_QP_SQ_PSN | IB_QP_RETRY_CNT | IB_QP_TIMEOUT | 2883 IB_QP_RNR_RETRY | IB_QP_MAX_QP_RD_ATOMIC; 2884 attr->qp_state = IB_QPS_RTS; 2885 attr->sq_psn = HNS_ROCE_FREE_MR_USED_PSN; 2886 attr->retry_cnt = HNS_ROCE_FREE_MR_USED_QP_RETRY_CNT; 2887 attr->timeout = HNS_ROCE_FREE_MR_USED_QP_TIMEOUT; 2888 ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_RTR, 2889 IB_QPS_RTS); 2890 if (ret) 2891 ibdev_err(ibdev, "failed to modify qp to rts, ret = %d.\n", 2892 ret); 2893 2894 return ret; 2895 } 2896 2897 static int free_mr_modify_qp(struct hns_roce_dev *hr_dev) 2898 { 2899 struct hns_roce_v2_priv *priv = hr_dev->priv; 2900 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr; 2901 struct ib_qp_attr attr = {}; 2902 int ret; 2903 int i; 2904 2905 rdma_ah_set_grh(&attr.ah_attr, NULL, 0, 0, 1, 0); 2906 rdma_ah_set_static_rate(&attr.ah_attr, 3); 2907 rdma_ah_set_port_num(&attr.ah_attr, 1); 2908 2909 for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) { 2910 ret = free_mr_modify_rsv_qp(hr_dev, &attr, i); 2911 if (ret) 2912 return ret; 2913 } 2914 2915 return 0; 2916 } 2917 2918 static int free_mr_init(struct hns_roce_dev *hr_dev) 2919 { 2920 struct hns_roce_v2_priv *priv = hr_dev->priv; 2921 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr; 2922 int ret; 2923 2924 mutex_init(&free_mr->mutex); 2925 2926 ret = free_mr_alloc_res(hr_dev); 2927 if (ret) 2928 return ret; 2929 2930 ret = free_mr_modify_qp(hr_dev); 2931 if (ret) 2932 goto err_modify_qp; 2933 2934 return 0; 2935 2936 err_modify_qp: 2937 free_mr_exit(hr_dev); 2938 2939 return ret; 2940 } 2941 2942 static int get_hem_table(struct hns_roce_dev *hr_dev) 2943 { 2944 unsigned int qpc_count; 2945 unsigned int cqc_count; 2946 unsigned int gmv_count; 2947 int ret; 2948 int i; 2949 2950 /* Alloc memory for source address table buffer space chunk */ 2951 for (gmv_count = 0; gmv_count < hr_dev->caps.gmv_entry_num; 2952 gmv_count++) { 2953 ret = hns_roce_table_get(hr_dev, &hr_dev->gmv_table, gmv_count); 2954 if (ret) 2955 goto err_gmv_failed; 2956 } 2957 2958 if (hr_dev->is_vf) 2959 return 0; 2960 2961 /* Alloc memory for QPC Timer buffer space chunk */ 2962 for (qpc_count = 0; qpc_count < hr_dev->caps.qpc_timer_bt_num; 2963 qpc_count++) { 2964 ret = hns_roce_table_get(hr_dev, &hr_dev->qpc_timer_table, 2965 qpc_count); 2966 if (ret) { 2967 dev_err(hr_dev->dev, "QPC Timer get failed\n"); 2968 goto err_qpc_timer_failed; 2969 } 2970 } 2971 2972 /* Alloc memory for CQC Timer buffer space chunk */ 2973 for (cqc_count = 0; cqc_count < hr_dev->caps.cqc_timer_bt_num; 2974 cqc_count++) { 2975 ret = hns_roce_table_get(hr_dev, &hr_dev->cqc_timer_table, 2976 cqc_count); 2977 if (ret) { 2978 dev_err(hr_dev->dev, "CQC Timer get failed\n"); 2979 goto err_cqc_timer_failed; 2980 } 2981 } 2982 2983 return 0; 2984 2985 err_cqc_timer_failed: 2986 for (i = 0; i < cqc_count; i++) 2987 hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i); 2988 2989 err_qpc_timer_failed: 2990 for (i = 0; i < qpc_count; i++) 2991 hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i); 2992 2993 err_gmv_failed: 2994 for (i = 0; i < gmv_count; i++) 2995 hns_roce_table_put(hr_dev, &hr_dev->gmv_table, i); 2996 2997 return ret; 2998 } 2999 3000 static void put_hem_table(struct hns_roce_dev *hr_dev) 3001 { 3002 int i; 3003 3004 for (i = 0; i < hr_dev->caps.gmv_entry_num; i++) 3005 hns_roce_table_put(hr_dev, &hr_dev->gmv_table, i); 3006 3007 if (hr_dev->is_vf) 3008 return; 3009 3010 for (i = 0; i < hr_dev->caps.qpc_timer_bt_num; i++) 3011 hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i); 3012 3013 for (i = 0; i < hr_dev->caps.cqc_timer_bt_num; i++) 3014 hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i); 3015 } 3016 3017 static int hns_roce_v2_init(struct hns_roce_dev *hr_dev) 3018 { 3019 int ret; 3020 3021 /* The hns ROCEE requires the extdb info to be cleared before using */ 3022 ret = hns_roce_clear_extdb_list_info(hr_dev); 3023 if (ret) 3024 return ret; 3025 3026 ret = get_hem_table(hr_dev); 3027 if (ret) 3028 return ret; 3029 3030 if (hr_dev->is_vf) 3031 return 0; 3032 3033 ret = hns_roce_init_link_table(hr_dev); 3034 if (ret) { 3035 dev_err(hr_dev->dev, "failed to init llm, ret = %d.\n", ret); 3036 goto err_llm_init_failed; 3037 } 3038 3039 return 0; 3040 3041 err_llm_init_failed: 3042 put_hem_table(hr_dev); 3043 3044 return ret; 3045 } 3046 3047 static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev) 3048 { 3049 hns_roce_function_clear(hr_dev); 3050 3051 if (!hr_dev->is_vf) 3052 hns_roce_free_link_table(hr_dev); 3053 3054 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP09) 3055 free_dip_list(hr_dev); 3056 } 3057 3058 static int hns_roce_mbox_post(struct hns_roce_dev *hr_dev, 3059 struct hns_roce_mbox_msg *mbox_msg) 3060 { 3061 struct hns_roce_cmq_desc desc; 3062 struct hns_roce_post_mbox *mb = (struct hns_roce_post_mbox *)desc.data; 3063 3064 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_POST_MB, false); 3065 3066 mb->in_param_l = cpu_to_le32(mbox_msg->in_param); 3067 mb->in_param_h = cpu_to_le32(mbox_msg->in_param >> 32); 3068 mb->out_param_l = cpu_to_le32(mbox_msg->out_param); 3069 mb->out_param_h = cpu_to_le32(mbox_msg->out_param >> 32); 3070 mb->cmd_tag = cpu_to_le32(mbox_msg->tag << 8 | mbox_msg->cmd); 3071 mb->token_event_en = cpu_to_le32(mbox_msg->event_en << 16 | 3072 mbox_msg->token); 3073 3074 return hns_roce_cmq_send(hr_dev, &desc, 1); 3075 } 3076 3077 static int v2_wait_mbox_complete(struct hns_roce_dev *hr_dev, u32 timeout, 3078 u8 *complete_status) 3079 { 3080 struct hns_roce_mbox_status *mb_st; 3081 struct hns_roce_cmq_desc desc; 3082 unsigned long end; 3083 int ret = -EBUSY; 3084 u32 status; 3085 bool busy; 3086 3087 mb_st = (struct hns_roce_mbox_status *)desc.data; 3088 end = msecs_to_jiffies(timeout) + jiffies; 3089 while (v2_chk_mbox_is_avail(hr_dev, &busy)) { 3090 if (hr_dev->cmd.state == HNS_ROCE_CMDQ_STATE_FATAL_ERR) 3091 return -EIO; 3092 3093 status = 0; 3094 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_MB_ST, 3095 true); 3096 ret = __hns_roce_cmq_send(hr_dev, &desc, 1); 3097 if (!ret) { 3098 status = le32_to_cpu(mb_st->mb_status_hw_run); 3099 /* No pending message exists in ROCEE mbox. */ 3100 if (!(status & MB_ST_HW_RUN_M)) 3101 break; 3102 } else if (!v2_chk_mbox_is_avail(hr_dev, &busy)) { 3103 break; 3104 } 3105 3106 if (time_after(jiffies, end)) { 3107 dev_err_ratelimited(hr_dev->dev, 3108 "failed to wait mbox status 0x%x\n", 3109 status); 3110 return -ETIMEDOUT; 3111 } 3112 3113 cond_resched(); 3114 ret = -EBUSY; 3115 } 3116 3117 if (!ret) { 3118 *complete_status = (u8)(status & MB_ST_COMPLETE_M); 3119 } else if (!v2_chk_mbox_is_avail(hr_dev, &busy)) { 3120 /* Ignore all errors if the mbox is unavailable. */ 3121 ret = 0; 3122 *complete_status = MB_ST_COMPLETE_M; 3123 } 3124 3125 return ret; 3126 } 3127 3128 static int v2_post_mbox(struct hns_roce_dev *hr_dev, 3129 struct hns_roce_mbox_msg *mbox_msg) 3130 { 3131 u8 status = 0; 3132 int ret; 3133 3134 /* Waiting for the mbox to be idle */ 3135 ret = v2_wait_mbox_complete(hr_dev, HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS, 3136 &status); 3137 if (unlikely(ret)) { 3138 dev_err_ratelimited(hr_dev->dev, 3139 "failed to check post mbox status = 0x%x, ret = %d.\n", 3140 status, ret); 3141 return ret; 3142 } 3143 3144 /* Post new message to mbox */ 3145 ret = hns_roce_mbox_post(hr_dev, mbox_msg); 3146 if (ret) 3147 dev_err_ratelimited(hr_dev->dev, 3148 "failed to post mailbox, ret = %d.\n", ret); 3149 3150 return ret; 3151 } 3152 3153 static int v2_poll_mbox_done(struct hns_roce_dev *hr_dev) 3154 { 3155 u8 status = 0; 3156 int ret; 3157 3158 ret = v2_wait_mbox_complete(hr_dev, HNS_ROCE_CMD_TIMEOUT_MSECS, 3159 &status); 3160 if (!ret) { 3161 if (status != MB_ST_COMPLETE_SUCC) 3162 return -EBUSY; 3163 } else { 3164 dev_err_ratelimited(hr_dev->dev, 3165 "failed to check mbox status = 0x%x, ret = %d.\n", 3166 status, ret); 3167 } 3168 3169 return ret; 3170 } 3171 3172 static void copy_gid(void *dest, const union ib_gid *gid) 3173 { 3174 #define GID_SIZE 4 3175 const union ib_gid *src = gid; 3176 __le32 (*p)[GID_SIZE] = dest; 3177 int i; 3178 3179 if (!gid) 3180 src = &zgid; 3181 3182 for (i = 0; i < GID_SIZE; i++) 3183 (*p)[i] = cpu_to_le32(*(u32 *)&src->raw[i * sizeof(u32)]); 3184 } 3185 3186 static int config_sgid_table(struct hns_roce_dev *hr_dev, 3187 int gid_index, const union ib_gid *gid, 3188 enum hns_roce_sgid_type sgid_type) 3189 { 3190 struct hns_roce_cmq_desc desc; 3191 struct hns_roce_cfg_sgid_tb *sgid_tb = 3192 (struct hns_roce_cfg_sgid_tb *)desc.data; 3193 3194 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SGID_TB, false); 3195 3196 hr_reg_write(sgid_tb, CFG_SGID_TB_TABLE_IDX, gid_index); 3197 hr_reg_write(sgid_tb, CFG_SGID_TB_VF_SGID_TYPE, sgid_type); 3198 3199 copy_gid(&sgid_tb->vf_sgid_l, gid); 3200 3201 return hns_roce_cmq_send(hr_dev, &desc, 1); 3202 } 3203 3204 static int config_gmv_table(struct hns_roce_dev *hr_dev, 3205 int gid_index, const union ib_gid *gid, 3206 enum hns_roce_sgid_type sgid_type, 3207 const struct ib_gid_attr *attr) 3208 { 3209 struct hns_roce_cmq_desc desc[2]; 3210 struct hns_roce_cfg_gmv_tb_a *tb_a = 3211 (struct hns_roce_cfg_gmv_tb_a *)desc[0].data; 3212 struct hns_roce_cfg_gmv_tb_b *tb_b = 3213 (struct hns_roce_cfg_gmv_tb_b *)desc[1].data; 3214 3215 u16 vlan_id = VLAN_CFI_MASK; 3216 u8 mac[ETH_ALEN] = {}; 3217 int ret; 3218 3219 if (gid) { 3220 ret = rdma_read_gid_l2_fields(attr, &vlan_id, mac); 3221 if (ret) 3222 return ret; 3223 } 3224 3225 hns_roce_cmq_setup_basic_desc(&desc[0], HNS_ROCE_OPC_CFG_GMV_TBL, false); 3226 desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); 3227 3228 hns_roce_cmq_setup_basic_desc(&desc[1], HNS_ROCE_OPC_CFG_GMV_TBL, false); 3229 3230 copy_gid(&tb_a->vf_sgid_l, gid); 3231 3232 hr_reg_write(tb_a, GMV_TB_A_VF_SGID_TYPE, sgid_type); 3233 hr_reg_write(tb_a, GMV_TB_A_VF_VLAN_EN, vlan_id < VLAN_CFI_MASK); 3234 hr_reg_write(tb_a, GMV_TB_A_VF_VLAN_ID, vlan_id); 3235 3236 tb_b->vf_smac_l = cpu_to_le32(*(u32 *)mac); 3237 3238 hr_reg_write(tb_b, GMV_TB_B_SMAC_H, *(u16 *)&mac[4]); 3239 hr_reg_write(tb_b, GMV_TB_B_SGID_IDX, gid_index); 3240 3241 return hns_roce_cmq_send(hr_dev, desc, 2); 3242 } 3243 3244 static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, int gid_index, 3245 const union ib_gid *gid, 3246 const struct ib_gid_attr *attr) 3247 { 3248 enum hns_roce_sgid_type sgid_type = GID_TYPE_FLAG_ROCE_V1; 3249 int ret; 3250 3251 if (gid) { 3252 if (attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) { 3253 if (ipv6_addr_v4mapped((void *)gid)) 3254 sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV4; 3255 else 3256 sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV6; 3257 } else if (attr->gid_type == IB_GID_TYPE_ROCE) { 3258 sgid_type = GID_TYPE_FLAG_ROCE_V1; 3259 } 3260 } 3261 3262 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) 3263 ret = config_gmv_table(hr_dev, gid_index, gid, sgid_type, attr); 3264 else 3265 ret = config_sgid_table(hr_dev, gid_index, gid, sgid_type); 3266 3267 if (ret) 3268 ibdev_err(&hr_dev->ib_dev, "failed to set gid, ret = %d!\n", 3269 ret); 3270 3271 return ret; 3272 } 3273 3274 static int hns_roce_v2_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port, 3275 const u8 *addr) 3276 { 3277 struct hns_roce_cmq_desc desc; 3278 struct hns_roce_cfg_smac_tb *smac_tb = 3279 (struct hns_roce_cfg_smac_tb *)desc.data; 3280 u16 reg_smac_h; 3281 u32 reg_smac_l; 3282 3283 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SMAC_TB, false); 3284 3285 reg_smac_l = *(u32 *)(&addr[0]); 3286 reg_smac_h = *(u16 *)(&addr[4]); 3287 3288 hr_reg_write(smac_tb, CFG_SMAC_TB_IDX, phy_port); 3289 hr_reg_write(smac_tb, CFG_SMAC_TB_VF_SMAC_H, reg_smac_h); 3290 smac_tb->vf_smac_l = cpu_to_le32(reg_smac_l); 3291 3292 return hns_roce_cmq_send(hr_dev, &desc, 1); 3293 } 3294 3295 static int set_mtpt_pbl(struct hns_roce_dev *hr_dev, 3296 struct hns_roce_v2_mpt_entry *mpt_entry, 3297 struct hns_roce_mr *mr) 3298 { 3299 u64 pages[HNS_ROCE_V2_MAX_INNER_MTPT_NUM] = { 0 }; 3300 struct ib_device *ibdev = &hr_dev->ib_dev; 3301 dma_addr_t pbl_ba; 3302 int i, count; 3303 3304 count = hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, pages, 3305 min_t(int, ARRAY_SIZE(pages), mr->npages), 3306 &pbl_ba); 3307 if (count < 1) { 3308 ibdev_err(ibdev, "failed to find PBL mtr, count = %d.\n", 3309 count); 3310 return -ENOBUFS; 3311 } 3312 3313 /* Aligned to the hardware address access unit */ 3314 for (i = 0; i < count; i++) 3315 pages[i] >>= 6; 3316 3317 mpt_entry->pbl_size = cpu_to_le32(mr->npages); 3318 mpt_entry->pbl_ba_l = cpu_to_le32(pbl_ba >> 3); 3319 hr_reg_write(mpt_entry, MPT_PBL_BA_H, upper_32_bits(pbl_ba >> 3)); 3320 3321 mpt_entry->pa0_l = cpu_to_le32(lower_32_bits(pages[0])); 3322 hr_reg_write(mpt_entry, MPT_PA0_H, upper_32_bits(pages[0])); 3323 3324 mpt_entry->pa1_l = cpu_to_le32(lower_32_bits(pages[1])); 3325 hr_reg_write(mpt_entry, MPT_PA1_H, upper_32_bits(pages[1])); 3326 hr_reg_write(mpt_entry, MPT_PBL_BUF_PG_SZ, 3327 to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift)); 3328 3329 return 0; 3330 } 3331 3332 static int hns_roce_v2_write_mtpt(struct hns_roce_dev *hr_dev, 3333 void *mb_buf, struct hns_roce_mr *mr) 3334 { 3335 struct hns_roce_v2_mpt_entry *mpt_entry; 3336 3337 mpt_entry = mb_buf; 3338 memset(mpt_entry, 0, sizeof(*mpt_entry)); 3339 3340 hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_VALID); 3341 hr_reg_write(mpt_entry, MPT_PD, mr->pd); 3342 3343 hr_reg_write_bool(mpt_entry, MPT_BIND_EN, 3344 mr->access & IB_ACCESS_MW_BIND); 3345 hr_reg_write_bool(mpt_entry, MPT_ATOMIC_EN, 3346 mr->access & IB_ACCESS_REMOTE_ATOMIC); 3347 hr_reg_write_bool(mpt_entry, MPT_RR_EN, 3348 mr->access & IB_ACCESS_REMOTE_READ); 3349 hr_reg_write_bool(mpt_entry, MPT_RW_EN, 3350 mr->access & IB_ACCESS_REMOTE_WRITE); 3351 hr_reg_write_bool(mpt_entry, MPT_LW_EN, 3352 mr->access & IB_ACCESS_LOCAL_WRITE); 3353 3354 mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size)); 3355 mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size)); 3356 mpt_entry->lkey = cpu_to_le32(mr->key); 3357 mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova)); 3358 mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova)); 3359 3360 if (mr->type != MR_TYPE_MR) 3361 hr_reg_enable(mpt_entry, MPT_PA); 3362 3363 if (mr->type == MR_TYPE_DMA) 3364 return 0; 3365 3366 if (mr->pbl_hop_num != HNS_ROCE_HOP_NUM_0) 3367 hr_reg_write(mpt_entry, MPT_PBL_HOP_NUM, mr->pbl_hop_num); 3368 3369 hr_reg_write(mpt_entry, MPT_PBL_BA_PG_SZ, 3370 to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.ba_pg_shift)); 3371 hr_reg_enable(mpt_entry, MPT_INNER_PA_VLD); 3372 3373 return set_mtpt_pbl(hr_dev, mpt_entry, mr); 3374 } 3375 3376 static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev, 3377 struct hns_roce_mr *mr, int flags, 3378 void *mb_buf) 3379 { 3380 struct hns_roce_v2_mpt_entry *mpt_entry = mb_buf; 3381 u32 mr_access_flags = mr->access; 3382 int ret = 0; 3383 3384 hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_VALID); 3385 hr_reg_write(mpt_entry, MPT_PD, mr->pd); 3386 3387 if (flags & IB_MR_REREG_ACCESS) { 3388 hr_reg_write(mpt_entry, MPT_BIND_EN, 3389 (mr_access_flags & IB_ACCESS_MW_BIND ? 1 : 0)); 3390 hr_reg_write(mpt_entry, MPT_ATOMIC_EN, 3391 mr_access_flags & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0); 3392 hr_reg_write(mpt_entry, MPT_RR_EN, 3393 mr_access_flags & IB_ACCESS_REMOTE_READ ? 1 : 0); 3394 hr_reg_write(mpt_entry, MPT_RW_EN, 3395 mr_access_flags & IB_ACCESS_REMOTE_WRITE ? 1 : 0); 3396 hr_reg_write(mpt_entry, MPT_LW_EN, 3397 mr_access_flags & IB_ACCESS_LOCAL_WRITE ? 1 : 0); 3398 } 3399 3400 if (flags & IB_MR_REREG_TRANS) { 3401 mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova)); 3402 mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova)); 3403 mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size)); 3404 mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size)); 3405 3406 ret = set_mtpt_pbl(hr_dev, mpt_entry, mr); 3407 } 3408 3409 return ret; 3410 } 3411 3412 static int hns_roce_v2_frmr_write_mtpt(struct hns_roce_dev *hr_dev, 3413 void *mb_buf, struct hns_roce_mr *mr) 3414 { 3415 struct ib_device *ibdev = &hr_dev->ib_dev; 3416 struct hns_roce_v2_mpt_entry *mpt_entry; 3417 dma_addr_t pbl_ba = 0; 3418 3419 mpt_entry = mb_buf; 3420 memset(mpt_entry, 0, sizeof(*mpt_entry)); 3421 3422 if (hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, NULL, 0, &pbl_ba) < 0) { 3423 ibdev_err(ibdev, "failed to find frmr mtr.\n"); 3424 return -ENOBUFS; 3425 } 3426 3427 hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_FREE); 3428 hr_reg_write(mpt_entry, MPT_PD, mr->pd); 3429 3430 hr_reg_enable(mpt_entry, MPT_RA_EN); 3431 hr_reg_enable(mpt_entry, MPT_R_INV_EN); 3432 3433 hr_reg_enable(mpt_entry, MPT_FRE); 3434 hr_reg_clear(mpt_entry, MPT_MR_MW); 3435 hr_reg_enable(mpt_entry, MPT_BPD); 3436 hr_reg_clear(mpt_entry, MPT_PA); 3437 3438 hr_reg_write(mpt_entry, MPT_PBL_HOP_NUM, 1); 3439 hr_reg_write(mpt_entry, MPT_PBL_BA_PG_SZ, 3440 to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.ba_pg_shift)); 3441 hr_reg_write(mpt_entry, MPT_PBL_BUF_PG_SZ, 3442 to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift)); 3443 3444 mpt_entry->pbl_size = cpu_to_le32(mr->npages); 3445 3446 mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(pbl_ba >> 3)); 3447 hr_reg_write(mpt_entry, MPT_PBL_BA_H, upper_32_bits(pbl_ba >> 3)); 3448 3449 return 0; 3450 } 3451 3452 static int hns_roce_v2_mw_write_mtpt(void *mb_buf, struct hns_roce_mw *mw) 3453 { 3454 struct hns_roce_v2_mpt_entry *mpt_entry; 3455 3456 mpt_entry = mb_buf; 3457 memset(mpt_entry, 0, sizeof(*mpt_entry)); 3458 3459 hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_FREE); 3460 hr_reg_write(mpt_entry, MPT_PD, mw->pdn); 3461 3462 hr_reg_enable(mpt_entry, MPT_R_INV_EN); 3463 hr_reg_enable(mpt_entry, MPT_LW_EN); 3464 3465 hr_reg_enable(mpt_entry, MPT_MR_MW); 3466 hr_reg_enable(mpt_entry, MPT_BPD); 3467 hr_reg_clear(mpt_entry, MPT_PA); 3468 hr_reg_write(mpt_entry, MPT_BQP, 3469 mw->ibmw.type == IB_MW_TYPE_1 ? 0 : 1); 3470 3471 mpt_entry->lkey = cpu_to_le32(mw->rkey); 3472 3473 hr_reg_write(mpt_entry, MPT_PBL_HOP_NUM, 3474 mw->pbl_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : 3475 mw->pbl_hop_num); 3476 hr_reg_write(mpt_entry, MPT_PBL_BA_PG_SZ, 3477 mw->pbl_ba_pg_sz + PG_SHIFT_OFFSET); 3478 hr_reg_write(mpt_entry, MPT_PBL_BUF_PG_SZ, 3479 mw->pbl_buf_pg_sz + PG_SHIFT_OFFSET); 3480 3481 return 0; 3482 } 3483 3484 static int free_mr_post_send_lp_wqe(struct hns_roce_qp *hr_qp) 3485 { 3486 struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp->ibqp.device); 3487 struct ib_device *ibdev = &hr_dev->ib_dev; 3488 const struct ib_send_wr *bad_wr; 3489 struct ib_rdma_wr rdma_wr = {}; 3490 struct ib_send_wr *send_wr; 3491 int ret; 3492 3493 send_wr = &rdma_wr.wr; 3494 send_wr->opcode = IB_WR_RDMA_WRITE; 3495 3496 ret = hns_roce_v2_post_send(&hr_qp->ibqp, send_wr, &bad_wr); 3497 if (ret) { 3498 ibdev_err(ibdev, "failed to post wqe for free mr, ret = %d.\n", 3499 ret); 3500 return ret; 3501 } 3502 3503 return 0; 3504 } 3505 3506 static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries, 3507 struct ib_wc *wc); 3508 3509 static void free_mr_send_cmd_to_hw(struct hns_roce_dev *hr_dev) 3510 { 3511 struct hns_roce_v2_priv *priv = hr_dev->priv; 3512 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr; 3513 struct ib_wc wc[ARRAY_SIZE(free_mr->rsv_qp)]; 3514 struct ib_device *ibdev = &hr_dev->ib_dev; 3515 struct hns_roce_qp *hr_qp; 3516 unsigned long end; 3517 int cqe_cnt = 0; 3518 int npolled; 3519 int ret; 3520 int i; 3521 3522 /* 3523 * If the device initialization is not complete or in the uninstall 3524 * process, then there is no need to execute free mr. 3525 */ 3526 if (priv->handle->rinfo.reset_state == HNS_ROCE_STATE_RST_INIT || 3527 priv->handle->rinfo.instance_state == HNS_ROCE_STATE_INIT || 3528 hr_dev->state == HNS_ROCE_DEVICE_STATE_UNINIT) 3529 return; 3530 3531 mutex_lock(&free_mr->mutex); 3532 3533 for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) { 3534 hr_qp = free_mr->rsv_qp[i]; 3535 3536 ret = free_mr_post_send_lp_wqe(hr_qp); 3537 if (ret) { 3538 ibdev_err(ibdev, 3539 "failed to send wqe (qp:0x%lx) for free mr, ret = %d.\n", 3540 hr_qp->qpn, ret); 3541 break; 3542 } 3543 3544 cqe_cnt++; 3545 } 3546 3547 end = msecs_to_jiffies(HNS_ROCE_V2_FREE_MR_TIMEOUT) + jiffies; 3548 while (cqe_cnt) { 3549 npolled = hns_roce_v2_poll_cq(&free_mr->rsv_cq->ib_cq, cqe_cnt, wc); 3550 if (npolled < 0) { 3551 ibdev_err(ibdev, 3552 "failed to poll cqe for free mr, remain %d cqe.\n", 3553 cqe_cnt); 3554 goto out; 3555 } 3556 3557 if (time_after(jiffies, end)) { 3558 ibdev_err(ibdev, 3559 "failed to poll cqe for free mr and timeout, remain %d cqe.\n", 3560 cqe_cnt); 3561 goto out; 3562 } 3563 cqe_cnt -= npolled; 3564 } 3565 3566 out: 3567 mutex_unlock(&free_mr->mutex); 3568 } 3569 3570 static void hns_roce_v2_dereg_mr(struct hns_roce_dev *hr_dev) 3571 { 3572 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) 3573 free_mr_send_cmd_to_hw(hr_dev); 3574 } 3575 3576 static void *get_cqe_v2(struct hns_roce_cq *hr_cq, int n) 3577 { 3578 return hns_roce_buf_offset(hr_cq->mtr.kmem, n * hr_cq->cqe_size); 3579 } 3580 3581 static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, unsigned int n) 3582 { 3583 struct hns_roce_v2_cqe *cqe = get_cqe_v2(hr_cq, n & hr_cq->ib_cq.cqe); 3584 3585 /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */ 3586 return (hr_reg_read(cqe, CQE_OWNER) ^ !!(n & hr_cq->cq_depth)) ? cqe : 3587 NULL; 3588 } 3589 3590 static inline void update_cq_db(struct hns_roce_dev *hr_dev, 3591 struct hns_roce_cq *hr_cq) 3592 { 3593 if (likely(hr_cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB)) { 3594 *hr_cq->set_ci_db = hr_cq->cons_index & V2_CQ_DB_CONS_IDX_M; 3595 } else { 3596 struct hns_roce_v2_db cq_db = {}; 3597 3598 hr_reg_write(&cq_db, DB_TAG, hr_cq->cqn); 3599 hr_reg_write(&cq_db, DB_CMD, HNS_ROCE_V2_CQ_DB); 3600 hr_reg_write(&cq_db, DB_CQ_CI, hr_cq->cons_index); 3601 hr_reg_write(&cq_db, DB_CQ_CMD_SN, 1); 3602 3603 hns_roce_write64(hr_dev, (__le32 *)&cq_db, hr_cq->db_reg); 3604 } 3605 } 3606 3607 static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn, 3608 struct hns_roce_srq *srq) 3609 { 3610 struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device); 3611 struct hns_roce_v2_cqe *cqe, *dest; 3612 u32 prod_index; 3613 int nfreed = 0; 3614 int wqe_index; 3615 u8 owner_bit; 3616 3617 for (prod_index = hr_cq->cons_index; get_sw_cqe_v2(hr_cq, prod_index); 3618 ++prod_index) { 3619 if (prod_index > hr_cq->cons_index + hr_cq->ib_cq.cqe) 3620 break; 3621 } 3622 3623 /* 3624 * Now backwards through the CQ, removing CQ entries 3625 * that match our QP by overwriting them with next entries. 3626 */ 3627 while ((int) --prod_index - (int) hr_cq->cons_index >= 0) { 3628 cqe = get_cqe_v2(hr_cq, prod_index & hr_cq->ib_cq.cqe); 3629 if (hr_reg_read(cqe, CQE_LCL_QPN) == qpn) { 3630 if (srq && hr_reg_read(cqe, CQE_S_R)) { 3631 wqe_index = hr_reg_read(cqe, CQE_WQE_IDX); 3632 hns_roce_free_srq_wqe(srq, wqe_index); 3633 } 3634 ++nfreed; 3635 } else if (nfreed) { 3636 dest = get_cqe_v2(hr_cq, (prod_index + nfreed) & 3637 hr_cq->ib_cq.cqe); 3638 owner_bit = hr_reg_read(dest, CQE_OWNER); 3639 memcpy(dest, cqe, hr_cq->cqe_size); 3640 hr_reg_write(dest, CQE_OWNER, owner_bit); 3641 } 3642 } 3643 3644 if (nfreed) { 3645 hr_cq->cons_index += nfreed; 3646 update_cq_db(hr_dev, hr_cq); 3647 } 3648 } 3649 3650 static void hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn, 3651 struct hns_roce_srq *srq) 3652 { 3653 spin_lock_irq(&hr_cq->lock); 3654 __hns_roce_v2_cq_clean(hr_cq, qpn, srq); 3655 spin_unlock_irq(&hr_cq->lock); 3656 } 3657 3658 static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev, 3659 struct hns_roce_cq *hr_cq, void *mb_buf, 3660 u64 *mtts, dma_addr_t dma_handle) 3661 { 3662 struct hns_roce_v2_cq_context *cq_context; 3663 3664 cq_context = mb_buf; 3665 memset(cq_context, 0, sizeof(*cq_context)); 3666 3667 hr_reg_write(cq_context, CQC_CQ_ST, V2_CQ_STATE_VALID); 3668 hr_reg_write(cq_context, CQC_ARM_ST, NO_ARMED); 3669 hr_reg_write(cq_context, CQC_SHIFT, ilog2(hr_cq->cq_depth)); 3670 hr_reg_write(cq_context, CQC_CEQN, hr_cq->vector); 3671 hr_reg_write(cq_context, CQC_CQN, hr_cq->cqn); 3672 3673 if (hr_cq->cqe_size == HNS_ROCE_V3_CQE_SIZE) 3674 hr_reg_write(cq_context, CQC_CQE_SIZE, CQE_SIZE_64B); 3675 3676 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_STASH) 3677 hr_reg_enable(cq_context, CQC_STASH); 3678 3679 hr_reg_write(cq_context, CQC_CQE_CUR_BLK_ADDR_L, 3680 to_hr_hw_page_addr(mtts[0])); 3681 hr_reg_write(cq_context, CQC_CQE_CUR_BLK_ADDR_H, 3682 upper_32_bits(to_hr_hw_page_addr(mtts[0]))); 3683 hr_reg_write(cq_context, CQC_CQE_HOP_NUM, hr_dev->caps.cqe_hop_num == 3684 HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev->caps.cqe_hop_num); 3685 hr_reg_write(cq_context, CQC_CQE_NEX_BLK_ADDR_L, 3686 to_hr_hw_page_addr(mtts[1])); 3687 hr_reg_write(cq_context, CQC_CQE_NEX_BLK_ADDR_H, 3688 upper_32_bits(to_hr_hw_page_addr(mtts[1]))); 3689 hr_reg_write(cq_context, CQC_CQE_BAR_PG_SZ, 3690 to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.ba_pg_shift)); 3691 hr_reg_write(cq_context, CQC_CQE_BUF_PG_SZ, 3692 to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.buf_pg_shift)); 3693 hr_reg_write(cq_context, CQC_CQE_BA_L, dma_handle >> 3); 3694 hr_reg_write(cq_context, CQC_CQE_BA_H, (dma_handle >> (32 + 3))); 3695 hr_reg_write_bool(cq_context, CQC_DB_RECORD_EN, 3696 hr_cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB); 3697 hr_reg_write(cq_context, CQC_CQE_DB_RECORD_ADDR_L, 3698 ((u32)hr_cq->db.dma) >> 1); 3699 hr_reg_write(cq_context, CQC_CQE_DB_RECORD_ADDR_H, 3700 hr_cq->db.dma >> 32); 3701 hr_reg_write(cq_context, CQC_CQ_MAX_CNT, 3702 HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM); 3703 hr_reg_write(cq_context, CQC_CQ_PERIOD, 3704 HNS_ROCE_V2_CQ_DEFAULT_INTERVAL); 3705 } 3706 3707 static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq, 3708 enum ib_cq_notify_flags flags) 3709 { 3710 struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device); 3711 struct hns_roce_cq *hr_cq = to_hr_cq(ibcq); 3712 struct hns_roce_v2_db cq_db = {}; 3713 u32 notify_flag; 3714 3715 /* 3716 * flags = 0, then notify_flag : next 3717 * flags = 1, then notify flag : solocited 3718 */ 3719 notify_flag = (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ? 3720 V2_CQ_DB_REQ_NOT : V2_CQ_DB_REQ_NOT_SOL; 3721 3722 hr_reg_write(&cq_db, DB_TAG, hr_cq->cqn); 3723 hr_reg_write(&cq_db, DB_CMD, HNS_ROCE_V2_CQ_DB_NOTIFY); 3724 hr_reg_write(&cq_db, DB_CQ_CI, hr_cq->cons_index); 3725 hr_reg_write(&cq_db, DB_CQ_CMD_SN, hr_cq->arm_sn); 3726 hr_reg_write(&cq_db, DB_CQ_NOTIFY, notify_flag); 3727 3728 hns_roce_write64(hr_dev, (__le32 *)&cq_db, hr_cq->db_reg); 3729 3730 return 0; 3731 } 3732 3733 static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe, 3734 struct hns_roce_qp *qp, 3735 struct ib_wc *wc) 3736 { 3737 struct hns_roce_rinl_sge *sge_list; 3738 u32 wr_num, wr_cnt, sge_num; 3739 u32 sge_cnt, data_len, size; 3740 void *wqe_buf; 3741 3742 wr_num = hr_reg_read(cqe, CQE_WQE_IDX); 3743 wr_cnt = wr_num & (qp->rq.wqe_cnt - 1); 3744 3745 sge_list = qp->rq_inl_buf.wqe_list[wr_cnt].sg_list; 3746 sge_num = qp->rq_inl_buf.wqe_list[wr_cnt].sge_cnt; 3747 wqe_buf = hns_roce_get_recv_wqe(qp, wr_cnt); 3748 data_len = wc->byte_len; 3749 3750 for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len); sge_cnt++) { 3751 size = min(sge_list[sge_cnt].len, data_len); 3752 memcpy((void *)sge_list[sge_cnt].addr, wqe_buf, size); 3753 3754 data_len -= size; 3755 wqe_buf += size; 3756 } 3757 3758 if (unlikely(data_len)) { 3759 wc->status = IB_WC_LOC_LEN_ERR; 3760 return -EAGAIN; 3761 } 3762 3763 return 0; 3764 } 3765 3766 static int sw_comp(struct hns_roce_qp *hr_qp, struct hns_roce_wq *wq, 3767 int num_entries, struct ib_wc *wc) 3768 { 3769 unsigned int left; 3770 int npolled = 0; 3771 3772 left = wq->head - wq->tail; 3773 if (left == 0) 3774 return 0; 3775 3776 left = min_t(unsigned int, (unsigned int)num_entries, left); 3777 while (npolled < left) { 3778 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; 3779 wc->status = IB_WC_WR_FLUSH_ERR; 3780 wc->vendor_err = 0; 3781 wc->qp = &hr_qp->ibqp; 3782 3783 wq->tail++; 3784 wc++; 3785 npolled++; 3786 } 3787 3788 return npolled; 3789 } 3790 3791 static int hns_roce_v2_sw_poll_cq(struct hns_roce_cq *hr_cq, int num_entries, 3792 struct ib_wc *wc) 3793 { 3794 struct hns_roce_qp *hr_qp; 3795 int npolled = 0; 3796 3797 list_for_each_entry(hr_qp, &hr_cq->sq_list, sq_node) { 3798 npolled += sw_comp(hr_qp, &hr_qp->sq, 3799 num_entries - npolled, wc + npolled); 3800 if (npolled >= num_entries) 3801 goto out; 3802 } 3803 3804 list_for_each_entry(hr_qp, &hr_cq->rq_list, rq_node) { 3805 npolled += sw_comp(hr_qp, &hr_qp->rq, 3806 num_entries - npolled, wc + npolled); 3807 if (npolled >= num_entries) 3808 goto out; 3809 } 3810 3811 out: 3812 return npolled; 3813 } 3814 3815 static void get_cqe_status(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp, 3816 struct hns_roce_cq *cq, struct hns_roce_v2_cqe *cqe, 3817 struct ib_wc *wc) 3818 { 3819 static const struct { 3820 u32 cqe_status; 3821 enum ib_wc_status wc_status; 3822 } map[] = { 3823 { HNS_ROCE_CQE_V2_SUCCESS, IB_WC_SUCCESS }, 3824 { HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR, IB_WC_LOC_LEN_ERR }, 3825 { HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR, IB_WC_LOC_QP_OP_ERR }, 3826 { HNS_ROCE_CQE_V2_LOCAL_PROT_ERR, IB_WC_LOC_PROT_ERR }, 3827 { HNS_ROCE_CQE_V2_WR_FLUSH_ERR, IB_WC_WR_FLUSH_ERR }, 3828 { HNS_ROCE_CQE_V2_MW_BIND_ERR, IB_WC_MW_BIND_ERR }, 3829 { HNS_ROCE_CQE_V2_BAD_RESP_ERR, IB_WC_BAD_RESP_ERR }, 3830 { HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR, IB_WC_LOC_ACCESS_ERR }, 3831 { HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR, IB_WC_REM_INV_REQ_ERR }, 3832 { HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR, IB_WC_REM_ACCESS_ERR }, 3833 { HNS_ROCE_CQE_V2_REMOTE_OP_ERR, IB_WC_REM_OP_ERR }, 3834 { HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR, 3835 IB_WC_RETRY_EXC_ERR }, 3836 { HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR, IB_WC_RNR_RETRY_EXC_ERR }, 3837 { HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR, IB_WC_REM_ABORT_ERR }, 3838 { HNS_ROCE_CQE_V2_GENERAL_ERR, IB_WC_GENERAL_ERR} 3839 }; 3840 3841 u32 cqe_status = hr_reg_read(cqe, CQE_STATUS); 3842 int i; 3843 3844 wc->status = IB_WC_GENERAL_ERR; 3845 for (i = 0; i < ARRAY_SIZE(map); i++) 3846 if (cqe_status == map[i].cqe_status) { 3847 wc->status = map[i].wc_status; 3848 break; 3849 } 3850 3851 if (likely(wc->status == IB_WC_SUCCESS || 3852 wc->status == IB_WC_WR_FLUSH_ERR)) 3853 return; 3854 3855 ibdev_err(&hr_dev->ib_dev, "error cqe status 0x%x:\n", cqe_status); 3856 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_NONE, 16, 4, cqe, 3857 cq->cqe_size, false); 3858 wc->vendor_err = hr_reg_read(cqe, CQE_SUB_STATUS); 3859 3860 /* 3861 * For hns ROCEE, GENERAL_ERR is an error type that is not defined in 3862 * the standard protocol, the driver must ignore it and needn't to set 3863 * the QP to an error state. 3864 */ 3865 if (cqe_status == HNS_ROCE_CQE_V2_GENERAL_ERR) 3866 return; 3867 3868 flush_cqe(hr_dev, qp); 3869 } 3870 3871 static int get_cur_qp(struct hns_roce_cq *hr_cq, struct hns_roce_v2_cqe *cqe, 3872 struct hns_roce_qp **cur_qp) 3873 { 3874 struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device); 3875 struct hns_roce_qp *hr_qp = *cur_qp; 3876 u32 qpn; 3877 3878 qpn = hr_reg_read(cqe, CQE_LCL_QPN); 3879 3880 if (!hr_qp || qpn != hr_qp->qpn) { 3881 hr_qp = __hns_roce_qp_lookup(hr_dev, qpn); 3882 if (unlikely(!hr_qp)) { 3883 ibdev_err(&hr_dev->ib_dev, 3884 "CQ %06lx with entry for unknown QPN %06x\n", 3885 hr_cq->cqn, qpn); 3886 return -EINVAL; 3887 } 3888 *cur_qp = hr_qp; 3889 } 3890 3891 return 0; 3892 } 3893 3894 /* 3895 * mapped-value = 1 + real-value 3896 * The ib wc opcode's real value is start from 0, In order to distinguish 3897 * between initialized and uninitialized map values, we plus 1 to the actual 3898 * value when defining the mapping, so that the validity can be identified by 3899 * checking whether the mapped value is greater than 0. 3900 */ 3901 #define HR_WC_OP_MAP(hr_key, ib_key) \ 3902 [HNS_ROCE_V2_WQE_OP_ ## hr_key] = 1 + IB_WC_ ## ib_key 3903 3904 static const u32 wc_send_op_map[] = { 3905 HR_WC_OP_MAP(SEND, SEND), 3906 HR_WC_OP_MAP(SEND_WITH_INV, SEND), 3907 HR_WC_OP_MAP(SEND_WITH_IMM, SEND), 3908 HR_WC_OP_MAP(RDMA_READ, RDMA_READ), 3909 HR_WC_OP_MAP(RDMA_WRITE, RDMA_WRITE), 3910 HR_WC_OP_MAP(RDMA_WRITE_WITH_IMM, RDMA_WRITE), 3911 HR_WC_OP_MAP(ATOM_CMP_AND_SWAP, COMP_SWAP), 3912 HR_WC_OP_MAP(ATOM_FETCH_AND_ADD, FETCH_ADD), 3913 HR_WC_OP_MAP(ATOM_MSK_CMP_AND_SWAP, MASKED_COMP_SWAP), 3914 HR_WC_OP_MAP(ATOM_MSK_FETCH_AND_ADD, MASKED_FETCH_ADD), 3915 HR_WC_OP_MAP(FAST_REG_PMR, REG_MR), 3916 HR_WC_OP_MAP(BIND_MW, REG_MR), 3917 }; 3918 3919 static int to_ib_wc_send_op(u32 hr_opcode) 3920 { 3921 if (hr_opcode >= ARRAY_SIZE(wc_send_op_map)) 3922 return -EINVAL; 3923 3924 return wc_send_op_map[hr_opcode] ? wc_send_op_map[hr_opcode] - 1 : 3925 -EINVAL; 3926 } 3927 3928 static const u32 wc_recv_op_map[] = { 3929 HR_WC_OP_MAP(RDMA_WRITE_WITH_IMM, WITH_IMM), 3930 HR_WC_OP_MAP(SEND, RECV), 3931 HR_WC_OP_MAP(SEND_WITH_IMM, WITH_IMM), 3932 HR_WC_OP_MAP(SEND_WITH_INV, RECV), 3933 }; 3934 3935 static int to_ib_wc_recv_op(u32 hr_opcode) 3936 { 3937 if (hr_opcode >= ARRAY_SIZE(wc_recv_op_map)) 3938 return -EINVAL; 3939 3940 return wc_recv_op_map[hr_opcode] ? wc_recv_op_map[hr_opcode] - 1 : 3941 -EINVAL; 3942 } 3943 3944 static void fill_send_wc(struct ib_wc *wc, struct hns_roce_v2_cqe *cqe) 3945 { 3946 u32 hr_opcode; 3947 int ib_opcode; 3948 3949 wc->wc_flags = 0; 3950 3951 hr_opcode = hr_reg_read(cqe, CQE_OPCODE); 3952 switch (hr_opcode) { 3953 case HNS_ROCE_V2_WQE_OP_RDMA_READ: 3954 wc->byte_len = le32_to_cpu(cqe->byte_cnt); 3955 break; 3956 case HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM: 3957 case HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM: 3958 wc->wc_flags |= IB_WC_WITH_IMM; 3959 break; 3960 case HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP: 3961 case HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD: 3962 case HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP: 3963 case HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD: 3964 wc->byte_len = 8; 3965 break; 3966 default: 3967 break; 3968 } 3969 3970 ib_opcode = to_ib_wc_send_op(hr_opcode); 3971 if (ib_opcode < 0) 3972 wc->status = IB_WC_GENERAL_ERR; 3973 else 3974 wc->opcode = ib_opcode; 3975 } 3976 3977 static inline bool is_rq_inl_enabled(struct ib_wc *wc, u32 hr_opcode, 3978 struct hns_roce_v2_cqe *cqe) 3979 { 3980 return wc->qp->qp_type != IB_QPT_UD && wc->qp->qp_type != IB_QPT_GSI && 3981 (hr_opcode == HNS_ROCE_V2_OPCODE_SEND || 3982 hr_opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_IMM || 3983 hr_opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_INV) && 3984 hr_reg_read(cqe, CQE_RQ_INLINE); 3985 } 3986 3987 static int fill_recv_wc(struct ib_wc *wc, struct hns_roce_v2_cqe *cqe) 3988 { 3989 struct hns_roce_qp *qp = to_hr_qp(wc->qp); 3990 u32 hr_opcode; 3991 int ib_opcode; 3992 int ret; 3993 3994 wc->byte_len = le32_to_cpu(cqe->byte_cnt); 3995 3996 hr_opcode = hr_reg_read(cqe, CQE_OPCODE); 3997 switch (hr_opcode) { 3998 case HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM: 3999 case HNS_ROCE_V2_OPCODE_SEND_WITH_IMM: 4000 wc->wc_flags = IB_WC_WITH_IMM; 4001 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(cqe->immtdata)); 4002 break; 4003 case HNS_ROCE_V2_OPCODE_SEND_WITH_INV: 4004 wc->wc_flags = IB_WC_WITH_INVALIDATE; 4005 wc->ex.invalidate_rkey = le32_to_cpu(cqe->rkey); 4006 break; 4007 default: 4008 wc->wc_flags = 0; 4009 } 4010 4011 ib_opcode = to_ib_wc_recv_op(hr_opcode); 4012 if (ib_opcode < 0) 4013 wc->status = IB_WC_GENERAL_ERR; 4014 else 4015 wc->opcode = ib_opcode; 4016 4017 if (is_rq_inl_enabled(wc, hr_opcode, cqe)) { 4018 ret = hns_roce_handle_recv_inl_wqe(cqe, qp, wc); 4019 if (unlikely(ret)) 4020 return ret; 4021 } 4022 4023 wc->sl = hr_reg_read(cqe, CQE_SL); 4024 wc->src_qp = hr_reg_read(cqe, CQE_RMT_QPN); 4025 wc->slid = 0; 4026 wc->wc_flags |= hr_reg_read(cqe, CQE_GRH) ? IB_WC_GRH : 0; 4027 wc->port_num = hr_reg_read(cqe, CQE_PORTN); 4028 wc->pkey_index = 0; 4029 4030 if (hr_reg_read(cqe, CQE_VID_VLD)) { 4031 wc->vlan_id = hr_reg_read(cqe, CQE_VID); 4032 wc->wc_flags |= IB_WC_WITH_VLAN; 4033 } else { 4034 wc->vlan_id = 0xffff; 4035 } 4036 4037 wc->network_hdr_type = hr_reg_read(cqe, CQE_PORT_TYPE); 4038 4039 return 0; 4040 } 4041 4042 static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq, 4043 struct hns_roce_qp **cur_qp, struct ib_wc *wc) 4044 { 4045 struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device); 4046 struct hns_roce_qp *qp = *cur_qp; 4047 struct hns_roce_srq *srq = NULL; 4048 struct hns_roce_v2_cqe *cqe; 4049 struct hns_roce_wq *wq; 4050 int is_send; 4051 u16 wqe_idx; 4052 int ret; 4053 4054 cqe = get_sw_cqe_v2(hr_cq, hr_cq->cons_index); 4055 if (!cqe) 4056 return -EAGAIN; 4057 4058 ++hr_cq->cons_index; 4059 /* Memory barrier */ 4060 rmb(); 4061 4062 ret = get_cur_qp(hr_cq, cqe, &qp); 4063 if (ret) 4064 return ret; 4065 4066 wc->qp = &qp->ibqp; 4067 wc->vendor_err = 0; 4068 4069 wqe_idx = hr_reg_read(cqe, CQE_WQE_IDX); 4070 4071 is_send = !hr_reg_read(cqe, CQE_S_R); 4072 if (is_send) { 4073 wq = &qp->sq; 4074 4075 /* If sg_signal_bit is set, tail pointer will be updated to 4076 * the WQE corresponding to the current CQE. 4077 */ 4078 if (qp->sq_signal_bits) 4079 wq->tail += (wqe_idx - (u16)wq->tail) & 4080 (wq->wqe_cnt - 1); 4081 4082 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; 4083 ++wq->tail; 4084 4085 fill_send_wc(wc, cqe); 4086 } else { 4087 if (qp->ibqp.srq) { 4088 srq = to_hr_srq(qp->ibqp.srq); 4089 wc->wr_id = srq->wrid[wqe_idx]; 4090 hns_roce_free_srq_wqe(srq, wqe_idx); 4091 } else { 4092 wq = &qp->rq; 4093 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)]; 4094 ++wq->tail; 4095 } 4096 4097 ret = fill_recv_wc(wc, cqe); 4098 } 4099 4100 get_cqe_status(hr_dev, qp, hr_cq, cqe, wc); 4101 if (unlikely(wc->status != IB_WC_SUCCESS)) 4102 return 0; 4103 4104 return ret; 4105 } 4106 4107 static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries, 4108 struct ib_wc *wc) 4109 { 4110 struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device); 4111 struct hns_roce_cq *hr_cq = to_hr_cq(ibcq); 4112 struct hns_roce_qp *cur_qp = NULL; 4113 unsigned long flags; 4114 int npolled; 4115 4116 spin_lock_irqsave(&hr_cq->lock, flags); 4117 4118 /* 4119 * When the device starts to reset, the state is RST_DOWN. At this time, 4120 * there may still be some valid CQEs in the hardware that are not 4121 * polled. Therefore, it is not allowed to switch to the software mode 4122 * immediately. When the state changes to UNINIT, CQE no longer exists 4123 * in the hardware, and then switch to software mode. 4124 */ 4125 if (hr_dev->state == HNS_ROCE_DEVICE_STATE_UNINIT) { 4126 npolled = hns_roce_v2_sw_poll_cq(hr_cq, num_entries, wc); 4127 goto out; 4128 } 4129 4130 for (npolled = 0; npolled < num_entries; ++npolled) { 4131 if (hns_roce_v2_poll_one(hr_cq, &cur_qp, wc + npolled)) 4132 break; 4133 } 4134 4135 if (npolled) 4136 update_cq_db(hr_dev, hr_cq); 4137 4138 out: 4139 spin_unlock_irqrestore(&hr_cq->lock, flags); 4140 4141 return npolled; 4142 } 4143 4144 static int get_op_for_set_hem(struct hns_roce_dev *hr_dev, u32 type, 4145 u32 step_idx, u8 *mbox_cmd) 4146 { 4147 u8 cmd; 4148 4149 switch (type) { 4150 case HEM_TYPE_QPC: 4151 cmd = HNS_ROCE_CMD_WRITE_QPC_BT0; 4152 break; 4153 case HEM_TYPE_MTPT: 4154 cmd = HNS_ROCE_CMD_WRITE_MPT_BT0; 4155 break; 4156 case HEM_TYPE_CQC: 4157 cmd = HNS_ROCE_CMD_WRITE_CQC_BT0; 4158 break; 4159 case HEM_TYPE_SRQC: 4160 cmd = HNS_ROCE_CMD_WRITE_SRQC_BT0; 4161 break; 4162 case HEM_TYPE_SCCC: 4163 cmd = HNS_ROCE_CMD_WRITE_SCCC_BT0; 4164 break; 4165 case HEM_TYPE_QPC_TIMER: 4166 cmd = HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0; 4167 break; 4168 case HEM_TYPE_CQC_TIMER: 4169 cmd = HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0; 4170 break; 4171 default: 4172 dev_warn(hr_dev->dev, "failed to check hem type %u.\n", type); 4173 return -EINVAL; 4174 } 4175 4176 *mbox_cmd = cmd + step_idx; 4177 4178 return 0; 4179 } 4180 4181 static int config_gmv_ba_to_hw(struct hns_roce_dev *hr_dev, unsigned long obj, 4182 dma_addr_t base_addr) 4183 { 4184 struct hns_roce_cmq_desc desc; 4185 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data; 4186 u32 idx = obj / (HNS_HW_PAGE_SIZE / hr_dev->caps.gmv_entry_sz); 4187 u64 addr = to_hr_hw_page_addr(base_addr); 4188 4189 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT, false); 4190 4191 hr_reg_write(req, CFG_GMV_BT_BA_L, lower_32_bits(addr)); 4192 hr_reg_write(req, CFG_GMV_BT_BA_H, upper_32_bits(addr)); 4193 hr_reg_write(req, CFG_GMV_BT_IDX, idx); 4194 4195 return hns_roce_cmq_send(hr_dev, &desc, 1); 4196 } 4197 4198 static int set_hem_to_hw(struct hns_roce_dev *hr_dev, int obj, 4199 dma_addr_t base_addr, u32 hem_type, u32 step_idx) 4200 { 4201 int ret; 4202 u8 cmd; 4203 4204 if (unlikely(hem_type == HEM_TYPE_GMV)) 4205 return config_gmv_ba_to_hw(hr_dev, obj, base_addr); 4206 4207 if (unlikely(hem_type == HEM_TYPE_SCCC && step_idx)) 4208 return 0; 4209 4210 ret = get_op_for_set_hem(hr_dev, hem_type, step_idx, &cmd); 4211 if (ret < 0) 4212 return ret; 4213 4214 return config_hem_ba_to_hw(hr_dev, base_addr, cmd, obj); 4215 } 4216 4217 static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev, 4218 struct hns_roce_hem_table *table, int obj, 4219 u32 step_idx) 4220 { 4221 struct hns_roce_hem_iter iter; 4222 struct hns_roce_hem_mhop mhop; 4223 struct hns_roce_hem *hem; 4224 unsigned long mhop_obj = obj; 4225 int i, j, k; 4226 int ret = 0; 4227 u64 hem_idx = 0; 4228 u64 l1_idx = 0; 4229 u64 bt_ba = 0; 4230 u32 chunk_ba_num; 4231 u32 hop_num; 4232 4233 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) 4234 return 0; 4235 4236 hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop); 4237 i = mhop.l0_idx; 4238 j = mhop.l1_idx; 4239 k = mhop.l2_idx; 4240 hop_num = mhop.hop_num; 4241 chunk_ba_num = mhop.bt_chunk_size / 8; 4242 4243 if (hop_num == 2) { 4244 hem_idx = i * chunk_ba_num * chunk_ba_num + j * chunk_ba_num + 4245 k; 4246 l1_idx = i * chunk_ba_num + j; 4247 } else if (hop_num == 1) { 4248 hem_idx = i * chunk_ba_num + j; 4249 } else if (hop_num == HNS_ROCE_HOP_NUM_0) { 4250 hem_idx = i; 4251 } 4252 4253 if (table->type == HEM_TYPE_SCCC) 4254 obj = mhop.l0_idx; 4255 4256 if (check_whether_last_step(hop_num, step_idx)) { 4257 hem = table->hem[hem_idx]; 4258 for (hns_roce_hem_first(hem, &iter); 4259 !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) { 4260 bt_ba = hns_roce_hem_addr(&iter); 4261 ret = set_hem_to_hw(hr_dev, obj, bt_ba, table->type, 4262 step_idx); 4263 } 4264 } else { 4265 if (step_idx == 0) 4266 bt_ba = table->bt_l0_dma_addr[i]; 4267 else if (step_idx == 1 && hop_num == 2) 4268 bt_ba = table->bt_l1_dma_addr[l1_idx]; 4269 4270 ret = set_hem_to_hw(hr_dev, obj, bt_ba, table->type, step_idx); 4271 } 4272 4273 return ret; 4274 } 4275 4276 static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev, 4277 struct hns_roce_hem_table *table, 4278 int tag, u32 step_idx) 4279 { 4280 struct hns_roce_cmd_mailbox *mailbox; 4281 struct device *dev = hr_dev->dev; 4282 u8 cmd = 0xff; 4283 int ret; 4284 4285 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) 4286 return 0; 4287 4288 switch (table->type) { 4289 case HEM_TYPE_QPC: 4290 cmd = HNS_ROCE_CMD_DESTROY_QPC_BT0; 4291 break; 4292 case HEM_TYPE_MTPT: 4293 cmd = HNS_ROCE_CMD_DESTROY_MPT_BT0; 4294 break; 4295 case HEM_TYPE_CQC: 4296 cmd = HNS_ROCE_CMD_DESTROY_CQC_BT0; 4297 break; 4298 case HEM_TYPE_SRQC: 4299 cmd = HNS_ROCE_CMD_DESTROY_SRQC_BT0; 4300 break; 4301 case HEM_TYPE_SCCC: 4302 case HEM_TYPE_QPC_TIMER: 4303 case HEM_TYPE_CQC_TIMER: 4304 case HEM_TYPE_GMV: 4305 return 0; 4306 default: 4307 dev_warn(dev, "table %u not to be destroyed by mailbox!\n", 4308 table->type); 4309 return 0; 4310 } 4311 4312 cmd += step_idx; 4313 4314 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 4315 if (IS_ERR(mailbox)) 4316 return PTR_ERR(mailbox); 4317 4318 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, cmd, tag); 4319 4320 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 4321 return ret; 4322 } 4323 4324 static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev, 4325 struct hns_roce_v2_qp_context *context, 4326 struct hns_roce_v2_qp_context *qpc_mask, 4327 struct hns_roce_qp *hr_qp) 4328 { 4329 struct hns_roce_cmd_mailbox *mailbox; 4330 int qpc_size; 4331 int ret; 4332 4333 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 4334 if (IS_ERR(mailbox)) 4335 return PTR_ERR(mailbox); 4336 4337 /* The qpc size of HIP08 is only 256B, which is half of HIP09 */ 4338 qpc_size = hr_dev->caps.qpc_sz; 4339 memcpy(mailbox->buf, context, qpc_size); 4340 memcpy(mailbox->buf + qpc_size, qpc_mask, qpc_size); 4341 4342 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, 4343 HNS_ROCE_CMD_MODIFY_QPC, hr_qp->qpn); 4344 4345 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 4346 4347 return ret; 4348 } 4349 4350 static void set_access_flags(struct hns_roce_qp *hr_qp, 4351 struct hns_roce_v2_qp_context *context, 4352 struct hns_roce_v2_qp_context *qpc_mask, 4353 const struct ib_qp_attr *attr, int attr_mask) 4354 { 4355 u8 dest_rd_atomic; 4356 u32 access_flags; 4357 4358 dest_rd_atomic = (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) ? 4359 attr->max_dest_rd_atomic : hr_qp->resp_depth; 4360 4361 access_flags = (attr_mask & IB_QP_ACCESS_FLAGS) ? 4362 attr->qp_access_flags : hr_qp->atomic_rd_en; 4363 4364 if (!dest_rd_atomic) 4365 access_flags &= IB_ACCESS_REMOTE_WRITE; 4366 4367 hr_reg_write_bool(context, QPC_RRE, 4368 access_flags & IB_ACCESS_REMOTE_READ); 4369 hr_reg_clear(qpc_mask, QPC_RRE); 4370 4371 hr_reg_write_bool(context, QPC_RWE, 4372 access_flags & IB_ACCESS_REMOTE_WRITE); 4373 hr_reg_clear(qpc_mask, QPC_RWE); 4374 4375 hr_reg_write_bool(context, QPC_ATE, 4376 access_flags & IB_ACCESS_REMOTE_ATOMIC); 4377 hr_reg_clear(qpc_mask, QPC_ATE); 4378 hr_reg_write_bool(context, QPC_EXT_ATE, 4379 access_flags & IB_ACCESS_REMOTE_ATOMIC); 4380 hr_reg_clear(qpc_mask, QPC_EXT_ATE); 4381 } 4382 4383 static void set_qpc_wqe_cnt(struct hns_roce_qp *hr_qp, 4384 struct hns_roce_v2_qp_context *context, 4385 struct hns_roce_v2_qp_context *qpc_mask) 4386 { 4387 hr_reg_write(context, QPC_SGE_SHIFT, 4388 to_hr_hem_entries_shift(hr_qp->sge.sge_cnt, 4389 hr_qp->sge.sge_shift)); 4390 4391 hr_reg_write(context, QPC_SQ_SHIFT, ilog2(hr_qp->sq.wqe_cnt)); 4392 4393 hr_reg_write(context, QPC_RQ_SHIFT, ilog2(hr_qp->rq.wqe_cnt)); 4394 } 4395 4396 static inline int get_cqn(struct ib_cq *ib_cq) 4397 { 4398 return ib_cq ? to_hr_cq(ib_cq)->cqn : 0; 4399 } 4400 4401 static inline int get_pdn(struct ib_pd *ib_pd) 4402 { 4403 return ib_pd ? to_hr_pd(ib_pd)->pdn : 0; 4404 } 4405 4406 static void modify_qp_reset_to_init(struct ib_qp *ibqp, 4407 const struct ib_qp_attr *attr, 4408 struct hns_roce_v2_qp_context *context, 4409 struct hns_roce_v2_qp_context *qpc_mask) 4410 { 4411 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 4412 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 4413 4414 /* 4415 * In v2 engine, software pass context and context mask to hardware 4416 * when modifying qp. If software need modify some fields in context, 4417 * we should set all bits of the relevant fields in context mask to 4418 * 0 at the same time, else set them to 0x1. 4419 */ 4420 hr_reg_write(context, QPC_TST, to_hr_qp_type(ibqp->qp_type)); 4421 4422 hr_reg_write(context, QPC_PD, get_pdn(ibqp->pd)); 4423 4424 hr_reg_write(context, QPC_RQWS, ilog2(hr_qp->rq.max_gs)); 4425 4426 set_qpc_wqe_cnt(hr_qp, context, qpc_mask); 4427 4428 /* No VLAN need to set 0xFFF */ 4429 hr_reg_write(context, QPC_VLAN_ID, 0xfff); 4430 4431 if (ibqp->qp_type == IB_QPT_XRC_TGT) { 4432 context->qkey_xrcd = cpu_to_le32(hr_qp->xrcdn); 4433 4434 hr_reg_enable(context, QPC_XRC_QP_TYPE); 4435 } 4436 4437 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) 4438 hr_reg_enable(context, QPC_RQ_RECORD_EN); 4439 4440 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_OWNER_DB) 4441 hr_reg_enable(context, QPC_OWNER_MODE); 4442 4443 hr_reg_write(context, QPC_RQ_DB_RECORD_ADDR_L, 4444 lower_32_bits(hr_qp->rdb.dma) >> 1); 4445 hr_reg_write(context, QPC_RQ_DB_RECORD_ADDR_H, 4446 upper_32_bits(hr_qp->rdb.dma)); 4447 4448 if (ibqp->qp_type != IB_QPT_UD && ibqp->qp_type != IB_QPT_GSI) 4449 hr_reg_write_bool(context, QPC_RQIE, 4450 hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE); 4451 4452 hr_reg_write(context, QPC_RX_CQN, get_cqn(ibqp->recv_cq)); 4453 4454 if (ibqp->srq) { 4455 hr_reg_enable(context, QPC_SRQ_EN); 4456 hr_reg_write(context, QPC_SRQN, to_hr_srq(ibqp->srq)->srqn); 4457 } 4458 4459 hr_reg_enable(context, QPC_FRE); 4460 4461 hr_reg_write(context, QPC_TX_CQN, get_cqn(ibqp->send_cq)); 4462 4463 if (hr_dev->caps.qpc_sz < HNS_ROCE_V3_QPC_SZ) 4464 return; 4465 4466 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_STASH) 4467 hr_reg_enable(&context->ext, QPCEX_STASH); 4468 } 4469 4470 static void modify_qp_init_to_init(struct ib_qp *ibqp, 4471 const struct ib_qp_attr *attr, 4472 struct hns_roce_v2_qp_context *context, 4473 struct hns_roce_v2_qp_context *qpc_mask) 4474 { 4475 /* 4476 * In v2 engine, software pass context and context mask to hardware 4477 * when modifying qp. If software need modify some fields in context, 4478 * we should set all bits of the relevant fields in context mask to 4479 * 0 at the same time, else set them to 0x1. 4480 */ 4481 hr_reg_write(context, QPC_TST, to_hr_qp_type(ibqp->qp_type)); 4482 hr_reg_clear(qpc_mask, QPC_TST); 4483 4484 hr_reg_write(context, QPC_PD, get_pdn(ibqp->pd)); 4485 hr_reg_clear(qpc_mask, QPC_PD); 4486 4487 hr_reg_write(context, QPC_RX_CQN, get_cqn(ibqp->recv_cq)); 4488 hr_reg_clear(qpc_mask, QPC_RX_CQN); 4489 4490 hr_reg_write(context, QPC_TX_CQN, get_cqn(ibqp->send_cq)); 4491 hr_reg_clear(qpc_mask, QPC_TX_CQN); 4492 4493 if (ibqp->srq) { 4494 hr_reg_enable(context, QPC_SRQ_EN); 4495 hr_reg_clear(qpc_mask, QPC_SRQ_EN); 4496 hr_reg_write(context, QPC_SRQN, to_hr_srq(ibqp->srq)->srqn); 4497 hr_reg_clear(qpc_mask, QPC_SRQN); 4498 } 4499 } 4500 4501 static int config_qp_rq_buf(struct hns_roce_dev *hr_dev, 4502 struct hns_roce_qp *hr_qp, 4503 struct hns_roce_v2_qp_context *context, 4504 struct hns_roce_v2_qp_context *qpc_mask) 4505 { 4506 u64 mtts[MTT_MIN_COUNT] = { 0 }; 4507 u64 wqe_sge_ba; 4508 int count; 4509 4510 /* Search qp buf's mtts */ 4511 count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->rq.offset, mtts, 4512 MTT_MIN_COUNT, &wqe_sge_ba); 4513 if (hr_qp->rq.wqe_cnt && count < 1) { 4514 ibdev_err(&hr_dev->ib_dev, 4515 "failed to find RQ WQE, QPN = 0x%lx.\n", hr_qp->qpn); 4516 return -EINVAL; 4517 } 4518 4519 context->wqe_sge_ba = cpu_to_le32(wqe_sge_ba >> 3); 4520 qpc_mask->wqe_sge_ba = 0; 4521 4522 /* 4523 * In v2 engine, software pass context and context mask to hardware 4524 * when modifying qp. If software need modify some fields in context, 4525 * we should set all bits of the relevant fields in context mask to 4526 * 0 at the same time, else set them to 0x1. 4527 */ 4528 hr_reg_write(context, QPC_WQE_SGE_BA_H, wqe_sge_ba >> (32 + 3)); 4529 hr_reg_clear(qpc_mask, QPC_WQE_SGE_BA_H); 4530 4531 hr_reg_write(context, QPC_SQ_HOP_NUM, 4532 to_hr_hem_hopnum(hr_dev->caps.wqe_sq_hop_num, 4533 hr_qp->sq.wqe_cnt)); 4534 hr_reg_clear(qpc_mask, QPC_SQ_HOP_NUM); 4535 4536 hr_reg_write(context, QPC_SGE_HOP_NUM, 4537 to_hr_hem_hopnum(hr_dev->caps.wqe_sge_hop_num, 4538 hr_qp->sge.sge_cnt)); 4539 hr_reg_clear(qpc_mask, QPC_SGE_HOP_NUM); 4540 4541 hr_reg_write(context, QPC_RQ_HOP_NUM, 4542 to_hr_hem_hopnum(hr_dev->caps.wqe_rq_hop_num, 4543 hr_qp->rq.wqe_cnt)); 4544 4545 hr_reg_clear(qpc_mask, QPC_RQ_HOP_NUM); 4546 4547 hr_reg_write(context, QPC_WQE_SGE_BA_PG_SZ, 4548 to_hr_hw_page_shift(hr_qp->mtr.hem_cfg.ba_pg_shift)); 4549 hr_reg_clear(qpc_mask, QPC_WQE_SGE_BA_PG_SZ); 4550 4551 hr_reg_write(context, QPC_WQE_SGE_BUF_PG_SZ, 4552 to_hr_hw_page_shift(hr_qp->mtr.hem_cfg.buf_pg_shift)); 4553 hr_reg_clear(qpc_mask, QPC_WQE_SGE_BUF_PG_SZ); 4554 4555 context->rq_cur_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[0])); 4556 qpc_mask->rq_cur_blk_addr = 0; 4557 4558 hr_reg_write(context, QPC_RQ_CUR_BLK_ADDR_H, 4559 upper_32_bits(to_hr_hw_page_addr(mtts[0]))); 4560 hr_reg_clear(qpc_mask, QPC_RQ_CUR_BLK_ADDR_H); 4561 4562 context->rq_nxt_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[1])); 4563 qpc_mask->rq_nxt_blk_addr = 0; 4564 4565 hr_reg_write(context, QPC_RQ_NXT_BLK_ADDR_H, 4566 upper_32_bits(to_hr_hw_page_addr(mtts[1]))); 4567 hr_reg_clear(qpc_mask, QPC_RQ_NXT_BLK_ADDR_H); 4568 4569 return 0; 4570 } 4571 4572 static int config_qp_sq_buf(struct hns_roce_dev *hr_dev, 4573 struct hns_roce_qp *hr_qp, 4574 struct hns_roce_v2_qp_context *context, 4575 struct hns_roce_v2_qp_context *qpc_mask) 4576 { 4577 struct ib_device *ibdev = &hr_dev->ib_dev; 4578 u64 sge_cur_blk = 0; 4579 u64 sq_cur_blk = 0; 4580 int count; 4581 4582 /* search qp buf's mtts */ 4583 count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, &sq_cur_blk, 1, NULL); 4584 if (count < 1) { 4585 ibdev_err(ibdev, "failed to find QP(0x%lx) SQ buf.\n", 4586 hr_qp->qpn); 4587 return -EINVAL; 4588 } 4589 if (hr_qp->sge.sge_cnt > 0) { 4590 count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 4591 hr_qp->sge.offset, 4592 &sge_cur_blk, 1, NULL); 4593 if (count < 1) { 4594 ibdev_err(ibdev, "failed to find QP(0x%lx) SGE buf.\n", 4595 hr_qp->qpn); 4596 return -EINVAL; 4597 } 4598 } 4599 4600 /* 4601 * In v2 engine, software pass context and context mask to hardware 4602 * when modifying qp. If software need modify some fields in context, 4603 * we should set all bits of the relevant fields in context mask to 4604 * 0 at the same time, else set them to 0x1. 4605 */ 4606 hr_reg_write(context, QPC_SQ_CUR_BLK_ADDR_L, 4607 lower_32_bits(to_hr_hw_page_addr(sq_cur_blk))); 4608 hr_reg_write(context, QPC_SQ_CUR_BLK_ADDR_H, 4609 upper_32_bits(to_hr_hw_page_addr(sq_cur_blk))); 4610 hr_reg_clear(qpc_mask, QPC_SQ_CUR_BLK_ADDR_L); 4611 hr_reg_clear(qpc_mask, QPC_SQ_CUR_BLK_ADDR_H); 4612 4613 hr_reg_write(context, QPC_SQ_CUR_SGE_BLK_ADDR_L, 4614 lower_32_bits(to_hr_hw_page_addr(sge_cur_blk))); 4615 hr_reg_write(context, QPC_SQ_CUR_SGE_BLK_ADDR_H, 4616 upper_32_bits(to_hr_hw_page_addr(sge_cur_blk))); 4617 hr_reg_clear(qpc_mask, QPC_SQ_CUR_SGE_BLK_ADDR_L); 4618 hr_reg_clear(qpc_mask, QPC_SQ_CUR_SGE_BLK_ADDR_H); 4619 4620 hr_reg_write(context, QPC_RX_SQ_CUR_BLK_ADDR_L, 4621 lower_32_bits(to_hr_hw_page_addr(sq_cur_blk))); 4622 hr_reg_write(context, QPC_RX_SQ_CUR_BLK_ADDR_H, 4623 upper_32_bits(to_hr_hw_page_addr(sq_cur_blk))); 4624 hr_reg_clear(qpc_mask, QPC_RX_SQ_CUR_BLK_ADDR_L); 4625 hr_reg_clear(qpc_mask, QPC_RX_SQ_CUR_BLK_ADDR_H); 4626 4627 return 0; 4628 } 4629 4630 static inline enum ib_mtu get_mtu(struct ib_qp *ibqp, 4631 const struct ib_qp_attr *attr) 4632 { 4633 if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_UD) 4634 return IB_MTU_4096; 4635 4636 return attr->path_mtu; 4637 } 4638 4639 static int modify_qp_init_to_rtr(struct ib_qp *ibqp, 4640 const struct ib_qp_attr *attr, int attr_mask, 4641 struct hns_roce_v2_qp_context *context, 4642 struct hns_roce_v2_qp_context *qpc_mask) 4643 { 4644 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 4645 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 4646 struct ib_device *ibdev = &hr_dev->ib_dev; 4647 dma_addr_t trrl_ba; 4648 dma_addr_t irrl_ba; 4649 enum ib_mtu ib_mtu; 4650 const u8 *smac; 4651 u8 lp_pktn_ini; 4652 u64 *mtts; 4653 u8 *dmac; 4654 u32 port; 4655 int mtu; 4656 int ret; 4657 4658 ret = config_qp_rq_buf(hr_dev, hr_qp, context, qpc_mask); 4659 if (ret) { 4660 ibdev_err(ibdev, "failed to config rq buf, ret = %d.\n", ret); 4661 return ret; 4662 } 4663 4664 /* Search IRRL's mtts */ 4665 mtts = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table, 4666 hr_qp->qpn, &irrl_ba); 4667 if (!mtts) { 4668 ibdev_err(ibdev, "failed to find qp irrl_table.\n"); 4669 return -EINVAL; 4670 } 4671 4672 /* Search TRRL's mtts */ 4673 mtts = hns_roce_table_find(hr_dev, &hr_dev->qp_table.trrl_table, 4674 hr_qp->qpn, &trrl_ba); 4675 if (!mtts) { 4676 ibdev_err(ibdev, "failed to find qp trrl_table.\n"); 4677 return -EINVAL; 4678 } 4679 4680 if (attr_mask & IB_QP_ALT_PATH) { 4681 ibdev_err(ibdev, "INIT2RTR attr_mask (0x%x) error.\n", 4682 attr_mask); 4683 return -EINVAL; 4684 } 4685 4686 hr_reg_write(context, QPC_TRRL_BA_L, trrl_ba >> 4); 4687 hr_reg_clear(qpc_mask, QPC_TRRL_BA_L); 4688 context->trrl_ba = cpu_to_le32(trrl_ba >> (16 + 4)); 4689 qpc_mask->trrl_ba = 0; 4690 hr_reg_write(context, QPC_TRRL_BA_H, trrl_ba >> (32 + 16 + 4)); 4691 hr_reg_clear(qpc_mask, QPC_TRRL_BA_H); 4692 4693 context->irrl_ba = cpu_to_le32(irrl_ba >> 6); 4694 qpc_mask->irrl_ba = 0; 4695 hr_reg_write(context, QPC_IRRL_BA_H, irrl_ba >> (32 + 6)); 4696 hr_reg_clear(qpc_mask, QPC_IRRL_BA_H); 4697 4698 hr_reg_enable(context, QPC_RMT_E2E); 4699 hr_reg_clear(qpc_mask, QPC_RMT_E2E); 4700 4701 hr_reg_write(context, QPC_SIG_TYPE, hr_qp->sq_signal_bits); 4702 hr_reg_clear(qpc_mask, QPC_SIG_TYPE); 4703 4704 port = (attr_mask & IB_QP_PORT) ? (attr->port_num - 1) : hr_qp->port; 4705 4706 smac = (const u8 *)hr_dev->dev_addr[port]; 4707 dmac = (u8 *)attr->ah_attr.roce.dmac; 4708 /* when dmac equals smac or loop_idc is 1, it should loopback */ 4709 if (ether_addr_equal_unaligned(dmac, smac) || 4710 hr_dev->loop_idc == 0x1) { 4711 hr_reg_write(context, QPC_LBI, hr_dev->loop_idc); 4712 hr_reg_clear(qpc_mask, QPC_LBI); 4713 } 4714 4715 if (attr_mask & IB_QP_DEST_QPN) { 4716 hr_reg_write(context, QPC_DQPN, attr->dest_qp_num); 4717 hr_reg_clear(qpc_mask, QPC_DQPN); 4718 } 4719 4720 memcpy(&context->dmac, dmac, sizeof(u32)); 4721 hr_reg_write(context, QPC_DMAC_H, *((u16 *)(&dmac[4]))); 4722 qpc_mask->dmac = 0; 4723 hr_reg_clear(qpc_mask, QPC_DMAC_H); 4724 4725 ib_mtu = get_mtu(ibqp, attr); 4726 hr_qp->path_mtu = ib_mtu; 4727 4728 mtu = ib_mtu_enum_to_int(ib_mtu); 4729 if (WARN_ON(mtu <= 0)) 4730 return -EINVAL; 4731 #define MAX_LP_MSG_LEN 16384 4732 /* MTU * (2 ^ LP_PKTN_INI) shouldn't be bigger than 16KB */ 4733 lp_pktn_ini = ilog2(MAX_LP_MSG_LEN / mtu); 4734 if (WARN_ON(lp_pktn_ini >= 0xF)) 4735 return -EINVAL; 4736 4737 if (attr_mask & IB_QP_PATH_MTU) { 4738 hr_reg_write(context, QPC_MTU, ib_mtu); 4739 hr_reg_clear(qpc_mask, QPC_MTU); 4740 } 4741 4742 hr_reg_write(context, QPC_LP_PKTN_INI, lp_pktn_ini); 4743 hr_reg_clear(qpc_mask, QPC_LP_PKTN_INI); 4744 4745 /* ACK_REQ_FREQ should be larger than or equal to LP_PKTN_INI */ 4746 hr_reg_write(context, QPC_ACK_REQ_FREQ, lp_pktn_ini); 4747 hr_reg_clear(qpc_mask, QPC_ACK_REQ_FREQ); 4748 4749 hr_reg_clear(qpc_mask, QPC_RX_REQ_PSN_ERR); 4750 hr_reg_clear(qpc_mask, QPC_RX_REQ_MSN); 4751 hr_reg_clear(qpc_mask, QPC_RX_REQ_LAST_OPTYPE); 4752 4753 context->rq_rnr_timer = 0; 4754 qpc_mask->rq_rnr_timer = 0; 4755 4756 hr_reg_clear(qpc_mask, QPC_TRRL_HEAD_MAX); 4757 hr_reg_clear(qpc_mask, QPC_TRRL_TAIL_MAX); 4758 4759 /* rocee send 2^lp_sgen_ini segs every time */ 4760 hr_reg_write(context, QPC_LP_SGEN_INI, 3); 4761 hr_reg_clear(qpc_mask, QPC_LP_SGEN_INI); 4762 4763 return 0; 4764 } 4765 4766 static int modify_qp_rtr_to_rts(struct ib_qp *ibqp, 4767 const struct ib_qp_attr *attr, int attr_mask, 4768 struct hns_roce_v2_qp_context *context, 4769 struct hns_roce_v2_qp_context *qpc_mask) 4770 { 4771 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 4772 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 4773 struct ib_device *ibdev = &hr_dev->ib_dev; 4774 int ret; 4775 4776 /* Not support alternate path and path migration */ 4777 if (attr_mask & (IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE)) { 4778 ibdev_err(ibdev, "RTR2RTS attr_mask (0x%x)error\n", attr_mask); 4779 return -EINVAL; 4780 } 4781 4782 ret = config_qp_sq_buf(hr_dev, hr_qp, context, qpc_mask); 4783 if (ret) { 4784 ibdev_err(ibdev, "failed to config sq buf, ret = %d.\n", ret); 4785 return ret; 4786 } 4787 4788 /* 4789 * Set some fields in context to zero, Because the default values 4790 * of all fields in context are zero, we need not set them to 0 again. 4791 * but we should set the relevant fields of context mask to 0. 4792 */ 4793 hr_reg_clear(qpc_mask, QPC_IRRL_SGE_IDX); 4794 4795 hr_reg_clear(qpc_mask, QPC_RX_ACK_MSN); 4796 4797 hr_reg_clear(qpc_mask, QPC_ACK_LAST_OPTYPE); 4798 hr_reg_clear(qpc_mask, QPC_IRRL_PSN_VLD); 4799 hr_reg_clear(qpc_mask, QPC_IRRL_PSN); 4800 4801 hr_reg_clear(qpc_mask, QPC_IRRL_TAIL_REAL); 4802 4803 hr_reg_clear(qpc_mask, QPC_RETRY_MSG_MSN); 4804 4805 hr_reg_clear(qpc_mask, QPC_RNR_RETRY_FLAG); 4806 4807 hr_reg_clear(qpc_mask, QPC_CHECK_FLG); 4808 4809 hr_reg_clear(qpc_mask, QPC_V2_IRRL_HEAD); 4810 4811 return 0; 4812 } 4813 4814 static int get_dip_ctx_idx(struct ib_qp *ibqp, const struct ib_qp_attr *attr, 4815 u32 *dip_idx) 4816 { 4817 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 4818 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 4819 u32 *spare_idx = hr_dev->qp_table.idx_table.spare_idx; 4820 u32 *head = &hr_dev->qp_table.idx_table.head; 4821 u32 *tail = &hr_dev->qp_table.idx_table.tail; 4822 struct hns_roce_dip *hr_dip; 4823 unsigned long flags; 4824 int ret = 0; 4825 4826 spin_lock_irqsave(&hr_dev->dip_list_lock, flags); 4827 4828 spare_idx[*tail] = ibqp->qp_num; 4829 *tail = (*tail == hr_dev->caps.num_qps - 1) ? 0 : (*tail + 1); 4830 4831 list_for_each_entry(hr_dip, &hr_dev->dip_list, node) { 4832 if (!memcmp(grh->dgid.raw, hr_dip->dgid, 16)) { 4833 *dip_idx = hr_dip->dip_idx; 4834 goto out; 4835 } 4836 } 4837 4838 /* If no dgid is found, a new dip and a mapping between dgid and 4839 * dip_idx will be created. 4840 */ 4841 hr_dip = kzalloc(sizeof(*hr_dip), GFP_ATOMIC); 4842 if (!hr_dip) { 4843 ret = -ENOMEM; 4844 goto out; 4845 } 4846 4847 memcpy(hr_dip->dgid, grh->dgid.raw, sizeof(grh->dgid.raw)); 4848 hr_dip->dip_idx = *dip_idx = spare_idx[*head]; 4849 *head = (*head == hr_dev->caps.num_qps - 1) ? 0 : (*head + 1); 4850 list_add_tail(&hr_dip->node, &hr_dev->dip_list); 4851 4852 out: 4853 spin_unlock_irqrestore(&hr_dev->dip_list_lock, flags); 4854 return ret; 4855 } 4856 4857 enum { 4858 CONG_DCQCN, 4859 CONG_WINDOW, 4860 }; 4861 4862 enum { 4863 UNSUPPORT_CONG_LEVEL, 4864 SUPPORT_CONG_LEVEL, 4865 }; 4866 4867 enum { 4868 CONG_LDCP, 4869 CONG_HC3, 4870 }; 4871 4872 enum { 4873 DIP_INVALID, 4874 DIP_VALID, 4875 }; 4876 4877 enum { 4878 WND_LIMIT, 4879 WND_UNLIMIT, 4880 }; 4881 4882 static int check_cong_type(struct ib_qp *ibqp, 4883 struct hns_roce_congestion_algorithm *cong_alg) 4884 { 4885 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 4886 4887 /* different congestion types match different configurations */ 4888 switch (hr_dev->caps.cong_type) { 4889 case CONG_TYPE_DCQCN: 4890 cong_alg->alg_sel = CONG_DCQCN; 4891 cong_alg->alg_sub_sel = UNSUPPORT_CONG_LEVEL; 4892 cong_alg->dip_vld = DIP_INVALID; 4893 cong_alg->wnd_mode_sel = WND_LIMIT; 4894 break; 4895 case CONG_TYPE_LDCP: 4896 cong_alg->alg_sel = CONG_WINDOW; 4897 cong_alg->alg_sub_sel = CONG_LDCP; 4898 cong_alg->dip_vld = DIP_INVALID; 4899 cong_alg->wnd_mode_sel = WND_UNLIMIT; 4900 break; 4901 case CONG_TYPE_HC3: 4902 cong_alg->alg_sel = CONG_WINDOW; 4903 cong_alg->alg_sub_sel = CONG_HC3; 4904 cong_alg->dip_vld = DIP_INVALID; 4905 cong_alg->wnd_mode_sel = WND_LIMIT; 4906 break; 4907 case CONG_TYPE_DIP: 4908 cong_alg->alg_sel = CONG_DCQCN; 4909 cong_alg->alg_sub_sel = UNSUPPORT_CONG_LEVEL; 4910 cong_alg->dip_vld = DIP_VALID; 4911 cong_alg->wnd_mode_sel = WND_LIMIT; 4912 break; 4913 default: 4914 ibdev_err(&hr_dev->ib_dev, 4915 "error type(%u) for congestion selection.\n", 4916 hr_dev->caps.cong_type); 4917 return -EINVAL; 4918 } 4919 4920 return 0; 4921 } 4922 4923 static int fill_cong_field(struct ib_qp *ibqp, const struct ib_qp_attr *attr, 4924 struct hns_roce_v2_qp_context *context, 4925 struct hns_roce_v2_qp_context *qpc_mask) 4926 { 4927 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 4928 struct hns_roce_congestion_algorithm cong_field; 4929 struct ib_device *ibdev = ibqp->device; 4930 struct hns_roce_dev *hr_dev = to_hr_dev(ibdev); 4931 u32 dip_idx = 0; 4932 int ret; 4933 4934 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 || 4935 grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE) 4936 return 0; 4937 4938 ret = check_cong_type(ibqp, &cong_field); 4939 if (ret) 4940 return ret; 4941 4942 hr_reg_write(context, QPC_CONG_ALGO_TMPL_ID, hr_dev->cong_algo_tmpl_id + 4943 hr_dev->caps.cong_type * HNS_ROCE_CONG_SIZE); 4944 hr_reg_clear(qpc_mask, QPC_CONG_ALGO_TMPL_ID); 4945 hr_reg_write(&context->ext, QPCEX_CONG_ALG_SEL, cong_field.alg_sel); 4946 hr_reg_clear(&qpc_mask->ext, QPCEX_CONG_ALG_SEL); 4947 hr_reg_write(&context->ext, QPCEX_CONG_ALG_SUB_SEL, 4948 cong_field.alg_sub_sel); 4949 hr_reg_clear(&qpc_mask->ext, QPCEX_CONG_ALG_SUB_SEL); 4950 hr_reg_write(&context->ext, QPCEX_DIP_CTX_IDX_VLD, cong_field.dip_vld); 4951 hr_reg_clear(&qpc_mask->ext, QPCEX_DIP_CTX_IDX_VLD); 4952 hr_reg_write(&context->ext, QPCEX_SQ_RQ_NOT_FORBID_EN, 4953 cong_field.wnd_mode_sel); 4954 hr_reg_clear(&qpc_mask->ext, QPCEX_SQ_RQ_NOT_FORBID_EN); 4955 4956 /* if dip is disabled, there is no need to set dip idx */ 4957 if (cong_field.dip_vld == 0) 4958 return 0; 4959 4960 ret = get_dip_ctx_idx(ibqp, attr, &dip_idx); 4961 if (ret) { 4962 ibdev_err(ibdev, "failed to fill cong field, ret = %d.\n", ret); 4963 return ret; 4964 } 4965 4966 hr_reg_write(&context->ext, QPCEX_DIP_CTX_IDX, dip_idx); 4967 hr_reg_write(&qpc_mask->ext, QPCEX_DIP_CTX_IDX, 0); 4968 4969 return 0; 4970 } 4971 4972 static int hns_roce_v2_set_path(struct ib_qp *ibqp, 4973 const struct ib_qp_attr *attr, 4974 int attr_mask, 4975 struct hns_roce_v2_qp_context *context, 4976 struct hns_roce_v2_qp_context *qpc_mask) 4977 { 4978 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 4979 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 4980 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 4981 struct ib_device *ibdev = &hr_dev->ib_dev; 4982 const struct ib_gid_attr *gid_attr = NULL; 4983 int is_roce_protocol; 4984 u16 vlan_id = 0xffff; 4985 bool is_udp = false; 4986 u8 ib_port; 4987 u8 hr_port; 4988 int ret; 4989 4990 /* 4991 * If free_mr_en of qp is set, it means that this qp comes from 4992 * free mr. This qp will perform the loopback operation. 4993 * In the loopback scenario, only sl needs to be set. 4994 */ 4995 if (hr_qp->free_mr_en) { 4996 hr_reg_write(context, QPC_SL, rdma_ah_get_sl(&attr->ah_attr)); 4997 hr_reg_clear(qpc_mask, QPC_SL); 4998 hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr); 4999 return 0; 5000 } 5001 5002 ib_port = (attr_mask & IB_QP_PORT) ? attr->port_num : hr_qp->port + 1; 5003 hr_port = ib_port - 1; 5004 is_roce_protocol = rdma_cap_eth_ah(&hr_dev->ib_dev, ib_port) && 5005 rdma_ah_get_ah_flags(&attr->ah_attr) & IB_AH_GRH; 5006 5007 if (is_roce_protocol) { 5008 gid_attr = attr->ah_attr.grh.sgid_attr; 5009 ret = rdma_read_gid_l2_fields(gid_attr, &vlan_id, NULL); 5010 if (ret) 5011 return ret; 5012 5013 is_udp = (gid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP); 5014 } 5015 5016 /* Only HIP08 needs to set the vlan_en bits in QPC */ 5017 if (vlan_id < VLAN_N_VID && 5018 hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) { 5019 hr_reg_enable(context, QPC_RQ_VLAN_EN); 5020 hr_reg_clear(qpc_mask, QPC_RQ_VLAN_EN); 5021 hr_reg_enable(context, QPC_SQ_VLAN_EN); 5022 hr_reg_clear(qpc_mask, QPC_SQ_VLAN_EN); 5023 } 5024 5025 hr_reg_write(context, QPC_VLAN_ID, vlan_id); 5026 hr_reg_clear(qpc_mask, QPC_VLAN_ID); 5027 5028 if (grh->sgid_index >= hr_dev->caps.gid_table_len[hr_port]) { 5029 ibdev_err(ibdev, "sgid_index(%u) too large. max is %d\n", 5030 grh->sgid_index, hr_dev->caps.gid_table_len[hr_port]); 5031 return -EINVAL; 5032 } 5033 5034 if (attr->ah_attr.type != RDMA_AH_ATTR_TYPE_ROCE) { 5035 ibdev_err(ibdev, "ah attr is not RDMA roce type\n"); 5036 return -EINVAL; 5037 } 5038 5039 hr_reg_write(context, QPC_UDPSPN, 5040 is_udp ? rdma_get_udp_sport(grh->flow_label, ibqp->qp_num, 5041 attr->dest_qp_num) : 5042 0); 5043 5044 hr_reg_clear(qpc_mask, QPC_UDPSPN); 5045 5046 hr_reg_write(context, QPC_GMV_IDX, grh->sgid_index); 5047 5048 hr_reg_clear(qpc_mask, QPC_GMV_IDX); 5049 5050 hr_reg_write(context, QPC_HOPLIMIT, grh->hop_limit); 5051 hr_reg_clear(qpc_mask, QPC_HOPLIMIT); 5052 5053 ret = fill_cong_field(ibqp, attr, context, qpc_mask); 5054 if (ret) 5055 return ret; 5056 5057 hr_reg_write(context, QPC_TC, get_tclass(&attr->ah_attr.grh)); 5058 hr_reg_clear(qpc_mask, QPC_TC); 5059 5060 hr_reg_write(context, QPC_FL, grh->flow_label); 5061 hr_reg_clear(qpc_mask, QPC_FL); 5062 memcpy(context->dgid, grh->dgid.raw, sizeof(grh->dgid.raw)); 5063 memset(qpc_mask->dgid, 0, sizeof(grh->dgid.raw)); 5064 5065 hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr); 5066 if (unlikely(hr_qp->sl > MAX_SERVICE_LEVEL)) { 5067 ibdev_err(ibdev, 5068 "failed to fill QPC, sl (%u) shouldn't be larger than %d.\n", 5069 hr_qp->sl, MAX_SERVICE_LEVEL); 5070 return -EINVAL; 5071 } 5072 5073 hr_reg_write(context, QPC_SL, hr_qp->sl); 5074 hr_reg_clear(qpc_mask, QPC_SL); 5075 5076 return 0; 5077 } 5078 5079 static bool check_qp_state(enum ib_qp_state cur_state, 5080 enum ib_qp_state new_state) 5081 { 5082 static const bool sm[][IB_QPS_ERR + 1] = { 5083 [IB_QPS_RESET] = { [IB_QPS_RESET] = true, 5084 [IB_QPS_INIT] = true }, 5085 [IB_QPS_INIT] = { [IB_QPS_RESET] = true, 5086 [IB_QPS_INIT] = true, 5087 [IB_QPS_RTR] = true, 5088 [IB_QPS_ERR] = true }, 5089 [IB_QPS_RTR] = { [IB_QPS_RESET] = true, 5090 [IB_QPS_RTS] = true, 5091 [IB_QPS_ERR] = true }, 5092 [IB_QPS_RTS] = { [IB_QPS_RESET] = true, 5093 [IB_QPS_RTS] = true, 5094 [IB_QPS_ERR] = true }, 5095 [IB_QPS_SQD] = {}, 5096 [IB_QPS_SQE] = {}, 5097 [IB_QPS_ERR] = { [IB_QPS_RESET] = true, 5098 [IB_QPS_ERR] = true } 5099 }; 5100 5101 return sm[cur_state][new_state]; 5102 } 5103 5104 static int hns_roce_v2_set_abs_fields(struct ib_qp *ibqp, 5105 const struct ib_qp_attr *attr, 5106 int attr_mask, 5107 enum ib_qp_state cur_state, 5108 enum ib_qp_state new_state, 5109 struct hns_roce_v2_qp_context *context, 5110 struct hns_roce_v2_qp_context *qpc_mask) 5111 { 5112 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 5113 int ret = 0; 5114 5115 if (!check_qp_state(cur_state, new_state)) { 5116 ibdev_err(&hr_dev->ib_dev, "Illegal state for QP!\n"); 5117 return -EINVAL; 5118 } 5119 5120 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { 5121 memset(qpc_mask, 0, hr_dev->caps.qpc_sz); 5122 modify_qp_reset_to_init(ibqp, attr, context, qpc_mask); 5123 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) { 5124 modify_qp_init_to_init(ibqp, attr, context, qpc_mask); 5125 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) { 5126 ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context, 5127 qpc_mask); 5128 } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) { 5129 ret = modify_qp_rtr_to_rts(ibqp, attr, attr_mask, context, 5130 qpc_mask); 5131 } 5132 5133 return ret; 5134 } 5135 5136 static bool check_qp_timeout_cfg_range(struct hns_roce_dev *hr_dev, u8 *timeout) 5137 { 5138 #define QP_ACK_TIMEOUT_MAX_HIP08 20 5139 #define QP_ACK_TIMEOUT_OFFSET 10 5140 #define QP_ACK_TIMEOUT_MAX 31 5141 5142 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) { 5143 if (*timeout > QP_ACK_TIMEOUT_MAX_HIP08) { 5144 ibdev_warn(&hr_dev->ib_dev, 5145 "local ACK timeout shall be 0 to 20.\n"); 5146 return false; 5147 } 5148 *timeout += QP_ACK_TIMEOUT_OFFSET; 5149 } else if (hr_dev->pci_dev->revision > PCI_REVISION_ID_HIP08) { 5150 if (*timeout > QP_ACK_TIMEOUT_MAX) { 5151 ibdev_warn(&hr_dev->ib_dev, 5152 "local ACK timeout shall be 0 to 31.\n"); 5153 return false; 5154 } 5155 } 5156 5157 return true; 5158 } 5159 5160 static int hns_roce_v2_set_opt_fields(struct ib_qp *ibqp, 5161 const struct ib_qp_attr *attr, 5162 int attr_mask, 5163 struct hns_roce_v2_qp_context *context, 5164 struct hns_roce_v2_qp_context *qpc_mask) 5165 { 5166 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 5167 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 5168 int ret = 0; 5169 u8 timeout; 5170 5171 if (attr_mask & IB_QP_AV) { 5172 ret = hns_roce_v2_set_path(ibqp, attr, attr_mask, context, 5173 qpc_mask); 5174 if (ret) 5175 return ret; 5176 } 5177 5178 if (attr_mask & IB_QP_TIMEOUT) { 5179 timeout = attr->timeout; 5180 if (check_qp_timeout_cfg_range(hr_dev, &timeout)) { 5181 hr_reg_write(context, QPC_AT, timeout); 5182 hr_reg_clear(qpc_mask, QPC_AT); 5183 } 5184 } 5185 5186 if (attr_mask & IB_QP_RETRY_CNT) { 5187 hr_reg_write(context, QPC_RETRY_NUM_INIT, attr->retry_cnt); 5188 hr_reg_clear(qpc_mask, QPC_RETRY_NUM_INIT); 5189 5190 hr_reg_write(context, QPC_RETRY_CNT, attr->retry_cnt); 5191 hr_reg_clear(qpc_mask, QPC_RETRY_CNT); 5192 } 5193 5194 if (attr_mask & IB_QP_RNR_RETRY) { 5195 hr_reg_write(context, QPC_RNR_NUM_INIT, attr->rnr_retry); 5196 hr_reg_clear(qpc_mask, QPC_RNR_NUM_INIT); 5197 5198 hr_reg_write(context, QPC_RNR_CNT, attr->rnr_retry); 5199 hr_reg_clear(qpc_mask, QPC_RNR_CNT); 5200 } 5201 5202 if (attr_mask & IB_QP_SQ_PSN) { 5203 hr_reg_write(context, QPC_SQ_CUR_PSN, attr->sq_psn); 5204 hr_reg_clear(qpc_mask, QPC_SQ_CUR_PSN); 5205 5206 hr_reg_write(context, QPC_SQ_MAX_PSN, attr->sq_psn); 5207 hr_reg_clear(qpc_mask, QPC_SQ_MAX_PSN); 5208 5209 hr_reg_write(context, QPC_RETRY_MSG_PSN_L, attr->sq_psn); 5210 hr_reg_clear(qpc_mask, QPC_RETRY_MSG_PSN_L); 5211 5212 hr_reg_write(context, QPC_RETRY_MSG_PSN_H, 5213 attr->sq_psn >> RETRY_MSG_PSN_SHIFT); 5214 hr_reg_clear(qpc_mask, QPC_RETRY_MSG_PSN_H); 5215 5216 hr_reg_write(context, QPC_RETRY_MSG_FPKT_PSN, attr->sq_psn); 5217 hr_reg_clear(qpc_mask, QPC_RETRY_MSG_FPKT_PSN); 5218 5219 hr_reg_write(context, QPC_RX_ACK_EPSN, attr->sq_psn); 5220 hr_reg_clear(qpc_mask, QPC_RX_ACK_EPSN); 5221 } 5222 5223 if ((attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) && 5224 attr->max_dest_rd_atomic) { 5225 hr_reg_write(context, QPC_RR_MAX, 5226 fls(attr->max_dest_rd_atomic - 1)); 5227 hr_reg_clear(qpc_mask, QPC_RR_MAX); 5228 } 5229 5230 if ((attr_mask & IB_QP_MAX_QP_RD_ATOMIC) && attr->max_rd_atomic) { 5231 hr_reg_write(context, QPC_SR_MAX, fls(attr->max_rd_atomic - 1)); 5232 hr_reg_clear(qpc_mask, QPC_SR_MAX); 5233 } 5234 5235 if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) 5236 set_access_flags(hr_qp, context, qpc_mask, attr, attr_mask); 5237 5238 if (attr_mask & IB_QP_MIN_RNR_TIMER) { 5239 hr_reg_write(context, QPC_MIN_RNR_TIME, 5240 hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 ? 5241 HNS_ROCE_RNR_TIMER_10NS : attr->min_rnr_timer); 5242 hr_reg_clear(qpc_mask, QPC_MIN_RNR_TIME); 5243 } 5244 5245 if (attr_mask & IB_QP_RQ_PSN) { 5246 hr_reg_write(context, QPC_RX_REQ_EPSN, attr->rq_psn); 5247 hr_reg_clear(qpc_mask, QPC_RX_REQ_EPSN); 5248 5249 hr_reg_write(context, QPC_RAQ_PSN, attr->rq_psn - 1); 5250 hr_reg_clear(qpc_mask, QPC_RAQ_PSN); 5251 } 5252 5253 if (attr_mask & IB_QP_QKEY) { 5254 context->qkey_xrcd = cpu_to_le32(attr->qkey); 5255 qpc_mask->qkey_xrcd = 0; 5256 hr_qp->qkey = attr->qkey; 5257 } 5258 5259 return ret; 5260 } 5261 5262 static void hns_roce_v2_record_opt_fields(struct ib_qp *ibqp, 5263 const struct ib_qp_attr *attr, 5264 int attr_mask) 5265 { 5266 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 5267 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 5268 5269 if (attr_mask & IB_QP_ACCESS_FLAGS) 5270 hr_qp->atomic_rd_en = attr->qp_access_flags; 5271 5272 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) 5273 hr_qp->resp_depth = attr->max_dest_rd_atomic; 5274 if (attr_mask & IB_QP_PORT) { 5275 hr_qp->port = attr->port_num - 1; 5276 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; 5277 } 5278 } 5279 5280 static void clear_qp(struct hns_roce_qp *hr_qp) 5281 { 5282 struct ib_qp *ibqp = &hr_qp->ibqp; 5283 5284 if (ibqp->send_cq) 5285 hns_roce_v2_cq_clean(to_hr_cq(ibqp->send_cq), 5286 hr_qp->qpn, NULL); 5287 5288 if (ibqp->recv_cq && ibqp->recv_cq != ibqp->send_cq) 5289 hns_roce_v2_cq_clean(to_hr_cq(ibqp->recv_cq), 5290 hr_qp->qpn, ibqp->srq ? 5291 to_hr_srq(ibqp->srq) : NULL); 5292 5293 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) 5294 *hr_qp->rdb.db_record = 0; 5295 5296 hr_qp->rq.head = 0; 5297 hr_qp->rq.tail = 0; 5298 hr_qp->sq.head = 0; 5299 hr_qp->sq.tail = 0; 5300 hr_qp->next_sge = 0; 5301 } 5302 5303 static void v2_set_flushed_fields(struct ib_qp *ibqp, 5304 struct hns_roce_v2_qp_context *context, 5305 struct hns_roce_v2_qp_context *qpc_mask) 5306 { 5307 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 5308 unsigned long sq_flag = 0; 5309 unsigned long rq_flag = 0; 5310 5311 if (ibqp->qp_type == IB_QPT_XRC_TGT) 5312 return; 5313 5314 spin_lock_irqsave(&hr_qp->sq.lock, sq_flag); 5315 hr_reg_write(context, QPC_SQ_PRODUCER_IDX, hr_qp->sq.head); 5316 hr_reg_clear(qpc_mask, QPC_SQ_PRODUCER_IDX); 5317 hr_qp->state = IB_QPS_ERR; 5318 spin_unlock_irqrestore(&hr_qp->sq.lock, sq_flag); 5319 5320 if (ibqp->srq || ibqp->qp_type == IB_QPT_XRC_INI) /* no RQ */ 5321 return; 5322 5323 spin_lock_irqsave(&hr_qp->rq.lock, rq_flag); 5324 hr_reg_write(context, QPC_RQ_PRODUCER_IDX, hr_qp->rq.head); 5325 hr_reg_clear(qpc_mask, QPC_RQ_PRODUCER_IDX); 5326 spin_unlock_irqrestore(&hr_qp->rq.lock, rq_flag); 5327 } 5328 5329 static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, 5330 const struct ib_qp_attr *attr, 5331 int attr_mask, enum ib_qp_state cur_state, 5332 enum ib_qp_state new_state) 5333 { 5334 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 5335 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 5336 struct hns_roce_v2_qp_context ctx[2]; 5337 struct hns_roce_v2_qp_context *context = ctx; 5338 struct hns_roce_v2_qp_context *qpc_mask = ctx + 1; 5339 struct ib_device *ibdev = &hr_dev->ib_dev; 5340 int ret; 5341 5342 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 5343 return -EOPNOTSUPP; 5344 5345 /* 5346 * In v2 engine, software pass context and context mask to hardware 5347 * when modifying qp. If software need modify some fields in context, 5348 * we should set all bits of the relevant fields in context mask to 5349 * 0 at the same time, else set them to 0x1. 5350 */ 5351 memset(context, 0, hr_dev->caps.qpc_sz); 5352 memset(qpc_mask, 0xff, hr_dev->caps.qpc_sz); 5353 5354 ret = hns_roce_v2_set_abs_fields(ibqp, attr, attr_mask, cur_state, 5355 new_state, context, qpc_mask); 5356 if (ret) 5357 goto out; 5358 5359 /* When QP state is err, SQ and RQ WQE should be flushed */ 5360 if (new_state == IB_QPS_ERR) 5361 v2_set_flushed_fields(ibqp, context, qpc_mask); 5362 5363 /* Configure the optional fields */ 5364 ret = hns_roce_v2_set_opt_fields(ibqp, attr, attr_mask, context, 5365 qpc_mask); 5366 if (ret) 5367 goto out; 5368 5369 hr_reg_write_bool(context, QPC_INV_CREDIT, 5370 to_hr_qp_type(hr_qp->ibqp.qp_type) == SERV_TYPE_XRC || 5371 ibqp->srq); 5372 hr_reg_clear(qpc_mask, QPC_INV_CREDIT); 5373 5374 /* Every status migrate must change state */ 5375 hr_reg_write(context, QPC_QP_ST, new_state); 5376 hr_reg_clear(qpc_mask, QPC_QP_ST); 5377 5378 /* SW pass context to HW */ 5379 ret = hns_roce_v2_qp_modify(hr_dev, context, qpc_mask, hr_qp); 5380 if (ret) { 5381 ibdev_err(ibdev, "failed to modify QP, ret = %d.\n", ret); 5382 goto out; 5383 } 5384 5385 hr_qp->state = new_state; 5386 5387 hns_roce_v2_record_opt_fields(ibqp, attr, attr_mask); 5388 5389 if (new_state == IB_QPS_RESET && !ibqp->uobject) 5390 clear_qp(hr_qp); 5391 5392 out: 5393 return ret; 5394 } 5395 5396 static int to_ib_qp_st(enum hns_roce_v2_qp_state state) 5397 { 5398 static const enum ib_qp_state map[] = { 5399 [HNS_ROCE_QP_ST_RST] = IB_QPS_RESET, 5400 [HNS_ROCE_QP_ST_INIT] = IB_QPS_INIT, 5401 [HNS_ROCE_QP_ST_RTR] = IB_QPS_RTR, 5402 [HNS_ROCE_QP_ST_RTS] = IB_QPS_RTS, 5403 [HNS_ROCE_QP_ST_SQD] = IB_QPS_SQD, 5404 [HNS_ROCE_QP_ST_SQER] = IB_QPS_SQE, 5405 [HNS_ROCE_QP_ST_ERR] = IB_QPS_ERR, 5406 [HNS_ROCE_QP_ST_SQ_DRAINING] = IB_QPS_SQD 5407 }; 5408 5409 return (state < ARRAY_SIZE(map)) ? map[state] : -1; 5410 } 5411 5412 static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev, u32 qpn, 5413 void *buffer) 5414 { 5415 struct hns_roce_cmd_mailbox *mailbox; 5416 int ret; 5417 5418 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 5419 if (IS_ERR(mailbox)) 5420 return PTR_ERR(mailbox); 5421 5422 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_QPC, 5423 qpn); 5424 if (ret) 5425 goto out; 5426 5427 memcpy(buffer, mailbox->buf, hr_dev->caps.qpc_sz); 5428 5429 out: 5430 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 5431 return ret; 5432 } 5433 5434 static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, 5435 int qp_attr_mask, 5436 struct ib_qp_init_attr *qp_init_attr) 5437 { 5438 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 5439 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 5440 struct hns_roce_v2_qp_context context = {}; 5441 struct ib_device *ibdev = &hr_dev->ib_dev; 5442 int tmp_qp_state; 5443 int state; 5444 int ret; 5445 5446 memset(qp_attr, 0, sizeof(*qp_attr)); 5447 memset(qp_init_attr, 0, sizeof(*qp_init_attr)); 5448 5449 mutex_lock(&hr_qp->mutex); 5450 5451 if (hr_qp->state == IB_QPS_RESET) { 5452 qp_attr->qp_state = IB_QPS_RESET; 5453 ret = 0; 5454 goto done; 5455 } 5456 5457 ret = hns_roce_v2_query_qpc(hr_dev, hr_qp->qpn, &context); 5458 if (ret) { 5459 ibdev_err(ibdev, "failed to query QPC, ret = %d.\n", ret); 5460 ret = -EINVAL; 5461 goto out; 5462 } 5463 5464 state = hr_reg_read(&context, QPC_QP_ST); 5465 tmp_qp_state = to_ib_qp_st((enum hns_roce_v2_qp_state)state); 5466 if (tmp_qp_state == -1) { 5467 ibdev_err(ibdev, "Illegal ib_qp_state\n"); 5468 ret = -EINVAL; 5469 goto out; 5470 } 5471 hr_qp->state = (u8)tmp_qp_state; 5472 qp_attr->qp_state = (enum ib_qp_state)hr_qp->state; 5473 qp_attr->path_mtu = (enum ib_mtu)hr_reg_read(&context, QPC_MTU); 5474 qp_attr->path_mig_state = IB_MIG_ARMED; 5475 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 5476 if (hr_qp->ibqp.qp_type == IB_QPT_UD) 5477 qp_attr->qkey = le32_to_cpu(context.qkey_xrcd); 5478 5479 qp_attr->rq_psn = hr_reg_read(&context, QPC_RX_REQ_EPSN); 5480 qp_attr->sq_psn = (u32)hr_reg_read(&context, QPC_SQ_CUR_PSN); 5481 qp_attr->dest_qp_num = hr_reg_read(&context, QPC_DQPN); 5482 qp_attr->qp_access_flags = 5483 ((hr_reg_read(&context, QPC_RRE)) << V2_QP_RRE_S) | 5484 ((hr_reg_read(&context, QPC_RWE)) << V2_QP_RWE_S) | 5485 ((hr_reg_read(&context, QPC_ATE)) << V2_QP_ATE_S); 5486 5487 if (hr_qp->ibqp.qp_type == IB_QPT_RC || 5488 hr_qp->ibqp.qp_type == IB_QPT_XRC_INI || 5489 hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT) { 5490 struct ib_global_route *grh = 5491 rdma_ah_retrieve_grh(&qp_attr->ah_attr); 5492 5493 rdma_ah_set_sl(&qp_attr->ah_attr, 5494 hr_reg_read(&context, QPC_SL)); 5495 rdma_ah_set_port_num(&qp_attr->ah_attr, hr_qp->port + 1); 5496 rdma_ah_set_ah_flags(&qp_attr->ah_attr, IB_AH_GRH); 5497 grh->flow_label = hr_reg_read(&context, QPC_FL); 5498 grh->sgid_index = hr_reg_read(&context, QPC_GMV_IDX); 5499 grh->hop_limit = hr_reg_read(&context, QPC_HOPLIMIT); 5500 grh->traffic_class = hr_reg_read(&context, QPC_TC); 5501 5502 memcpy(grh->dgid.raw, context.dgid, sizeof(grh->dgid.raw)); 5503 } 5504 5505 qp_attr->port_num = hr_qp->port + 1; 5506 qp_attr->sq_draining = 0; 5507 qp_attr->max_rd_atomic = 1 << hr_reg_read(&context, QPC_SR_MAX); 5508 qp_attr->max_dest_rd_atomic = 1 << hr_reg_read(&context, QPC_RR_MAX); 5509 5510 qp_attr->min_rnr_timer = (u8)hr_reg_read(&context, QPC_MIN_RNR_TIME); 5511 qp_attr->timeout = (u8)hr_reg_read(&context, QPC_AT); 5512 qp_attr->retry_cnt = hr_reg_read(&context, QPC_RETRY_NUM_INIT); 5513 qp_attr->rnr_retry = hr_reg_read(&context, QPC_RNR_NUM_INIT); 5514 5515 done: 5516 qp_attr->cur_qp_state = qp_attr->qp_state; 5517 qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt; 5518 qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge; 5519 qp_attr->cap.max_inline_data = hr_qp->max_inline_data; 5520 5521 qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt; 5522 qp_attr->cap.max_send_sge = hr_qp->sq.max_gs; 5523 5524 qp_init_attr->qp_context = ibqp->qp_context; 5525 qp_init_attr->qp_type = ibqp->qp_type; 5526 qp_init_attr->recv_cq = ibqp->recv_cq; 5527 qp_init_attr->send_cq = ibqp->send_cq; 5528 qp_init_attr->srq = ibqp->srq; 5529 qp_init_attr->cap = qp_attr->cap; 5530 qp_init_attr->sq_sig_type = hr_qp->sq_signal_bits; 5531 5532 out: 5533 mutex_unlock(&hr_qp->mutex); 5534 return ret; 5535 } 5536 5537 static inline int modify_qp_is_ok(struct hns_roce_qp *hr_qp) 5538 { 5539 return ((hr_qp->ibqp.qp_type == IB_QPT_RC || 5540 hr_qp->ibqp.qp_type == IB_QPT_UD || 5541 hr_qp->ibqp.qp_type == IB_QPT_XRC_INI || 5542 hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT) && 5543 hr_qp->state != IB_QPS_RESET); 5544 } 5545 5546 static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, 5547 struct hns_roce_qp *hr_qp, 5548 struct ib_udata *udata) 5549 { 5550 struct ib_device *ibdev = &hr_dev->ib_dev; 5551 struct hns_roce_cq *send_cq, *recv_cq; 5552 unsigned long flags; 5553 int ret = 0; 5554 5555 if (modify_qp_is_ok(hr_qp)) { 5556 /* Modify qp to reset before destroying qp */ 5557 ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0, 5558 hr_qp->state, IB_QPS_RESET); 5559 if (ret) 5560 ibdev_err(ibdev, 5561 "failed to modify QP to RST, ret = %d.\n", 5562 ret); 5563 } 5564 5565 send_cq = hr_qp->ibqp.send_cq ? to_hr_cq(hr_qp->ibqp.send_cq) : NULL; 5566 recv_cq = hr_qp->ibqp.recv_cq ? to_hr_cq(hr_qp->ibqp.recv_cq) : NULL; 5567 5568 spin_lock_irqsave(&hr_dev->qp_list_lock, flags); 5569 hns_roce_lock_cqs(send_cq, recv_cq); 5570 5571 if (!udata) { 5572 if (recv_cq) 5573 __hns_roce_v2_cq_clean(recv_cq, hr_qp->qpn, 5574 (hr_qp->ibqp.srq ? 5575 to_hr_srq(hr_qp->ibqp.srq) : 5576 NULL)); 5577 5578 if (send_cq && send_cq != recv_cq) 5579 __hns_roce_v2_cq_clean(send_cq, hr_qp->qpn, NULL); 5580 } 5581 5582 hns_roce_qp_remove(hr_dev, hr_qp); 5583 5584 hns_roce_unlock_cqs(send_cq, recv_cq); 5585 spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags); 5586 5587 return ret; 5588 } 5589 5590 int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 5591 { 5592 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 5593 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 5594 int ret; 5595 5596 ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata); 5597 if (ret) 5598 ibdev_err(&hr_dev->ib_dev, 5599 "failed to destroy QP, QPN = 0x%06lx, ret = %d.\n", 5600 hr_qp->qpn, ret); 5601 5602 hns_roce_qp_destroy(hr_dev, hr_qp, udata); 5603 5604 return 0; 5605 } 5606 5607 static int hns_roce_v2_qp_flow_control_init(struct hns_roce_dev *hr_dev, 5608 struct hns_roce_qp *hr_qp) 5609 { 5610 struct ib_device *ibdev = &hr_dev->ib_dev; 5611 struct hns_roce_sccc_clr_done *resp; 5612 struct hns_roce_sccc_clr *clr; 5613 struct hns_roce_cmq_desc desc; 5614 int ret, i; 5615 5616 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) 5617 return 0; 5618 5619 mutex_lock(&hr_dev->qp_table.scc_mutex); 5620 5621 /* set scc ctx clear done flag */ 5622 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_RESET_SCCC, false); 5623 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 5624 if (ret) { 5625 ibdev_err(ibdev, "failed to reset SCC ctx, ret = %d.\n", ret); 5626 goto out; 5627 } 5628 5629 /* clear scc context */ 5630 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CLR_SCCC, false); 5631 clr = (struct hns_roce_sccc_clr *)desc.data; 5632 clr->qpn = cpu_to_le32(hr_qp->qpn); 5633 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 5634 if (ret) { 5635 ibdev_err(ibdev, "failed to clear SCC ctx, ret = %d.\n", ret); 5636 goto out; 5637 } 5638 5639 /* query scc context clear is done or not */ 5640 resp = (struct hns_roce_sccc_clr_done *)desc.data; 5641 for (i = 0; i <= HNS_ROCE_CMQ_SCC_CLR_DONE_CNT; i++) { 5642 hns_roce_cmq_setup_basic_desc(&desc, 5643 HNS_ROCE_OPC_QUERY_SCCC, true); 5644 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 5645 if (ret) { 5646 ibdev_err(ibdev, "failed to query clr cmq, ret = %d\n", 5647 ret); 5648 goto out; 5649 } 5650 5651 if (resp->clr_done) 5652 goto out; 5653 5654 msleep(20); 5655 } 5656 5657 ibdev_err(ibdev, "query SCC clr done flag overtime.\n"); 5658 ret = -ETIMEDOUT; 5659 5660 out: 5661 mutex_unlock(&hr_dev->qp_table.scc_mutex); 5662 return ret; 5663 } 5664 5665 #define DMA_IDX_SHIFT 3 5666 #define DMA_WQE_SHIFT 3 5667 5668 static int hns_roce_v2_write_srqc_index_queue(struct hns_roce_srq *srq, 5669 struct hns_roce_srq_context *ctx) 5670 { 5671 struct hns_roce_idx_que *idx_que = &srq->idx_que; 5672 struct ib_device *ibdev = srq->ibsrq.device; 5673 struct hns_roce_dev *hr_dev = to_hr_dev(ibdev); 5674 u64 mtts_idx[MTT_MIN_COUNT] = {}; 5675 dma_addr_t dma_handle_idx = 0; 5676 int ret; 5677 5678 /* Get physical address of idx que buf */ 5679 ret = hns_roce_mtr_find(hr_dev, &idx_que->mtr, 0, mtts_idx, 5680 ARRAY_SIZE(mtts_idx), &dma_handle_idx); 5681 if (ret < 1) { 5682 ibdev_err(ibdev, "failed to find mtr for SRQ idx, ret = %d.\n", 5683 ret); 5684 return -ENOBUFS; 5685 } 5686 5687 hr_reg_write(ctx, SRQC_IDX_HOP_NUM, 5688 to_hr_hem_hopnum(hr_dev->caps.idx_hop_num, srq->wqe_cnt)); 5689 5690 hr_reg_write(ctx, SRQC_IDX_BT_BA_L, dma_handle_idx >> DMA_IDX_SHIFT); 5691 hr_reg_write(ctx, SRQC_IDX_BT_BA_H, 5692 upper_32_bits(dma_handle_idx >> DMA_IDX_SHIFT)); 5693 5694 hr_reg_write(ctx, SRQC_IDX_BA_PG_SZ, 5695 to_hr_hw_page_shift(idx_que->mtr.hem_cfg.ba_pg_shift)); 5696 hr_reg_write(ctx, SRQC_IDX_BUF_PG_SZ, 5697 to_hr_hw_page_shift(idx_que->mtr.hem_cfg.buf_pg_shift)); 5698 5699 hr_reg_write(ctx, SRQC_IDX_CUR_BLK_ADDR_L, 5700 to_hr_hw_page_addr(mtts_idx[0])); 5701 hr_reg_write(ctx, SRQC_IDX_CUR_BLK_ADDR_H, 5702 upper_32_bits(to_hr_hw_page_addr(mtts_idx[0]))); 5703 5704 hr_reg_write(ctx, SRQC_IDX_NXT_BLK_ADDR_L, 5705 to_hr_hw_page_addr(mtts_idx[1])); 5706 hr_reg_write(ctx, SRQC_IDX_NXT_BLK_ADDR_H, 5707 upper_32_bits(to_hr_hw_page_addr(mtts_idx[1]))); 5708 5709 return 0; 5710 } 5711 5712 static int hns_roce_v2_write_srqc(struct hns_roce_srq *srq, void *mb_buf) 5713 { 5714 struct ib_device *ibdev = srq->ibsrq.device; 5715 struct hns_roce_dev *hr_dev = to_hr_dev(ibdev); 5716 struct hns_roce_srq_context *ctx = mb_buf; 5717 u64 mtts_wqe[MTT_MIN_COUNT] = {}; 5718 dma_addr_t dma_handle_wqe = 0; 5719 int ret; 5720 5721 memset(ctx, 0, sizeof(*ctx)); 5722 5723 /* Get the physical address of srq buf */ 5724 ret = hns_roce_mtr_find(hr_dev, &srq->buf_mtr, 0, mtts_wqe, 5725 ARRAY_SIZE(mtts_wqe), &dma_handle_wqe); 5726 if (ret < 1) { 5727 ibdev_err(ibdev, "failed to find mtr for SRQ WQE, ret = %d.\n", 5728 ret); 5729 return -ENOBUFS; 5730 } 5731 5732 hr_reg_write(ctx, SRQC_SRQ_ST, 1); 5733 hr_reg_write_bool(ctx, SRQC_SRQ_TYPE, 5734 srq->ibsrq.srq_type == IB_SRQT_XRC); 5735 hr_reg_write(ctx, SRQC_PD, to_hr_pd(srq->ibsrq.pd)->pdn); 5736 hr_reg_write(ctx, SRQC_SRQN, srq->srqn); 5737 hr_reg_write(ctx, SRQC_XRCD, srq->xrcdn); 5738 hr_reg_write(ctx, SRQC_XRC_CQN, srq->cqn); 5739 hr_reg_write(ctx, SRQC_SHIFT, ilog2(srq->wqe_cnt)); 5740 hr_reg_write(ctx, SRQC_RQWS, 5741 srq->max_gs <= 0 ? 0 : fls(srq->max_gs - 1)); 5742 5743 hr_reg_write(ctx, SRQC_WQE_HOP_NUM, 5744 to_hr_hem_hopnum(hr_dev->caps.srqwqe_hop_num, 5745 srq->wqe_cnt)); 5746 5747 hr_reg_write(ctx, SRQC_WQE_BT_BA_L, dma_handle_wqe >> DMA_WQE_SHIFT); 5748 hr_reg_write(ctx, SRQC_WQE_BT_BA_H, 5749 upper_32_bits(dma_handle_wqe >> DMA_WQE_SHIFT)); 5750 5751 hr_reg_write(ctx, SRQC_WQE_BA_PG_SZ, 5752 to_hr_hw_page_shift(srq->buf_mtr.hem_cfg.ba_pg_shift)); 5753 hr_reg_write(ctx, SRQC_WQE_BUF_PG_SZ, 5754 to_hr_hw_page_shift(srq->buf_mtr.hem_cfg.buf_pg_shift)); 5755 5756 return hns_roce_v2_write_srqc_index_queue(srq, ctx); 5757 } 5758 5759 static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq, 5760 struct ib_srq_attr *srq_attr, 5761 enum ib_srq_attr_mask srq_attr_mask, 5762 struct ib_udata *udata) 5763 { 5764 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device); 5765 struct hns_roce_srq *srq = to_hr_srq(ibsrq); 5766 struct hns_roce_srq_context *srq_context; 5767 struct hns_roce_srq_context *srqc_mask; 5768 struct hns_roce_cmd_mailbox *mailbox; 5769 int ret; 5770 5771 /* Resizing SRQs is not supported yet */ 5772 if (srq_attr_mask & IB_SRQ_MAX_WR) 5773 return -EINVAL; 5774 5775 if (srq_attr_mask & IB_SRQ_LIMIT) { 5776 if (srq_attr->srq_limit > srq->wqe_cnt) 5777 return -EINVAL; 5778 5779 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 5780 if (IS_ERR(mailbox)) 5781 return PTR_ERR(mailbox); 5782 5783 srq_context = mailbox->buf; 5784 srqc_mask = (struct hns_roce_srq_context *)mailbox->buf + 1; 5785 5786 memset(srqc_mask, 0xff, sizeof(*srqc_mask)); 5787 5788 hr_reg_write(srq_context, SRQC_LIMIT_WL, srq_attr->srq_limit); 5789 hr_reg_clear(srqc_mask, SRQC_LIMIT_WL); 5790 5791 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, 5792 HNS_ROCE_CMD_MODIFY_SRQC, srq->srqn); 5793 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 5794 if (ret) { 5795 ibdev_err(&hr_dev->ib_dev, 5796 "failed to handle cmd of modifying SRQ, ret = %d.\n", 5797 ret); 5798 return ret; 5799 } 5800 } 5801 5802 return 0; 5803 } 5804 5805 static int hns_roce_v2_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr) 5806 { 5807 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device); 5808 struct hns_roce_srq *srq = to_hr_srq(ibsrq); 5809 struct hns_roce_srq_context *srq_context; 5810 struct hns_roce_cmd_mailbox *mailbox; 5811 int ret; 5812 5813 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 5814 if (IS_ERR(mailbox)) 5815 return PTR_ERR(mailbox); 5816 5817 srq_context = mailbox->buf; 5818 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, 5819 HNS_ROCE_CMD_QUERY_SRQC, srq->srqn); 5820 if (ret) { 5821 ibdev_err(&hr_dev->ib_dev, 5822 "failed to process cmd of querying SRQ, ret = %d.\n", 5823 ret); 5824 goto out; 5825 } 5826 5827 attr->srq_limit = hr_reg_read(srq_context, SRQC_LIMIT_WL); 5828 attr->max_wr = srq->wqe_cnt; 5829 attr->max_sge = srq->max_gs - srq->rsv_sge; 5830 5831 out: 5832 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 5833 return ret; 5834 } 5835 5836 static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) 5837 { 5838 struct hns_roce_dev *hr_dev = to_hr_dev(cq->device); 5839 struct hns_roce_v2_cq_context *cq_context; 5840 struct hns_roce_cq *hr_cq = to_hr_cq(cq); 5841 struct hns_roce_v2_cq_context *cqc_mask; 5842 struct hns_roce_cmd_mailbox *mailbox; 5843 int ret; 5844 5845 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 5846 if (IS_ERR(mailbox)) 5847 return PTR_ERR(mailbox); 5848 5849 cq_context = mailbox->buf; 5850 cqc_mask = (struct hns_roce_v2_cq_context *)mailbox->buf + 1; 5851 5852 memset(cqc_mask, 0xff, sizeof(*cqc_mask)); 5853 5854 hr_reg_write(cq_context, CQC_CQ_MAX_CNT, cq_count); 5855 hr_reg_clear(cqc_mask, CQC_CQ_MAX_CNT); 5856 5857 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) { 5858 if (cq_period * HNS_ROCE_CLOCK_ADJUST > USHRT_MAX) { 5859 dev_info(hr_dev->dev, 5860 "cq_period(%u) reached the upper limit, adjusted to 65.\n", 5861 cq_period); 5862 cq_period = HNS_ROCE_MAX_CQ_PERIOD; 5863 } 5864 cq_period *= HNS_ROCE_CLOCK_ADJUST; 5865 } 5866 hr_reg_write(cq_context, CQC_CQ_PERIOD, cq_period); 5867 hr_reg_clear(cqc_mask, CQC_CQ_PERIOD); 5868 5869 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, 5870 HNS_ROCE_CMD_MODIFY_CQC, hr_cq->cqn); 5871 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 5872 if (ret) 5873 ibdev_err(&hr_dev->ib_dev, 5874 "failed to process cmd when modifying CQ, ret = %d.\n", 5875 ret); 5876 5877 return ret; 5878 } 5879 5880 static int hns_roce_v2_query_cqc(struct hns_roce_dev *hr_dev, u32 cqn, 5881 void *buffer) 5882 { 5883 struct hns_roce_v2_cq_context *context; 5884 struct hns_roce_cmd_mailbox *mailbox; 5885 int ret; 5886 5887 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 5888 if (IS_ERR(mailbox)) 5889 return PTR_ERR(mailbox); 5890 5891 context = mailbox->buf; 5892 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, 5893 HNS_ROCE_CMD_QUERY_CQC, cqn); 5894 if (ret) { 5895 ibdev_err(&hr_dev->ib_dev, 5896 "failed to process cmd when querying CQ, ret = %d.\n", 5897 ret); 5898 goto err_mailbox; 5899 } 5900 5901 memcpy(buffer, context, sizeof(*context)); 5902 5903 err_mailbox: 5904 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 5905 5906 return ret; 5907 } 5908 5909 static int hns_roce_v2_query_mpt(struct hns_roce_dev *hr_dev, u32 key, 5910 void *buffer) 5911 { 5912 struct hns_roce_v2_mpt_entry *context; 5913 struct hns_roce_cmd_mailbox *mailbox; 5914 int ret; 5915 5916 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 5917 if (IS_ERR(mailbox)) 5918 return PTR_ERR(mailbox); 5919 5920 context = mailbox->buf; 5921 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_MPT, 5922 key_to_hw_index(key)); 5923 if (ret) { 5924 ibdev_err(&hr_dev->ib_dev, 5925 "failed to process cmd when querying MPT, ret = %d.\n", 5926 ret); 5927 goto err_mailbox; 5928 } 5929 5930 memcpy(buffer, context, sizeof(*context)); 5931 5932 err_mailbox: 5933 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 5934 5935 return ret; 5936 } 5937 5938 static void hns_roce_irq_work_handle(struct work_struct *work) 5939 { 5940 struct hns_roce_work *irq_work = 5941 container_of(work, struct hns_roce_work, work); 5942 struct ib_device *ibdev = &irq_work->hr_dev->ib_dev; 5943 5944 switch (irq_work->event_type) { 5945 case HNS_ROCE_EVENT_TYPE_PATH_MIG: 5946 ibdev_info(ibdev, "path migrated succeeded.\n"); 5947 break; 5948 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED: 5949 ibdev_warn(ibdev, "path migration failed.\n"); 5950 break; 5951 case HNS_ROCE_EVENT_TYPE_COMM_EST: 5952 break; 5953 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: 5954 ibdev_warn(ibdev, "send queue drained.\n"); 5955 break; 5956 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: 5957 ibdev_err(ibdev, "local work queue 0x%x catast error, sub_event type is: %d\n", 5958 irq_work->queue_num, irq_work->sub_type); 5959 break; 5960 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: 5961 ibdev_err(ibdev, "invalid request local work queue 0x%x error.\n", 5962 irq_work->queue_num); 5963 break; 5964 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: 5965 ibdev_err(ibdev, "local access violation work queue 0x%x error, sub_event type is: %d\n", 5966 irq_work->queue_num, irq_work->sub_type); 5967 break; 5968 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH: 5969 ibdev_warn(ibdev, "SRQ limit reach.\n"); 5970 break; 5971 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH: 5972 ibdev_warn(ibdev, "SRQ last wqe reach.\n"); 5973 break; 5974 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR: 5975 ibdev_err(ibdev, "SRQ catas error.\n"); 5976 break; 5977 case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR: 5978 ibdev_err(ibdev, "CQ 0x%x access err.\n", irq_work->queue_num); 5979 break; 5980 case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW: 5981 ibdev_warn(ibdev, "CQ 0x%x overflow\n", irq_work->queue_num); 5982 break; 5983 case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW: 5984 ibdev_warn(ibdev, "DB overflow.\n"); 5985 break; 5986 case HNS_ROCE_EVENT_TYPE_FLR: 5987 ibdev_warn(ibdev, "function level reset.\n"); 5988 break; 5989 case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION: 5990 ibdev_err(ibdev, "xrc domain violation error.\n"); 5991 break; 5992 case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH: 5993 ibdev_err(ibdev, "invalid xrceth error.\n"); 5994 break; 5995 default: 5996 break; 5997 } 5998 5999 kfree(irq_work); 6000 } 6001 6002 static void hns_roce_v2_init_irq_work(struct hns_roce_dev *hr_dev, 6003 struct hns_roce_eq *eq, u32 queue_num) 6004 { 6005 struct hns_roce_work *irq_work; 6006 6007 irq_work = kzalloc(sizeof(struct hns_roce_work), GFP_ATOMIC); 6008 if (!irq_work) 6009 return; 6010 6011 INIT_WORK(&irq_work->work, hns_roce_irq_work_handle); 6012 irq_work->hr_dev = hr_dev; 6013 irq_work->event_type = eq->event_type; 6014 irq_work->sub_type = eq->sub_type; 6015 irq_work->queue_num = queue_num; 6016 queue_work(hr_dev->irq_workq, &irq_work->work); 6017 } 6018 6019 static void update_eq_db(struct hns_roce_eq *eq) 6020 { 6021 struct hns_roce_dev *hr_dev = eq->hr_dev; 6022 struct hns_roce_v2_db eq_db = {}; 6023 6024 if (eq->type_flag == HNS_ROCE_AEQ) { 6025 hr_reg_write(&eq_db, EQ_DB_CMD, 6026 eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ? 6027 HNS_ROCE_EQ_DB_CMD_AEQ : 6028 HNS_ROCE_EQ_DB_CMD_AEQ_ARMED); 6029 } else { 6030 hr_reg_write(&eq_db, EQ_DB_TAG, eq->eqn); 6031 6032 hr_reg_write(&eq_db, EQ_DB_CMD, 6033 eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ? 6034 HNS_ROCE_EQ_DB_CMD_CEQ : 6035 HNS_ROCE_EQ_DB_CMD_CEQ_ARMED); 6036 } 6037 6038 hr_reg_write(&eq_db, EQ_DB_CI, eq->cons_index); 6039 6040 hns_roce_write64(hr_dev, (__le32 *)&eq_db, eq->db_reg); 6041 } 6042 6043 static struct hns_roce_aeqe *next_aeqe_sw_v2(struct hns_roce_eq *eq) 6044 { 6045 struct hns_roce_aeqe *aeqe; 6046 6047 aeqe = hns_roce_buf_offset(eq->mtr.kmem, 6048 (eq->cons_index & (eq->entries - 1)) * 6049 eq->eqe_size); 6050 6051 return (hr_reg_read(aeqe, AEQE_OWNER) ^ 6052 !!(eq->cons_index & eq->entries)) ? aeqe : NULL; 6053 } 6054 6055 static irqreturn_t hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev, 6056 struct hns_roce_eq *eq) 6057 { 6058 struct device *dev = hr_dev->dev; 6059 struct hns_roce_aeqe *aeqe = next_aeqe_sw_v2(eq); 6060 irqreturn_t aeqe_found = IRQ_NONE; 6061 int event_type; 6062 u32 queue_num; 6063 int sub_type; 6064 6065 while (aeqe) { 6066 /* Make sure we read AEQ entry after we have checked the 6067 * ownership bit 6068 */ 6069 dma_rmb(); 6070 6071 event_type = hr_reg_read(aeqe, AEQE_EVENT_TYPE); 6072 sub_type = hr_reg_read(aeqe, AEQE_SUB_TYPE); 6073 queue_num = hr_reg_read(aeqe, AEQE_EVENT_QUEUE_NUM); 6074 6075 switch (event_type) { 6076 case HNS_ROCE_EVENT_TYPE_PATH_MIG: 6077 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED: 6078 case HNS_ROCE_EVENT_TYPE_COMM_EST: 6079 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: 6080 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: 6081 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH: 6082 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: 6083 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: 6084 case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION: 6085 case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH: 6086 hns_roce_qp_event(hr_dev, queue_num, event_type); 6087 break; 6088 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH: 6089 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR: 6090 hns_roce_srq_event(hr_dev, queue_num, event_type); 6091 break; 6092 case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR: 6093 case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW: 6094 hns_roce_cq_event(hr_dev, queue_num, event_type); 6095 break; 6096 case HNS_ROCE_EVENT_TYPE_MB: 6097 hns_roce_cmd_event(hr_dev, 6098 le16_to_cpu(aeqe->event.cmd.token), 6099 aeqe->event.cmd.status, 6100 le64_to_cpu(aeqe->event.cmd.out_param)); 6101 break; 6102 case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW: 6103 case HNS_ROCE_EVENT_TYPE_FLR: 6104 break; 6105 default: 6106 dev_err(dev, "unhandled event %d on EQ %d at idx %u.\n", 6107 event_type, eq->eqn, eq->cons_index); 6108 break; 6109 } 6110 6111 eq->event_type = event_type; 6112 eq->sub_type = sub_type; 6113 ++eq->cons_index; 6114 aeqe_found = IRQ_HANDLED; 6115 6116 hns_roce_v2_init_irq_work(hr_dev, eq, queue_num); 6117 6118 aeqe = next_aeqe_sw_v2(eq); 6119 } 6120 6121 update_eq_db(eq); 6122 6123 return IRQ_RETVAL(aeqe_found); 6124 } 6125 6126 static struct hns_roce_ceqe *next_ceqe_sw_v2(struct hns_roce_eq *eq) 6127 { 6128 struct hns_roce_ceqe *ceqe; 6129 6130 ceqe = hns_roce_buf_offset(eq->mtr.kmem, 6131 (eq->cons_index & (eq->entries - 1)) * 6132 eq->eqe_size); 6133 6134 return (hr_reg_read(ceqe, CEQE_OWNER) ^ 6135 !!(eq->cons_index & eq->entries)) ? ceqe : NULL; 6136 } 6137 6138 static irqreturn_t hns_roce_v2_ceq_int(struct hns_roce_dev *hr_dev, 6139 struct hns_roce_eq *eq) 6140 { 6141 struct hns_roce_ceqe *ceqe = next_ceqe_sw_v2(eq); 6142 irqreturn_t ceqe_found = IRQ_NONE; 6143 u32 cqn; 6144 6145 while (ceqe) { 6146 /* Make sure we read CEQ entry after we have checked the 6147 * ownership bit 6148 */ 6149 dma_rmb(); 6150 6151 cqn = hr_reg_read(ceqe, CEQE_CQN); 6152 6153 hns_roce_cq_completion(hr_dev, cqn); 6154 6155 ++eq->cons_index; 6156 ceqe_found = IRQ_HANDLED; 6157 6158 ceqe = next_ceqe_sw_v2(eq); 6159 } 6160 6161 update_eq_db(eq); 6162 6163 return IRQ_RETVAL(ceqe_found); 6164 } 6165 6166 static irqreturn_t hns_roce_v2_msix_interrupt_eq(int irq, void *eq_ptr) 6167 { 6168 struct hns_roce_eq *eq = eq_ptr; 6169 struct hns_roce_dev *hr_dev = eq->hr_dev; 6170 irqreturn_t int_work; 6171 6172 if (eq->type_flag == HNS_ROCE_CEQ) 6173 /* Completion event interrupt */ 6174 int_work = hns_roce_v2_ceq_int(hr_dev, eq); 6175 else 6176 /* Asynchronous event interrupt */ 6177 int_work = hns_roce_v2_aeq_int(hr_dev, eq); 6178 6179 return IRQ_RETVAL(int_work); 6180 } 6181 6182 static irqreturn_t abnormal_interrupt_basic(struct hns_roce_dev *hr_dev, 6183 u32 int_st) 6184 { 6185 struct pci_dev *pdev = hr_dev->pci_dev; 6186 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev); 6187 const struct hnae3_ae_ops *ops = ae_dev->ops; 6188 irqreturn_t int_work = IRQ_NONE; 6189 u32 int_en; 6190 6191 int_en = roce_read(hr_dev, ROCEE_VF_ABN_INT_EN_REG); 6192 6193 if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S)) { 6194 dev_err(hr_dev->dev, "AEQ overflow!\n"); 6195 6196 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, 6197 1 << HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S); 6198 6199 /* Set reset level for reset_event() */ 6200 if (ops->set_default_reset_request) 6201 ops->set_default_reset_request(ae_dev, 6202 HNAE3_FUNC_RESET); 6203 if (ops->reset_event) 6204 ops->reset_event(pdev, NULL); 6205 6206 int_en |= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S; 6207 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en); 6208 6209 int_work = IRQ_HANDLED; 6210 } else { 6211 dev_err(hr_dev->dev, "there is no basic abn irq found.\n"); 6212 } 6213 6214 return IRQ_RETVAL(int_work); 6215 } 6216 6217 static int fmea_ram_ecc_query(struct hns_roce_dev *hr_dev, 6218 struct fmea_ram_ecc *ecc_info) 6219 { 6220 struct hns_roce_cmq_desc desc; 6221 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data; 6222 int ret; 6223 6224 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_QUERY_RAM_ECC, true); 6225 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 6226 if (ret) 6227 return ret; 6228 6229 ecc_info->is_ecc_err = hr_reg_read(req, QUERY_RAM_ECC_1BIT_ERR); 6230 ecc_info->res_type = hr_reg_read(req, QUERY_RAM_ECC_RES_TYPE); 6231 ecc_info->index = hr_reg_read(req, QUERY_RAM_ECC_TAG); 6232 6233 return 0; 6234 } 6235 6236 static int fmea_recover_gmv(struct hns_roce_dev *hr_dev, u32 idx) 6237 { 6238 struct hns_roce_cmq_desc desc; 6239 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data; 6240 u32 addr_upper; 6241 u32 addr_low; 6242 int ret; 6243 6244 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT, true); 6245 hr_reg_write(req, CFG_GMV_BT_IDX, idx); 6246 6247 ret = hns_roce_cmq_send(hr_dev, &desc, 1); 6248 if (ret) { 6249 dev_err(hr_dev->dev, 6250 "failed to execute cmd to read gmv, ret = %d.\n", ret); 6251 return ret; 6252 } 6253 6254 addr_low = hr_reg_read(req, CFG_GMV_BT_BA_L); 6255 addr_upper = hr_reg_read(req, CFG_GMV_BT_BA_H); 6256 6257 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT, false); 6258 hr_reg_write(req, CFG_GMV_BT_BA_L, addr_low); 6259 hr_reg_write(req, CFG_GMV_BT_BA_H, addr_upper); 6260 hr_reg_write(req, CFG_GMV_BT_IDX, idx); 6261 6262 return hns_roce_cmq_send(hr_dev, &desc, 1); 6263 } 6264 6265 static u64 fmea_get_ram_res_addr(u32 res_type, __le64 *data) 6266 { 6267 if (res_type == ECC_RESOURCE_QPC_TIMER || 6268 res_type == ECC_RESOURCE_CQC_TIMER || 6269 res_type == ECC_RESOURCE_SCCC) 6270 return le64_to_cpu(*data); 6271 6272 return le64_to_cpu(*data) << PAGE_SHIFT; 6273 } 6274 6275 static int fmea_recover_others(struct hns_roce_dev *hr_dev, u32 res_type, 6276 u32 index) 6277 { 6278 u8 write_bt0_op = fmea_ram_res[res_type].write_bt0_op; 6279 u8 read_bt0_op = fmea_ram_res[res_type].read_bt0_op; 6280 struct hns_roce_cmd_mailbox *mailbox; 6281 u64 addr; 6282 int ret; 6283 6284 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 6285 if (IS_ERR(mailbox)) 6286 return PTR_ERR(mailbox); 6287 6288 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, read_bt0_op, index); 6289 if (ret) { 6290 dev_err(hr_dev->dev, 6291 "failed to execute cmd to read fmea ram, ret = %d.\n", 6292 ret); 6293 goto out; 6294 } 6295 6296 addr = fmea_get_ram_res_addr(res_type, mailbox->buf); 6297 6298 ret = hns_roce_cmd_mbox(hr_dev, addr, 0, write_bt0_op, index); 6299 if (ret) 6300 dev_err(hr_dev->dev, 6301 "failed to execute cmd to write fmea ram, ret = %d.\n", 6302 ret); 6303 6304 out: 6305 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 6306 return ret; 6307 } 6308 6309 static void fmea_ram_ecc_recover(struct hns_roce_dev *hr_dev, 6310 struct fmea_ram_ecc *ecc_info) 6311 { 6312 u32 res_type = ecc_info->res_type; 6313 u32 index = ecc_info->index; 6314 int ret; 6315 6316 BUILD_BUG_ON(ARRAY_SIZE(fmea_ram_res) != ECC_RESOURCE_COUNT); 6317 6318 if (res_type >= ECC_RESOURCE_COUNT) { 6319 dev_err(hr_dev->dev, "unsupported fmea ram ecc type %u.\n", 6320 res_type); 6321 return; 6322 } 6323 6324 if (res_type == ECC_RESOURCE_GMV) 6325 ret = fmea_recover_gmv(hr_dev, index); 6326 else 6327 ret = fmea_recover_others(hr_dev, res_type, index); 6328 if (ret) 6329 dev_err(hr_dev->dev, 6330 "failed to recover %s, index = %u, ret = %d.\n", 6331 fmea_ram_res[res_type].name, index, ret); 6332 } 6333 6334 static void fmea_ram_ecc_work(struct work_struct *ecc_work) 6335 { 6336 struct hns_roce_dev *hr_dev = 6337 container_of(ecc_work, struct hns_roce_dev, ecc_work); 6338 struct fmea_ram_ecc ecc_info = {}; 6339 6340 if (fmea_ram_ecc_query(hr_dev, &ecc_info)) { 6341 dev_err(hr_dev->dev, "failed to query fmea ram ecc.\n"); 6342 return; 6343 } 6344 6345 if (!ecc_info.is_ecc_err) { 6346 dev_err(hr_dev->dev, "there is no fmea ram ecc err found.\n"); 6347 return; 6348 } 6349 6350 fmea_ram_ecc_recover(hr_dev, &ecc_info); 6351 } 6352 6353 static irqreturn_t hns_roce_v2_msix_interrupt_abn(int irq, void *dev_id) 6354 { 6355 struct hns_roce_dev *hr_dev = dev_id; 6356 irqreturn_t int_work = IRQ_NONE; 6357 u32 int_st; 6358 6359 int_st = roce_read(hr_dev, ROCEE_VF_ABN_INT_ST_REG); 6360 6361 if (int_st) { 6362 int_work = abnormal_interrupt_basic(hr_dev, int_st); 6363 } else if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) { 6364 queue_work(hr_dev->irq_workq, &hr_dev->ecc_work); 6365 int_work = IRQ_HANDLED; 6366 } else { 6367 dev_err(hr_dev->dev, "there is no abnormal irq found.\n"); 6368 } 6369 6370 return IRQ_RETVAL(int_work); 6371 } 6372 6373 static void hns_roce_v2_int_mask_enable(struct hns_roce_dev *hr_dev, 6374 int eq_num, u32 enable_flag) 6375 { 6376 int i; 6377 6378 for (i = 0; i < eq_num; i++) 6379 roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG + 6380 i * EQ_REG_OFFSET, enable_flag); 6381 6382 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, enable_flag); 6383 roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG, enable_flag); 6384 } 6385 6386 static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev, u32 eqn) 6387 { 6388 struct device *dev = hr_dev->dev; 6389 int ret; 6390 u8 cmd; 6391 6392 if (eqn < hr_dev->caps.num_comp_vectors) 6393 cmd = HNS_ROCE_CMD_DESTROY_CEQC; 6394 else 6395 cmd = HNS_ROCE_CMD_DESTROY_AEQC; 6396 6397 ret = hns_roce_destroy_hw_ctx(hr_dev, cmd, eqn & HNS_ROCE_V2_EQN_M); 6398 if (ret) 6399 dev_err(dev, "[mailbox cmd] destroy eqc(%u) failed.\n", eqn); 6400 } 6401 6402 static void free_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq) 6403 { 6404 hns_roce_mtr_destroy(hr_dev, &eq->mtr); 6405 } 6406 6407 static void init_eq_config(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq) 6408 { 6409 eq->db_reg = hr_dev->reg_base + ROCEE_VF_EQ_DB_CFG0_REG; 6410 eq->cons_index = 0; 6411 eq->over_ignore = HNS_ROCE_V2_EQ_OVER_IGNORE_0; 6412 eq->coalesce = HNS_ROCE_V2_EQ_COALESCE_0; 6413 eq->arm_st = HNS_ROCE_V2_EQ_ALWAYS_ARMED; 6414 eq->shift = ilog2((unsigned int)eq->entries); 6415 } 6416 6417 static int config_eqc(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq, 6418 void *mb_buf) 6419 { 6420 u64 eqe_ba[MTT_MIN_COUNT] = { 0 }; 6421 struct hns_roce_eq_context *eqc; 6422 u64 bt_ba = 0; 6423 int count; 6424 6425 eqc = mb_buf; 6426 memset(eqc, 0, sizeof(struct hns_roce_eq_context)); 6427 6428 init_eq_config(hr_dev, eq); 6429 6430 /* if not multi-hop, eqe buffer only use one trunk */ 6431 count = hns_roce_mtr_find(hr_dev, &eq->mtr, 0, eqe_ba, MTT_MIN_COUNT, 6432 &bt_ba); 6433 if (count < 1) { 6434 dev_err(hr_dev->dev, "failed to find EQE mtr\n"); 6435 return -ENOBUFS; 6436 } 6437 6438 hr_reg_write(eqc, EQC_EQ_ST, HNS_ROCE_V2_EQ_STATE_VALID); 6439 hr_reg_write(eqc, EQC_EQE_HOP_NUM, eq->hop_num); 6440 hr_reg_write(eqc, EQC_OVER_IGNORE, eq->over_ignore); 6441 hr_reg_write(eqc, EQC_COALESCE, eq->coalesce); 6442 hr_reg_write(eqc, EQC_ARM_ST, eq->arm_st); 6443 hr_reg_write(eqc, EQC_EQN, eq->eqn); 6444 hr_reg_write(eqc, EQC_EQE_CNT, HNS_ROCE_EQ_INIT_EQE_CNT); 6445 hr_reg_write(eqc, EQC_EQE_BA_PG_SZ, 6446 to_hr_hw_page_shift(eq->mtr.hem_cfg.ba_pg_shift)); 6447 hr_reg_write(eqc, EQC_EQE_BUF_PG_SZ, 6448 to_hr_hw_page_shift(eq->mtr.hem_cfg.buf_pg_shift)); 6449 hr_reg_write(eqc, EQC_EQ_PROD_INDX, HNS_ROCE_EQ_INIT_PROD_IDX); 6450 hr_reg_write(eqc, EQC_EQ_MAX_CNT, eq->eq_max_cnt); 6451 6452 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) { 6453 if (eq->eq_period * HNS_ROCE_CLOCK_ADJUST > USHRT_MAX) { 6454 dev_info(hr_dev->dev, "eq_period(%u) reached the upper limit, adjusted to 65.\n", 6455 eq->eq_period); 6456 eq->eq_period = HNS_ROCE_MAX_EQ_PERIOD; 6457 } 6458 eq->eq_period *= HNS_ROCE_CLOCK_ADJUST; 6459 } 6460 6461 hr_reg_write(eqc, EQC_EQ_PERIOD, eq->eq_period); 6462 hr_reg_write(eqc, EQC_EQE_REPORT_TIMER, HNS_ROCE_EQ_INIT_REPORT_TIMER); 6463 hr_reg_write(eqc, EQC_EQE_BA_L, bt_ba >> 3); 6464 hr_reg_write(eqc, EQC_EQE_BA_H, bt_ba >> 35); 6465 hr_reg_write(eqc, EQC_SHIFT, eq->shift); 6466 hr_reg_write(eqc, EQC_MSI_INDX, HNS_ROCE_EQ_INIT_MSI_IDX); 6467 hr_reg_write(eqc, EQC_CUR_EQE_BA_L, eqe_ba[0] >> 12); 6468 hr_reg_write(eqc, EQC_CUR_EQE_BA_M, eqe_ba[0] >> 28); 6469 hr_reg_write(eqc, EQC_CUR_EQE_BA_H, eqe_ba[0] >> 60); 6470 hr_reg_write(eqc, EQC_EQ_CONS_INDX, HNS_ROCE_EQ_INIT_CONS_IDX); 6471 hr_reg_write(eqc, EQC_NEX_EQE_BA_L, eqe_ba[1] >> 12); 6472 hr_reg_write(eqc, EQC_NEX_EQE_BA_H, eqe_ba[1] >> 44); 6473 hr_reg_write(eqc, EQC_EQE_SIZE, eq->eqe_size == HNS_ROCE_V3_EQE_SIZE); 6474 6475 return 0; 6476 } 6477 6478 static int alloc_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq) 6479 { 6480 struct hns_roce_buf_attr buf_attr = {}; 6481 int err; 6482 6483 if (hr_dev->caps.eqe_hop_num == HNS_ROCE_HOP_NUM_0) 6484 eq->hop_num = 0; 6485 else 6486 eq->hop_num = hr_dev->caps.eqe_hop_num; 6487 6488 buf_attr.page_shift = hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT; 6489 buf_attr.region[0].size = eq->entries * eq->eqe_size; 6490 buf_attr.region[0].hopnum = eq->hop_num; 6491 buf_attr.region_count = 1; 6492 6493 err = hns_roce_mtr_create(hr_dev, &eq->mtr, &buf_attr, 6494 hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT, NULL, 6495 0); 6496 if (err) 6497 dev_err(hr_dev->dev, "failed to alloc EQE mtr, err %d\n", err); 6498 6499 return err; 6500 } 6501 6502 static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev, 6503 struct hns_roce_eq *eq, u8 eq_cmd) 6504 { 6505 struct hns_roce_cmd_mailbox *mailbox; 6506 int ret; 6507 6508 /* Allocate mailbox memory */ 6509 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); 6510 if (IS_ERR(mailbox)) 6511 return PTR_ERR(mailbox); 6512 6513 ret = alloc_eq_buf(hr_dev, eq); 6514 if (ret) 6515 goto free_cmd_mbox; 6516 6517 ret = config_eqc(hr_dev, eq, mailbox->buf); 6518 if (ret) 6519 goto err_cmd_mbox; 6520 6521 ret = hns_roce_create_hw_ctx(hr_dev, mailbox, eq_cmd, eq->eqn); 6522 if (ret) { 6523 dev_err(hr_dev->dev, "[mailbox cmd] create eqc failed.\n"); 6524 goto err_cmd_mbox; 6525 } 6526 6527 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 6528 6529 return 0; 6530 6531 err_cmd_mbox: 6532 free_eq_buf(hr_dev, eq); 6533 6534 free_cmd_mbox: 6535 hns_roce_free_cmd_mailbox(hr_dev, mailbox); 6536 6537 return ret; 6538 } 6539 6540 static int __hns_roce_request_irq(struct hns_roce_dev *hr_dev, int irq_num, 6541 int comp_num, int aeq_num, int other_num) 6542 { 6543 struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; 6544 int i, j; 6545 int ret; 6546 6547 for (i = 0; i < irq_num; i++) { 6548 hr_dev->irq_names[i] = kzalloc(HNS_ROCE_INT_NAME_LEN, 6549 GFP_KERNEL); 6550 if (!hr_dev->irq_names[i]) { 6551 ret = -ENOMEM; 6552 goto err_kzalloc_failed; 6553 } 6554 } 6555 6556 /* irq contains: abnormal + AEQ + CEQ */ 6557 for (j = 0; j < other_num; j++) 6558 snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN, 6559 "hns-abn-%d", j); 6560 6561 for (j = other_num; j < (other_num + aeq_num); j++) 6562 snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN, 6563 "hns-aeq-%d", j - other_num); 6564 6565 for (j = (other_num + aeq_num); j < irq_num; j++) 6566 snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN, 6567 "hns-ceq-%d", j - other_num - aeq_num); 6568 6569 for (j = 0; j < irq_num; j++) { 6570 if (j < other_num) 6571 ret = request_irq(hr_dev->irq[j], 6572 hns_roce_v2_msix_interrupt_abn, 6573 0, hr_dev->irq_names[j], hr_dev); 6574 6575 else if (j < (other_num + comp_num)) 6576 ret = request_irq(eq_table->eq[j - other_num].irq, 6577 hns_roce_v2_msix_interrupt_eq, 6578 0, hr_dev->irq_names[j + aeq_num], 6579 &eq_table->eq[j - other_num]); 6580 else 6581 ret = request_irq(eq_table->eq[j - other_num].irq, 6582 hns_roce_v2_msix_interrupt_eq, 6583 0, hr_dev->irq_names[j - comp_num], 6584 &eq_table->eq[j - other_num]); 6585 if (ret) { 6586 dev_err(hr_dev->dev, "request irq error!\n"); 6587 goto err_request_failed; 6588 } 6589 } 6590 6591 return 0; 6592 6593 err_request_failed: 6594 for (j -= 1; j >= 0; j--) 6595 if (j < other_num) 6596 free_irq(hr_dev->irq[j], hr_dev); 6597 else 6598 free_irq(eq_table->eq[j - other_num].irq, 6599 &eq_table->eq[j - other_num]); 6600 6601 err_kzalloc_failed: 6602 for (i -= 1; i >= 0; i--) 6603 kfree(hr_dev->irq_names[i]); 6604 6605 return ret; 6606 } 6607 6608 static void __hns_roce_free_irq(struct hns_roce_dev *hr_dev) 6609 { 6610 int irq_num; 6611 int eq_num; 6612 int i; 6613 6614 eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors; 6615 irq_num = eq_num + hr_dev->caps.num_other_vectors; 6616 6617 for (i = 0; i < hr_dev->caps.num_other_vectors; i++) 6618 free_irq(hr_dev->irq[i], hr_dev); 6619 6620 for (i = 0; i < eq_num; i++) 6621 free_irq(hr_dev->eq_table.eq[i].irq, &hr_dev->eq_table.eq[i]); 6622 6623 for (i = 0; i < irq_num; i++) 6624 kfree(hr_dev->irq_names[i]); 6625 } 6626 6627 static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev) 6628 { 6629 struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; 6630 struct device *dev = hr_dev->dev; 6631 struct hns_roce_eq *eq; 6632 int other_num; 6633 int comp_num; 6634 int aeq_num; 6635 int irq_num; 6636 int eq_num; 6637 u8 eq_cmd; 6638 int ret; 6639 int i; 6640 6641 other_num = hr_dev->caps.num_other_vectors; 6642 comp_num = hr_dev->caps.num_comp_vectors; 6643 aeq_num = hr_dev->caps.num_aeq_vectors; 6644 6645 eq_num = comp_num + aeq_num; 6646 irq_num = eq_num + other_num; 6647 6648 eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL); 6649 if (!eq_table->eq) 6650 return -ENOMEM; 6651 6652 /* create eq */ 6653 for (i = 0; i < eq_num; i++) { 6654 eq = &eq_table->eq[i]; 6655 eq->hr_dev = hr_dev; 6656 eq->eqn = i; 6657 if (i < comp_num) { 6658 /* CEQ */ 6659 eq_cmd = HNS_ROCE_CMD_CREATE_CEQC; 6660 eq->type_flag = HNS_ROCE_CEQ; 6661 eq->entries = hr_dev->caps.ceqe_depth; 6662 eq->eqe_size = hr_dev->caps.ceqe_size; 6663 eq->irq = hr_dev->irq[i + other_num + aeq_num]; 6664 eq->eq_max_cnt = HNS_ROCE_CEQ_DEFAULT_BURST_NUM; 6665 eq->eq_period = HNS_ROCE_CEQ_DEFAULT_INTERVAL; 6666 } else { 6667 /* AEQ */ 6668 eq_cmd = HNS_ROCE_CMD_CREATE_AEQC; 6669 eq->type_flag = HNS_ROCE_AEQ; 6670 eq->entries = hr_dev->caps.aeqe_depth; 6671 eq->eqe_size = hr_dev->caps.aeqe_size; 6672 eq->irq = hr_dev->irq[i - comp_num + other_num]; 6673 eq->eq_max_cnt = HNS_ROCE_AEQ_DEFAULT_BURST_NUM; 6674 eq->eq_period = HNS_ROCE_AEQ_DEFAULT_INTERVAL; 6675 } 6676 6677 ret = hns_roce_v2_create_eq(hr_dev, eq, eq_cmd); 6678 if (ret) { 6679 dev_err(dev, "failed to create eq.\n"); 6680 goto err_create_eq_fail; 6681 } 6682 } 6683 6684 INIT_WORK(&hr_dev->ecc_work, fmea_ram_ecc_work); 6685 6686 hr_dev->irq_workq = alloc_ordered_workqueue("hns_roce_irq_workq", 0); 6687 if (!hr_dev->irq_workq) { 6688 dev_err(dev, "failed to create irq workqueue.\n"); 6689 ret = -ENOMEM; 6690 goto err_create_eq_fail; 6691 } 6692 6693 ret = __hns_roce_request_irq(hr_dev, irq_num, comp_num, aeq_num, 6694 other_num); 6695 if (ret) { 6696 dev_err(dev, "failed to request irq.\n"); 6697 goto err_request_irq_fail; 6698 } 6699 6700 /* enable irq */ 6701 hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_ENABLE); 6702 6703 return 0; 6704 6705 err_request_irq_fail: 6706 destroy_workqueue(hr_dev->irq_workq); 6707 6708 err_create_eq_fail: 6709 for (i -= 1; i >= 0; i--) 6710 free_eq_buf(hr_dev, &eq_table->eq[i]); 6711 kfree(eq_table->eq); 6712 6713 return ret; 6714 } 6715 6716 static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev) 6717 { 6718 struct hns_roce_eq_table *eq_table = &hr_dev->eq_table; 6719 int eq_num; 6720 int i; 6721 6722 eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors; 6723 6724 /* Disable irq */ 6725 hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE); 6726 6727 __hns_roce_free_irq(hr_dev); 6728 destroy_workqueue(hr_dev->irq_workq); 6729 6730 for (i = 0; i < eq_num; i++) { 6731 hns_roce_v2_destroy_eqc(hr_dev, i); 6732 6733 free_eq_buf(hr_dev, &eq_table->eq[i]); 6734 } 6735 6736 kfree(eq_table->eq); 6737 } 6738 6739 static const struct ib_device_ops hns_roce_v2_dev_ops = { 6740 .destroy_qp = hns_roce_v2_destroy_qp, 6741 .modify_cq = hns_roce_v2_modify_cq, 6742 .poll_cq = hns_roce_v2_poll_cq, 6743 .post_recv = hns_roce_v2_post_recv, 6744 .post_send = hns_roce_v2_post_send, 6745 .query_qp = hns_roce_v2_query_qp, 6746 .req_notify_cq = hns_roce_v2_req_notify_cq, 6747 }; 6748 6749 static const struct ib_device_ops hns_roce_v2_dev_srq_ops = { 6750 .modify_srq = hns_roce_v2_modify_srq, 6751 .post_srq_recv = hns_roce_v2_post_srq_recv, 6752 .query_srq = hns_roce_v2_query_srq, 6753 }; 6754 6755 static const struct hns_roce_hw hns_roce_hw_v2 = { 6756 .cmq_init = hns_roce_v2_cmq_init, 6757 .cmq_exit = hns_roce_v2_cmq_exit, 6758 .hw_profile = hns_roce_v2_profile, 6759 .hw_init = hns_roce_v2_init, 6760 .hw_exit = hns_roce_v2_exit, 6761 .post_mbox = v2_post_mbox, 6762 .poll_mbox_done = v2_poll_mbox_done, 6763 .chk_mbox_avail = v2_chk_mbox_is_avail, 6764 .set_gid = hns_roce_v2_set_gid, 6765 .set_mac = hns_roce_v2_set_mac, 6766 .write_mtpt = hns_roce_v2_write_mtpt, 6767 .rereg_write_mtpt = hns_roce_v2_rereg_write_mtpt, 6768 .frmr_write_mtpt = hns_roce_v2_frmr_write_mtpt, 6769 .mw_write_mtpt = hns_roce_v2_mw_write_mtpt, 6770 .write_cqc = hns_roce_v2_write_cqc, 6771 .set_hem = hns_roce_v2_set_hem, 6772 .clear_hem = hns_roce_v2_clear_hem, 6773 .modify_qp = hns_roce_v2_modify_qp, 6774 .dereg_mr = hns_roce_v2_dereg_mr, 6775 .qp_flow_control_init = hns_roce_v2_qp_flow_control_init, 6776 .init_eq = hns_roce_v2_init_eq_table, 6777 .cleanup_eq = hns_roce_v2_cleanup_eq_table, 6778 .write_srqc = hns_roce_v2_write_srqc, 6779 .query_cqc = hns_roce_v2_query_cqc, 6780 .query_qpc = hns_roce_v2_query_qpc, 6781 .query_mpt = hns_roce_v2_query_mpt, 6782 .hns_roce_dev_ops = &hns_roce_v2_dev_ops, 6783 .hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops, 6784 }; 6785 6786 static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = { 6787 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 0}, 6788 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 0}, 6789 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0}, 6790 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0}, 6791 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0}, 6792 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA), 0}, 6793 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_RDMA_DCB_PFC_VF), 6794 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS}, 6795 /* required last entry */ 6796 {0, } 6797 }; 6798 6799 MODULE_DEVICE_TABLE(pci, hns_roce_hw_v2_pci_tbl); 6800 6801 static void hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev, 6802 struct hnae3_handle *handle) 6803 { 6804 struct hns_roce_v2_priv *priv = hr_dev->priv; 6805 const struct pci_device_id *id; 6806 int i; 6807 6808 hr_dev->pci_dev = handle->pdev; 6809 id = pci_match_id(hns_roce_hw_v2_pci_tbl, hr_dev->pci_dev); 6810 hr_dev->is_vf = id->driver_data; 6811 hr_dev->dev = &handle->pdev->dev; 6812 hr_dev->hw = &hns_roce_hw_v2; 6813 hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG; 6814 hr_dev->odb_offset = hr_dev->sdb_offset; 6815 6816 /* Get info from NIC driver. */ 6817 hr_dev->reg_base = handle->rinfo.roce_io_base; 6818 hr_dev->mem_base = handle->rinfo.roce_mem_base; 6819 hr_dev->caps.num_ports = 1; 6820 hr_dev->iboe.netdevs[0] = handle->rinfo.netdev; 6821 hr_dev->iboe.phy_port[0] = 0; 6822 6823 addrconf_addr_eui48((u8 *)&hr_dev->ib_dev.node_guid, 6824 hr_dev->iboe.netdevs[0]->dev_addr); 6825 6826 for (i = 0; i < handle->rinfo.num_vectors; i++) 6827 hr_dev->irq[i] = pci_irq_vector(handle->pdev, 6828 i + handle->rinfo.base_vector); 6829 6830 /* cmd issue mode: 0 is poll, 1 is event */ 6831 hr_dev->cmd_mod = 1; 6832 hr_dev->loop_idc = 0; 6833 6834 hr_dev->reset_cnt = handle->ae_algo->ops->ae_dev_reset_cnt(handle); 6835 priv->handle = handle; 6836 } 6837 6838 static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle) 6839 { 6840 struct hns_roce_dev *hr_dev; 6841 int ret; 6842 6843 hr_dev = ib_alloc_device(hns_roce_dev, ib_dev); 6844 if (!hr_dev) 6845 return -ENOMEM; 6846 6847 hr_dev->priv = kzalloc(sizeof(struct hns_roce_v2_priv), GFP_KERNEL); 6848 if (!hr_dev->priv) { 6849 ret = -ENOMEM; 6850 goto error_failed_kzalloc; 6851 } 6852 6853 hns_roce_hw_v2_get_cfg(hr_dev, handle); 6854 6855 ret = hns_roce_init(hr_dev); 6856 if (ret) { 6857 dev_err(hr_dev->dev, "RoCE Engine init failed!\n"); 6858 goto error_failed_cfg; 6859 } 6860 6861 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) { 6862 ret = free_mr_init(hr_dev); 6863 if (ret) { 6864 dev_err(hr_dev->dev, "failed to init free mr!\n"); 6865 goto error_failed_roce_init; 6866 } 6867 } 6868 6869 handle->priv = hr_dev; 6870 6871 return 0; 6872 6873 error_failed_roce_init: 6874 hns_roce_exit(hr_dev); 6875 6876 error_failed_cfg: 6877 kfree(hr_dev->priv); 6878 6879 error_failed_kzalloc: 6880 ib_dealloc_device(&hr_dev->ib_dev); 6881 6882 return ret; 6883 } 6884 6885 static void __hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle, 6886 bool reset) 6887 { 6888 struct hns_roce_dev *hr_dev = handle->priv; 6889 6890 if (!hr_dev) 6891 return; 6892 6893 handle->priv = NULL; 6894 6895 hr_dev->state = HNS_ROCE_DEVICE_STATE_UNINIT; 6896 hns_roce_handle_device_err(hr_dev); 6897 6898 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) 6899 free_mr_exit(hr_dev); 6900 6901 hns_roce_exit(hr_dev); 6902 kfree(hr_dev->priv); 6903 ib_dealloc_device(&hr_dev->ib_dev); 6904 } 6905 6906 static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle) 6907 { 6908 const struct hnae3_ae_ops *ops = handle->ae_algo->ops; 6909 const struct pci_device_id *id; 6910 struct device *dev = &handle->pdev->dev; 6911 int ret; 6912 6913 handle->rinfo.instance_state = HNS_ROCE_STATE_INIT; 6914 6915 if (ops->ae_dev_resetting(handle) || ops->get_hw_reset_stat(handle)) { 6916 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT; 6917 goto reset_chk_err; 6918 } 6919 6920 id = pci_match_id(hns_roce_hw_v2_pci_tbl, handle->pdev); 6921 if (!id) 6922 return 0; 6923 6924 if (id->driver_data && handle->pdev->revision == PCI_REVISION_ID_HIP08) 6925 return 0; 6926 6927 ret = __hns_roce_hw_v2_init_instance(handle); 6928 if (ret) { 6929 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT; 6930 dev_err(dev, "RoCE instance init failed! ret = %d\n", ret); 6931 if (ops->ae_dev_resetting(handle) || 6932 ops->get_hw_reset_stat(handle)) 6933 goto reset_chk_err; 6934 else 6935 return ret; 6936 } 6937 6938 handle->rinfo.instance_state = HNS_ROCE_STATE_INITED; 6939 6940 return 0; 6941 6942 reset_chk_err: 6943 dev_err(dev, "Device is busy in resetting state.\n" 6944 "please retry later.\n"); 6945 6946 return -EBUSY; 6947 } 6948 6949 static void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle, 6950 bool reset) 6951 { 6952 if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED) 6953 return; 6954 6955 handle->rinfo.instance_state = HNS_ROCE_STATE_UNINIT; 6956 6957 __hns_roce_hw_v2_uninit_instance(handle, reset); 6958 6959 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT; 6960 } 6961 static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle) 6962 { 6963 struct hns_roce_dev *hr_dev; 6964 6965 if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED) { 6966 set_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state); 6967 return 0; 6968 } 6969 6970 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_DOWN; 6971 clear_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state); 6972 6973 hr_dev = handle->priv; 6974 if (!hr_dev) 6975 return 0; 6976 6977 hr_dev->active = false; 6978 hr_dev->dis_db = true; 6979 hr_dev->state = HNS_ROCE_DEVICE_STATE_RST_DOWN; 6980 6981 return 0; 6982 } 6983 6984 static int hns_roce_hw_v2_reset_notify_init(struct hnae3_handle *handle) 6985 { 6986 struct device *dev = &handle->pdev->dev; 6987 int ret; 6988 6989 if (test_and_clear_bit(HNS_ROCE_RST_DIRECT_RETURN, 6990 &handle->rinfo.state)) { 6991 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED; 6992 return 0; 6993 } 6994 6995 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INIT; 6996 6997 dev_info(&handle->pdev->dev, "In reset process RoCE client reinit.\n"); 6998 ret = __hns_roce_hw_v2_init_instance(handle); 6999 if (ret) { 7000 /* when reset notify type is HNAE3_INIT_CLIENT In reset notify 7001 * callback function, RoCE Engine reinitialize. If RoCE reinit 7002 * failed, we should inform NIC driver. 7003 */ 7004 handle->priv = NULL; 7005 dev_err(dev, "In reset process RoCE reinit failed %d.\n", ret); 7006 } else { 7007 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED; 7008 dev_info(dev, "reset done, RoCE client reinit finished.\n"); 7009 } 7010 7011 return ret; 7012 } 7013 7014 static int hns_roce_hw_v2_reset_notify_uninit(struct hnae3_handle *handle) 7015 { 7016 if (test_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state)) 7017 return 0; 7018 7019 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_UNINIT; 7020 dev_info(&handle->pdev->dev, "In reset process RoCE client uninit.\n"); 7021 msleep(HNS_ROCE_V2_HW_RST_UNINT_DELAY); 7022 __hns_roce_hw_v2_uninit_instance(handle, false); 7023 7024 return 0; 7025 } 7026 7027 static int hns_roce_hw_v2_reset_notify(struct hnae3_handle *handle, 7028 enum hnae3_reset_notify_type type) 7029 { 7030 int ret = 0; 7031 7032 switch (type) { 7033 case HNAE3_DOWN_CLIENT: 7034 ret = hns_roce_hw_v2_reset_notify_down(handle); 7035 break; 7036 case HNAE3_INIT_CLIENT: 7037 ret = hns_roce_hw_v2_reset_notify_init(handle); 7038 break; 7039 case HNAE3_UNINIT_CLIENT: 7040 ret = hns_roce_hw_v2_reset_notify_uninit(handle); 7041 break; 7042 default: 7043 break; 7044 } 7045 7046 return ret; 7047 } 7048 7049 static const struct hnae3_client_ops hns_roce_hw_v2_ops = { 7050 .init_instance = hns_roce_hw_v2_init_instance, 7051 .uninit_instance = hns_roce_hw_v2_uninit_instance, 7052 .reset_notify = hns_roce_hw_v2_reset_notify, 7053 }; 7054 7055 static struct hnae3_client hns_roce_hw_v2_client = { 7056 .name = "hns_roce_hw_v2", 7057 .type = HNAE3_CLIENT_ROCE, 7058 .ops = &hns_roce_hw_v2_ops, 7059 }; 7060 7061 static int __init hns_roce_hw_v2_init(void) 7062 { 7063 return hnae3_register_client(&hns_roce_hw_v2_client); 7064 } 7065 7066 static void __exit hns_roce_hw_v2_exit(void) 7067 { 7068 hnae3_unregister_client(&hns_roce_hw_v2_client); 7069 } 7070 7071 module_init(hns_roce_hw_v2_init); 7072 module_exit(hns_roce_hw_v2_exit); 7073 7074 MODULE_LICENSE("Dual BSD/GPL"); 7075 MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>"); 7076 MODULE_AUTHOR("Lijun Ou <oulijun@huawei.com>"); 7077 MODULE_AUTHOR("Shaobo Xu <xushaobo2@huawei.com>"); 7078 MODULE_DESCRIPTION("Hisilicon Hip08 Family RoCE Driver"); 7079