1 /*
2 * Copyright (c) 2016-2017 Hisilicon Limited.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <linux/acpi.h>
34 #include <linux/etherdevice.h>
35 #include <linux/interrupt.h>
36 #include <linux/iopoll.h>
37 #include <linux/kernel.h>
38 #include <linux/types.h>
39 #include <linux/workqueue.h>
40 #include <net/addrconf.h>
41 #include <rdma/ib_addr.h>
42 #include <rdma/ib_cache.h>
43 #include <rdma/ib_umem.h>
44 #include <rdma/uverbs_ioctl.h>
45
46 #include "hnae3.h"
47 #include "hns_roce_common.h"
48 #include "hns_roce_device.h"
49 #include "hns_roce_cmd.h"
50 #include "hns_roce_hem.h"
51 #include "hns_roce_hw_v2.h"
52
53 enum {
54 CMD_RST_PRC_OTHERS,
55 CMD_RST_PRC_SUCCESS,
56 CMD_RST_PRC_EBUSY,
57 };
58
59 enum ecc_resource_type {
60 ECC_RESOURCE_QPC,
61 ECC_RESOURCE_CQC,
62 ECC_RESOURCE_MPT,
63 ECC_RESOURCE_SRQC,
64 ECC_RESOURCE_GMV,
65 ECC_RESOURCE_QPC_TIMER,
66 ECC_RESOURCE_CQC_TIMER,
67 ECC_RESOURCE_SCCC,
68 ECC_RESOURCE_COUNT,
69 };
70
71 static const struct {
72 const char *name;
73 u8 read_bt0_op;
74 u8 write_bt0_op;
75 } fmea_ram_res[] = {
76 { "ECC_RESOURCE_QPC",
77 HNS_ROCE_CMD_READ_QPC_BT0, HNS_ROCE_CMD_WRITE_QPC_BT0 },
78 { "ECC_RESOURCE_CQC",
79 HNS_ROCE_CMD_READ_CQC_BT0, HNS_ROCE_CMD_WRITE_CQC_BT0 },
80 { "ECC_RESOURCE_MPT",
81 HNS_ROCE_CMD_READ_MPT_BT0, HNS_ROCE_CMD_WRITE_MPT_BT0 },
82 { "ECC_RESOURCE_SRQC",
83 HNS_ROCE_CMD_READ_SRQC_BT0, HNS_ROCE_CMD_WRITE_SRQC_BT0 },
84 /* ECC_RESOURCE_GMV is handled by cmdq, not mailbox */
85 { "ECC_RESOURCE_GMV",
86 0, 0 },
87 { "ECC_RESOURCE_QPC_TIMER",
88 HNS_ROCE_CMD_READ_QPC_TIMER_BT0, HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0 },
89 { "ECC_RESOURCE_CQC_TIMER",
90 HNS_ROCE_CMD_READ_CQC_TIMER_BT0, HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0 },
91 { "ECC_RESOURCE_SCCC",
92 HNS_ROCE_CMD_READ_SCCC_BT0, HNS_ROCE_CMD_WRITE_SCCC_BT0 },
93 };
94
set_data_seg_v2(struct hns_roce_v2_wqe_data_seg * dseg,struct ib_sge * sg)95 static inline void set_data_seg_v2(struct hns_roce_v2_wqe_data_seg *dseg,
96 struct ib_sge *sg)
97 {
98 dseg->lkey = cpu_to_le32(sg->lkey);
99 dseg->addr = cpu_to_le64(sg->addr);
100 dseg->len = cpu_to_le32(sg->length);
101 }
102
103 /*
104 * mapped-value = 1 + real-value
105 * The hns wr opcode real value is start from 0, In order to distinguish between
106 * initialized and uninitialized map values, we plus 1 to the actual value when
107 * defining the mapping, so that the validity can be identified by checking the
108 * mapped value is greater than 0.
109 */
110 #define HR_OPC_MAP(ib_key, hr_key) \
111 [IB_WR_ ## ib_key] = 1 + HNS_ROCE_V2_WQE_OP_ ## hr_key
112
113 static const u32 hns_roce_op_code[] = {
114 HR_OPC_MAP(RDMA_WRITE, RDMA_WRITE),
115 HR_OPC_MAP(RDMA_WRITE_WITH_IMM, RDMA_WRITE_WITH_IMM),
116 HR_OPC_MAP(SEND, SEND),
117 HR_OPC_MAP(SEND_WITH_IMM, SEND_WITH_IMM),
118 HR_OPC_MAP(RDMA_READ, RDMA_READ),
119 HR_OPC_MAP(ATOMIC_CMP_AND_SWP, ATOM_CMP_AND_SWAP),
120 HR_OPC_MAP(ATOMIC_FETCH_AND_ADD, ATOM_FETCH_AND_ADD),
121 HR_OPC_MAP(SEND_WITH_INV, SEND_WITH_INV),
122 HR_OPC_MAP(MASKED_ATOMIC_CMP_AND_SWP, ATOM_MSK_CMP_AND_SWAP),
123 HR_OPC_MAP(MASKED_ATOMIC_FETCH_AND_ADD, ATOM_MSK_FETCH_AND_ADD),
124 HR_OPC_MAP(REG_MR, FAST_REG_PMR),
125 };
126
to_hr_opcode(u32 ib_opcode)127 static u32 to_hr_opcode(u32 ib_opcode)
128 {
129 if (ib_opcode >= ARRAY_SIZE(hns_roce_op_code))
130 return HNS_ROCE_V2_WQE_OP_MASK;
131
132 return hns_roce_op_code[ib_opcode] ? hns_roce_op_code[ib_opcode] - 1 :
133 HNS_ROCE_V2_WQE_OP_MASK;
134 }
135
set_frmr_seg(struct hns_roce_v2_rc_send_wqe * rc_sq_wqe,const struct ib_reg_wr * wr)136 static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
137 const struct ib_reg_wr *wr)
138 {
139 struct hns_roce_wqe_frmr_seg *fseg =
140 (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);
141 struct hns_roce_mr *mr = to_hr_mr(wr->mr);
142 u64 pbl_ba;
143
144 /* use ib_access_flags */
145 hr_reg_write_bool(fseg, FRMR_BIND_EN, wr->access & IB_ACCESS_MW_BIND);
146 hr_reg_write_bool(fseg, FRMR_ATOMIC,
147 wr->access & IB_ACCESS_REMOTE_ATOMIC);
148 hr_reg_write_bool(fseg, FRMR_RR, wr->access & IB_ACCESS_REMOTE_READ);
149 hr_reg_write_bool(fseg, FRMR_RW, wr->access & IB_ACCESS_REMOTE_WRITE);
150 hr_reg_write_bool(fseg, FRMR_LW, wr->access & IB_ACCESS_LOCAL_WRITE);
151
152 /* Data structure reuse may lead to confusion */
153 pbl_ba = mr->pbl_mtr.hem_cfg.root_ba;
154 rc_sq_wqe->msg_len = cpu_to_le32(lower_32_bits(pbl_ba));
155 rc_sq_wqe->inv_key = cpu_to_le32(upper_32_bits(pbl_ba));
156
157 rc_sq_wqe->byte_16 = cpu_to_le32(wr->mr->length & 0xffffffff);
158 rc_sq_wqe->byte_20 = cpu_to_le32(wr->mr->length >> 32);
159 rc_sq_wqe->rkey = cpu_to_le32(wr->key);
160 rc_sq_wqe->va = cpu_to_le64(wr->mr->iova);
161
162 hr_reg_write(fseg, FRMR_PBL_SIZE, mr->npages);
163 hr_reg_write(fseg, FRMR_PBL_BUF_PG_SZ,
164 to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift));
165 hr_reg_clear(fseg, FRMR_BLK_MODE);
166 }
167
set_atomic_seg(const struct ib_send_wr * wr,struct hns_roce_v2_rc_send_wqe * rc_sq_wqe,unsigned int valid_num_sge)168 static void set_atomic_seg(const struct ib_send_wr *wr,
169 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
170 unsigned int valid_num_sge)
171 {
172 struct hns_roce_v2_wqe_data_seg *dseg =
173 (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);
174 struct hns_roce_wqe_atomic_seg *aseg =
175 (void *)dseg + sizeof(struct hns_roce_v2_wqe_data_seg);
176
177 set_data_seg_v2(dseg, wr->sg_list);
178
179 if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
180 aseg->fetchadd_swap_data = cpu_to_le64(atomic_wr(wr)->swap);
181 aseg->cmp_data = cpu_to_le64(atomic_wr(wr)->compare_add);
182 } else {
183 aseg->fetchadd_swap_data =
184 cpu_to_le64(atomic_wr(wr)->compare_add);
185 aseg->cmp_data = 0;
186 }
187
188 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_SGE_NUM, valid_num_sge);
189 }
190
fill_ext_sge_inl_data(struct hns_roce_qp * qp,const struct ib_send_wr * wr,unsigned int * sge_idx,u32 msg_len)191 static int fill_ext_sge_inl_data(struct hns_roce_qp *qp,
192 const struct ib_send_wr *wr,
193 unsigned int *sge_idx, u32 msg_len)
194 {
195 struct ib_device *ibdev = &(to_hr_dev(qp->ibqp.device))->ib_dev;
196 unsigned int left_len_in_pg;
197 unsigned int idx = *sge_idx;
198 unsigned int i = 0;
199 unsigned int len;
200 void *addr;
201 void *dseg;
202
203 if (msg_len > qp->sq.ext_sge_cnt * HNS_ROCE_SGE_SIZE) {
204 ibdev_err(ibdev,
205 "no enough extended sge space for inline data.\n");
206 return -EINVAL;
207 }
208
209 dseg = hns_roce_get_extend_sge(qp, idx & (qp->sge.sge_cnt - 1));
210 left_len_in_pg = hr_hw_page_align((uintptr_t)dseg) - (uintptr_t)dseg;
211 len = wr->sg_list[0].length;
212 addr = (void *)(unsigned long)(wr->sg_list[0].addr);
213
214 /* When copying data to extended sge space, the left length in page may
215 * not long enough for current user's sge. So the data should be
216 * splited into several parts, one in the first page, and the others in
217 * the subsequent pages.
218 */
219 while (1) {
220 if (len <= left_len_in_pg) {
221 memcpy(dseg, addr, len);
222
223 idx += len / HNS_ROCE_SGE_SIZE;
224
225 i++;
226 if (i >= wr->num_sge)
227 break;
228
229 left_len_in_pg -= len;
230 len = wr->sg_list[i].length;
231 addr = (void *)(unsigned long)(wr->sg_list[i].addr);
232 dseg += len;
233 } else {
234 memcpy(dseg, addr, left_len_in_pg);
235
236 len -= left_len_in_pg;
237 addr += left_len_in_pg;
238 idx += left_len_in_pg / HNS_ROCE_SGE_SIZE;
239 dseg = hns_roce_get_extend_sge(qp,
240 idx & (qp->sge.sge_cnt - 1));
241 left_len_in_pg = 1 << HNS_HW_PAGE_SHIFT;
242 }
243 }
244
245 *sge_idx = idx;
246
247 return 0;
248 }
249
set_extend_sge(struct hns_roce_qp * qp,struct ib_sge * sge,unsigned int * sge_ind,unsigned int cnt)250 static void set_extend_sge(struct hns_roce_qp *qp, struct ib_sge *sge,
251 unsigned int *sge_ind, unsigned int cnt)
252 {
253 struct hns_roce_v2_wqe_data_seg *dseg;
254 unsigned int idx = *sge_ind;
255
256 while (cnt > 0) {
257 dseg = hns_roce_get_extend_sge(qp, idx & (qp->sge.sge_cnt - 1));
258 if (likely(sge->length)) {
259 set_data_seg_v2(dseg, sge);
260 idx++;
261 cnt--;
262 }
263 sge++;
264 }
265
266 *sge_ind = idx;
267 }
268
check_inl_data_len(struct hns_roce_qp * qp,unsigned int len)269 static bool check_inl_data_len(struct hns_roce_qp *qp, unsigned int len)
270 {
271 struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device);
272 int mtu = ib_mtu_enum_to_int(qp->path_mtu);
273
274 if (mtu < 0 || len > qp->max_inline_data || len > mtu) {
275 ibdev_err(&hr_dev->ib_dev,
276 "invalid length of data, data len = %u, max inline len = %u, path mtu = %d.\n",
277 len, qp->max_inline_data, mtu);
278 return false;
279 }
280
281 return true;
282 }
283
set_rc_inl(struct hns_roce_qp * qp,const struct ib_send_wr * wr,struct hns_roce_v2_rc_send_wqe * rc_sq_wqe,unsigned int * sge_idx)284 static int set_rc_inl(struct hns_roce_qp *qp, const struct ib_send_wr *wr,
285 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
286 unsigned int *sge_idx)
287 {
288 struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device);
289 u32 msg_len = le32_to_cpu(rc_sq_wqe->msg_len);
290 struct ib_device *ibdev = &hr_dev->ib_dev;
291 unsigned int curr_idx = *sge_idx;
292 void *dseg = rc_sq_wqe;
293 unsigned int i;
294 int ret;
295
296 if (unlikely(wr->opcode == IB_WR_RDMA_READ)) {
297 ibdev_err(ibdev, "invalid inline parameters!\n");
298 return -EINVAL;
299 }
300
301 if (!check_inl_data_len(qp, msg_len))
302 return -EINVAL;
303
304 dseg += sizeof(struct hns_roce_v2_rc_send_wqe);
305
306 if (msg_len <= HNS_ROCE_V2_MAX_RC_INL_INN_SZ) {
307 hr_reg_clear(rc_sq_wqe, RC_SEND_WQE_INL_TYPE);
308
309 for (i = 0; i < wr->num_sge; i++) {
310 memcpy(dseg, ((void *)wr->sg_list[i].addr),
311 wr->sg_list[i].length);
312 dseg += wr->sg_list[i].length;
313 }
314 } else {
315 hr_reg_enable(rc_sq_wqe, RC_SEND_WQE_INL_TYPE);
316
317 ret = fill_ext_sge_inl_data(qp, wr, &curr_idx, msg_len);
318 if (ret)
319 return ret;
320
321 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_SGE_NUM, curr_idx - *sge_idx);
322 }
323
324 *sge_idx = curr_idx;
325
326 return 0;
327 }
328
set_rwqe_data_seg(struct ib_qp * ibqp,const struct ib_send_wr * wr,struct hns_roce_v2_rc_send_wqe * rc_sq_wqe,unsigned int * sge_ind,unsigned int valid_num_sge)329 static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr,
330 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
331 unsigned int *sge_ind,
332 unsigned int valid_num_sge)
333 {
334 struct hns_roce_v2_wqe_data_seg *dseg =
335 (void *)rc_sq_wqe + sizeof(struct hns_roce_v2_rc_send_wqe);
336 struct hns_roce_qp *qp = to_hr_qp(ibqp);
337 int j = 0;
338 int i;
339
340 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_MSG_START_SGE_IDX,
341 (*sge_ind) & (qp->sge.sge_cnt - 1));
342
343 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_INLINE,
344 !!(wr->send_flags & IB_SEND_INLINE));
345 if (wr->send_flags & IB_SEND_INLINE)
346 return set_rc_inl(qp, wr, rc_sq_wqe, sge_ind);
347
348 if (valid_num_sge <= HNS_ROCE_SGE_IN_WQE) {
349 for (i = 0; i < wr->num_sge; i++) {
350 if (likely(wr->sg_list[i].length)) {
351 set_data_seg_v2(dseg, wr->sg_list + i);
352 dseg++;
353 }
354 }
355 } else {
356 for (i = 0; i < wr->num_sge && j < HNS_ROCE_SGE_IN_WQE; i++) {
357 if (likely(wr->sg_list[i].length)) {
358 set_data_seg_v2(dseg, wr->sg_list + i);
359 dseg++;
360 j++;
361 }
362 }
363
364 set_extend_sge(qp, wr->sg_list + i, sge_ind,
365 valid_num_sge - HNS_ROCE_SGE_IN_WQE);
366 }
367
368 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_SGE_NUM, valid_num_sge);
369
370 return 0;
371 }
372
check_send_valid(struct hns_roce_dev * hr_dev,struct hns_roce_qp * hr_qp)373 static int check_send_valid(struct hns_roce_dev *hr_dev,
374 struct hns_roce_qp *hr_qp)
375 {
376 if (unlikely(hr_qp->state == IB_QPS_RESET ||
377 hr_qp->state == IB_QPS_INIT ||
378 hr_qp->state == IB_QPS_RTR))
379 return -EINVAL;
380 else if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN))
381 return -EIO;
382
383 return 0;
384 }
385
calc_wr_sge_num(const struct ib_send_wr * wr,unsigned int * sge_len)386 static unsigned int calc_wr_sge_num(const struct ib_send_wr *wr,
387 unsigned int *sge_len)
388 {
389 unsigned int valid_num = 0;
390 unsigned int len = 0;
391 int i;
392
393 for (i = 0; i < wr->num_sge; i++) {
394 if (likely(wr->sg_list[i].length)) {
395 len += wr->sg_list[i].length;
396 valid_num++;
397 }
398 }
399
400 *sge_len = len;
401 return valid_num;
402 }
403
get_immtdata(const struct ib_send_wr * wr)404 static __le32 get_immtdata(const struct ib_send_wr *wr)
405 {
406 switch (wr->opcode) {
407 case IB_WR_SEND_WITH_IMM:
408 case IB_WR_RDMA_WRITE_WITH_IMM:
409 return cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
410 default:
411 return 0;
412 }
413 }
414
set_ud_opcode(struct hns_roce_v2_ud_send_wqe * ud_sq_wqe,const struct ib_send_wr * wr)415 static int set_ud_opcode(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,
416 const struct ib_send_wr *wr)
417 {
418 u32 ib_op = wr->opcode;
419
420 if (ib_op != IB_WR_SEND && ib_op != IB_WR_SEND_WITH_IMM)
421 return -EINVAL;
422
423 ud_sq_wqe->immtdata = get_immtdata(wr);
424
425 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_OPCODE, to_hr_opcode(ib_op));
426
427 return 0;
428 }
429
fill_ud_av(struct hns_roce_v2_ud_send_wqe * ud_sq_wqe,struct hns_roce_ah * ah)430 static int fill_ud_av(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,
431 struct hns_roce_ah *ah)
432 {
433 struct ib_device *ib_dev = ah->ibah.device;
434 struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);
435
436 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_UDPSPN, ah->av.udp_sport);
437 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_HOPLIMIT, ah->av.hop_limit);
438 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_TCLASS, ah->av.tclass);
439 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_FLOW_LABEL, ah->av.flowlabel);
440 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SL, ah->av.sl);
441
442 ud_sq_wqe->sgid_index = ah->av.gid_index;
443
444 memcpy(ud_sq_wqe->dmac, ah->av.mac, ETH_ALEN);
445 memcpy(ud_sq_wqe->dgid, ah->av.dgid, GID_LEN_V2);
446
447 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
448 return 0;
449
450 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_VLAN_EN, ah->av.vlan_en);
451 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_VLAN, ah->av.vlan_id);
452
453 return 0;
454 }
455
set_ud_wqe(struct hns_roce_qp * qp,const struct ib_send_wr * wr,void * wqe,unsigned int * sge_idx,unsigned int owner_bit)456 static inline int set_ud_wqe(struct hns_roce_qp *qp,
457 const struct ib_send_wr *wr,
458 void *wqe, unsigned int *sge_idx,
459 unsigned int owner_bit)
460 {
461 struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
462 struct hns_roce_v2_ud_send_wqe *ud_sq_wqe = wqe;
463 unsigned int curr_idx = *sge_idx;
464 unsigned int valid_num_sge;
465 u32 msg_len = 0;
466 int ret;
467
468 valid_num_sge = calc_wr_sge_num(wr, &msg_len);
469
470 ret = set_ud_opcode(ud_sq_wqe, wr);
471 if (WARN_ON_ONCE(ret))
472 return ret;
473
474 ud_sq_wqe->msg_len = cpu_to_le32(msg_len);
475
476 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_CQE,
477 !!(wr->send_flags & IB_SEND_SIGNALED));
478 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SE,
479 !!(wr->send_flags & IB_SEND_SOLICITED));
480
481 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_PD, to_hr_pd(qp->ibqp.pd)->pdn);
482 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SGE_NUM, valid_num_sge);
483 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_MSG_START_SGE_IDX,
484 curr_idx & (qp->sge.sge_cnt - 1));
485
486 ud_sq_wqe->qkey = cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ?
487 qp->qkey : ud_wr(wr)->remote_qkey);
488 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_DQPN, ud_wr(wr)->remote_qpn);
489
490 ret = fill_ud_av(ud_sq_wqe, ah);
491 if (ret)
492 return ret;
493
494 qp->sl = to_hr_ah(ud_wr(wr)->ah)->av.sl;
495
496 set_extend_sge(qp, wr->sg_list, &curr_idx, valid_num_sge);
497
498 /*
499 * The pipeline can sequentially post all valid WQEs into WQ buffer,
500 * including new WQEs waiting for the doorbell to update the PI again.
501 * Therefore, the owner bit of WQE MUST be updated after all fields
502 * and extSGEs have been written into DDR instead of cache.
503 */
504 if (qp->en_flags & HNS_ROCE_QP_CAP_OWNER_DB)
505 dma_wmb();
506
507 *sge_idx = curr_idx;
508 hr_reg_write(ud_sq_wqe, UD_SEND_WQE_OWNER, owner_bit);
509
510 return 0;
511 }
512
set_rc_opcode(struct hns_roce_dev * hr_dev,struct hns_roce_v2_rc_send_wqe * rc_sq_wqe,const struct ib_send_wr * wr)513 static int set_rc_opcode(struct hns_roce_dev *hr_dev,
514 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
515 const struct ib_send_wr *wr)
516 {
517 u32 ib_op = wr->opcode;
518 int ret = 0;
519
520 rc_sq_wqe->immtdata = get_immtdata(wr);
521
522 switch (ib_op) {
523 case IB_WR_RDMA_READ:
524 case IB_WR_RDMA_WRITE:
525 case IB_WR_RDMA_WRITE_WITH_IMM:
526 rc_sq_wqe->rkey = cpu_to_le32(rdma_wr(wr)->rkey);
527 rc_sq_wqe->va = cpu_to_le64(rdma_wr(wr)->remote_addr);
528 break;
529 case IB_WR_SEND:
530 case IB_WR_SEND_WITH_IMM:
531 break;
532 case IB_WR_ATOMIC_CMP_AND_SWP:
533 case IB_WR_ATOMIC_FETCH_AND_ADD:
534 rc_sq_wqe->rkey = cpu_to_le32(atomic_wr(wr)->rkey);
535 rc_sq_wqe->va = cpu_to_le64(atomic_wr(wr)->remote_addr);
536 break;
537 case IB_WR_REG_MR:
538 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
539 set_frmr_seg(rc_sq_wqe, reg_wr(wr));
540 else
541 ret = -EOPNOTSUPP;
542 break;
543 case IB_WR_SEND_WITH_INV:
544 rc_sq_wqe->inv_key = cpu_to_le32(wr->ex.invalidate_rkey);
545 break;
546 default:
547 ret = -EINVAL;
548 }
549
550 if (unlikely(ret))
551 return ret;
552
553 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_OPCODE, to_hr_opcode(ib_op));
554
555 return ret;
556 }
557
set_rc_wqe(struct hns_roce_qp * qp,const struct ib_send_wr * wr,void * wqe,unsigned int * sge_idx,unsigned int owner_bit)558 static inline int set_rc_wqe(struct hns_roce_qp *qp,
559 const struct ib_send_wr *wr,
560 void *wqe, unsigned int *sge_idx,
561 unsigned int owner_bit)
562 {
563 struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device);
564 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe = wqe;
565 unsigned int curr_idx = *sge_idx;
566 unsigned int valid_num_sge;
567 u32 msg_len = 0;
568 int ret;
569
570 valid_num_sge = calc_wr_sge_num(wr, &msg_len);
571
572 rc_sq_wqe->msg_len = cpu_to_le32(msg_len);
573
574 ret = set_rc_opcode(hr_dev, rc_sq_wqe, wr);
575 if (WARN_ON_ONCE(ret))
576 return ret;
577
578 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_SO,
579 (wr->send_flags & IB_SEND_FENCE) ? 1 : 0);
580
581 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_SE,
582 (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
583
584 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_CQE,
585 (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
586
587 if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
588 wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
589 if (msg_len != ATOMIC_WR_LEN)
590 return -EINVAL;
591 set_atomic_seg(wr, rc_sq_wqe, valid_num_sge);
592 } else if (wr->opcode != IB_WR_REG_MR) {
593 ret = set_rwqe_data_seg(&qp->ibqp, wr, rc_sq_wqe,
594 &curr_idx, valid_num_sge);
595 if (ret)
596 return ret;
597 }
598
599 /*
600 * The pipeline can sequentially post all valid WQEs into WQ buffer,
601 * including new WQEs waiting for the doorbell to update the PI again.
602 * Therefore, the owner bit of WQE MUST be updated after all fields
603 * and extSGEs have been written into DDR instead of cache.
604 */
605 if (qp->en_flags & HNS_ROCE_QP_CAP_OWNER_DB)
606 dma_wmb();
607
608 *sge_idx = curr_idx;
609 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_OWNER, owner_bit);
610
611 return ret;
612 }
613
update_sq_db(struct hns_roce_dev * hr_dev,struct hns_roce_qp * qp)614 static inline void update_sq_db(struct hns_roce_dev *hr_dev,
615 struct hns_roce_qp *qp)
616 {
617 if (unlikely(qp->state == IB_QPS_ERR)) {
618 flush_cqe(hr_dev, qp);
619 } else {
620 struct hns_roce_v2_db sq_db = {};
621
622 hr_reg_write(&sq_db, DB_TAG, qp->qpn);
623 hr_reg_write(&sq_db, DB_CMD, HNS_ROCE_V2_SQ_DB);
624 hr_reg_write(&sq_db, DB_PI, qp->sq.head);
625 hr_reg_write(&sq_db, DB_SL, qp->sl);
626
627 hns_roce_write64(hr_dev, (__le32 *)&sq_db, qp->sq.db_reg);
628 }
629 }
630
update_rq_db(struct hns_roce_dev * hr_dev,struct hns_roce_qp * qp)631 static inline void update_rq_db(struct hns_roce_dev *hr_dev,
632 struct hns_roce_qp *qp)
633 {
634 if (unlikely(qp->state == IB_QPS_ERR)) {
635 flush_cqe(hr_dev, qp);
636 } else {
637 if (likely(qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)) {
638 *qp->rdb.db_record =
639 qp->rq.head & V2_DB_PRODUCER_IDX_M;
640 } else {
641 struct hns_roce_v2_db rq_db = {};
642
643 hr_reg_write(&rq_db, DB_TAG, qp->qpn);
644 hr_reg_write(&rq_db, DB_CMD, HNS_ROCE_V2_RQ_DB);
645 hr_reg_write(&rq_db, DB_PI, qp->rq.head);
646
647 hns_roce_write64(hr_dev, (__le32 *)&rq_db,
648 qp->rq.db_reg);
649 }
650 }
651 }
652
hns_roce_write512(struct hns_roce_dev * hr_dev,u64 * val,u64 __iomem * dest)653 static void hns_roce_write512(struct hns_roce_dev *hr_dev, u64 *val,
654 u64 __iomem *dest)
655 {
656 #define HNS_ROCE_WRITE_TIMES 8
657 struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
658 struct hnae3_handle *handle = priv->handle;
659 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
660 int i;
661
662 if (!hr_dev->dis_db && !ops->get_hw_reset_stat(handle))
663 for (i = 0; i < HNS_ROCE_WRITE_TIMES; i++)
664 writeq_relaxed(*(val + i), dest + i);
665 }
666
write_dwqe(struct hns_roce_dev * hr_dev,struct hns_roce_qp * qp,void * wqe)667 static void write_dwqe(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp,
668 void *wqe)
669 {
670 #define HNS_ROCE_SL_SHIFT 2
671 struct hns_roce_v2_rc_send_wqe *rc_sq_wqe = wqe;
672
673 if (unlikely(qp->state == IB_QPS_ERR)) {
674 flush_cqe(hr_dev, qp);
675 return;
676 }
677 /* All kinds of DirectWQE have the same header field layout */
678 hr_reg_enable(rc_sq_wqe, RC_SEND_WQE_FLAG);
679 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_DB_SL_L, qp->sl);
680 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_DB_SL_H,
681 qp->sl >> HNS_ROCE_SL_SHIFT);
682 hr_reg_write(rc_sq_wqe, RC_SEND_WQE_WQE_INDEX, qp->sq.head);
683
684 hns_roce_write512(hr_dev, wqe, qp->sq.db_reg);
685 }
686
hns_roce_v2_post_send(struct ib_qp * ibqp,const struct ib_send_wr * wr,const struct ib_send_wr ** bad_wr)687 static int hns_roce_v2_post_send(struct ib_qp *ibqp,
688 const struct ib_send_wr *wr,
689 const struct ib_send_wr **bad_wr)
690 {
691 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
692 struct ib_device *ibdev = &hr_dev->ib_dev;
693 struct hns_roce_qp *qp = to_hr_qp(ibqp);
694 unsigned long flags = 0;
695 unsigned int owner_bit;
696 unsigned int sge_idx;
697 unsigned int wqe_idx;
698 void *wqe = NULL;
699 u32 nreq;
700 int ret;
701
702 spin_lock_irqsave(&qp->sq.lock, flags);
703
704 ret = check_send_valid(hr_dev, qp);
705 if (unlikely(ret)) {
706 *bad_wr = wr;
707 nreq = 0;
708 goto out;
709 }
710
711 sge_idx = qp->next_sge;
712
713 for (nreq = 0; wr; ++nreq, wr = wr->next) {
714 if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
715 ret = -ENOMEM;
716 *bad_wr = wr;
717 goto out;
718 }
719
720 wqe_idx = (qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1);
721
722 if (unlikely(wr->num_sge > qp->sq.max_gs)) {
723 ibdev_err(ibdev, "num_sge = %d > qp->sq.max_gs = %u.\n",
724 wr->num_sge, qp->sq.max_gs);
725 ret = -EINVAL;
726 *bad_wr = wr;
727 goto out;
728 }
729
730 wqe = hns_roce_get_send_wqe(qp, wqe_idx);
731 qp->sq.wrid[wqe_idx] = wr->wr_id;
732 owner_bit =
733 ~(((qp->sq.head + nreq) >> ilog2(qp->sq.wqe_cnt)) & 0x1);
734
735 /* Corresponding to the QP type, wqe process separately */
736 if (ibqp->qp_type == IB_QPT_RC)
737 ret = set_rc_wqe(qp, wr, wqe, &sge_idx, owner_bit);
738 else
739 ret = set_ud_wqe(qp, wr, wqe, &sge_idx, owner_bit);
740
741 if (unlikely(ret)) {
742 *bad_wr = wr;
743 goto out;
744 }
745 }
746
747 out:
748 if (likely(nreq)) {
749 qp->sq.head += nreq;
750 qp->next_sge = sge_idx;
751
752 if (nreq == 1 && !ret &&
753 (qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE))
754 write_dwqe(hr_dev, qp, wqe);
755 else
756 update_sq_db(hr_dev, qp);
757 }
758
759 spin_unlock_irqrestore(&qp->sq.lock, flags);
760
761 return ret;
762 }
763
check_recv_valid(struct hns_roce_dev * hr_dev,struct hns_roce_qp * hr_qp)764 static int check_recv_valid(struct hns_roce_dev *hr_dev,
765 struct hns_roce_qp *hr_qp)
766 {
767 if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN))
768 return -EIO;
769
770 if (hr_qp->state == IB_QPS_RESET)
771 return -EINVAL;
772
773 return 0;
774 }
775
fill_recv_sge_to_wqe(const struct ib_recv_wr * wr,void * wqe,u32 max_sge,bool rsv)776 static void fill_recv_sge_to_wqe(const struct ib_recv_wr *wr, void *wqe,
777 u32 max_sge, bool rsv)
778 {
779 struct hns_roce_v2_wqe_data_seg *dseg = wqe;
780 u32 i, cnt;
781
782 for (i = 0, cnt = 0; i < wr->num_sge; i++) {
783 /* Skip zero-length sge */
784 if (!wr->sg_list[i].length)
785 continue;
786 set_data_seg_v2(dseg + cnt, wr->sg_list + i);
787 cnt++;
788 }
789
790 /* Fill a reserved sge to make hw stop reading remaining segments */
791 if (rsv) {
792 dseg[cnt].lkey = cpu_to_le32(HNS_ROCE_INVALID_LKEY);
793 dseg[cnt].addr = 0;
794 dseg[cnt].len = cpu_to_le32(HNS_ROCE_INVALID_SGE_LENGTH);
795 } else {
796 /* Clear remaining segments to make ROCEE ignore sges */
797 if (cnt < max_sge)
798 memset(dseg + cnt, 0,
799 (max_sge - cnt) * HNS_ROCE_SGE_SIZE);
800 }
801 }
802
fill_rq_wqe(struct hns_roce_qp * hr_qp,const struct ib_recv_wr * wr,u32 wqe_idx,u32 max_sge)803 static void fill_rq_wqe(struct hns_roce_qp *hr_qp, const struct ib_recv_wr *wr,
804 u32 wqe_idx, u32 max_sge)
805 {
806 void *wqe = NULL;
807
808 wqe = hns_roce_get_recv_wqe(hr_qp, wqe_idx);
809 fill_recv_sge_to_wqe(wr, wqe, max_sge, hr_qp->rq.rsv_sge);
810 }
811
hns_roce_v2_post_recv(struct ib_qp * ibqp,const struct ib_recv_wr * wr,const struct ib_recv_wr ** bad_wr)812 static int hns_roce_v2_post_recv(struct ib_qp *ibqp,
813 const struct ib_recv_wr *wr,
814 const struct ib_recv_wr **bad_wr)
815 {
816 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
817 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
818 struct ib_device *ibdev = &hr_dev->ib_dev;
819 u32 wqe_idx, nreq, max_sge;
820 unsigned long flags;
821 int ret;
822
823 spin_lock_irqsave(&hr_qp->rq.lock, flags);
824
825 ret = check_recv_valid(hr_dev, hr_qp);
826 if (unlikely(ret)) {
827 *bad_wr = wr;
828 nreq = 0;
829 goto out;
830 }
831
832 max_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge;
833 for (nreq = 0; wr; ++nreq, wr = wr->next) {
834 if (unlikely(hns_roce_wq_overflow(&hr_qp->rq, nreq,
835 hr_qp->ibqp.recv_cq))) {
836 ret = -ENOMEM;
837 *bad_wr = wr;
838 goto out;
839 }
840
841 if (unlikely(wr->num_sge > max_sge)) {
842 ibdev_err(ibdev, "num_sge = %d >= max_sge = %u.\n",
843 wr->num_sge, max_sge);
844 ret = -EINVAL;
845 *bad_wr = wr;
846 goto out;
847 }
848
849 wqe_idx = (hr_qp->rq.head + nreq) & (hr_qp->rq.wqe_cnt - 1);
850 fill_rq_wqe(hr_qp, wr, wqe_idx, max_sge);
851 hr_qp->rq.wrid[wqe_idx] = wr->wr_id;
852 }
853
854 out:
855 if (likely(nreq)) {
856 hr_qp->rq.head += nreq;
857
858 update_rq_db(hr_dev, hr_qp);
859 }
860 spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
861
862 return ret;
863 }
864
get_srq_wqe_buf(struct hns_roce_srq * srq,u32 n)865 static void *get_srq_wqe_buf(struct hns_roce_srq *srq, u32 n)
866 {
867 return hns_roce_buf_offset(srq->buf_mtr.kmem, n << srq->wqe_shift);
868 }
869
get_idx_buf(struct hns_roce_idx_que * idx_que,u32 n)870 static void *get_idx_buf(struct hns_roce_idx_que *idx_que, u32 n)
871 {
872 return hns_roce_buf_offset(idx_que->mtr.kmem,
873 n << idx_que->entry_shift);
874 }
875
hns_roce_free_srq_wqe(struct hns_roce_srq * srq,u32 wqe_index)876 static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, u32 wqe_index)
877 {
878 /* always called with interrupts disabled. */
879 spin_lock(&srq->lock);
880
881 bitmap_clear(srq->idx_que.bitmap, wqe_index, 1);
882 srq->idx_que.tail++;
883
884 spin_unlock(&srq->lock);
885 }
886
hns_roce_srqwq_overflow(struct hns_roce_srq * srq)887 static int hns_roce_srqwq_overflow(struct hns_roce_srq *srq)
888 {
889 struct hns_roce_idx_que *idx_que = &srq->idx_que;
890
891 return idx_que->head - idx_que->tail >= srq->wqe_cnt;
892 }
893
check_post_srq_valid(struct hns_roce_srq * srq,u32 max_sge,const struct ib_recv_wr * wr)894 static int check_post_srq_valid(struct hns_roce_srq *srq, u32 max_sge,
895 const struct ib_recv_wr *wr)
896 {
897 struct ib_device *ib_dev = srq->ibsrq.device;
898
899 if (unlikely(wr->num_sge > max_sge)) {
900 ibdev_err(ib_dev,
901 "failed to check sge, wr->num_sge = %d, max_sge = %u.\n",
902 wr->num_sge, max_sge);
903 return -EINVAL;
904 }
905
906 if (unlikely(hns_roce_srqwq_overflow(srq))) {
907 ibdev_err(ib_dev,
908 "failed to check srqwq status, srqwq is full.\n");
909 return -ENOMEM;
910 }
911
912 return 0;
913 }
914
get_srq_wqe_idx(struct hns_roce_srq * srq,u32 * wqe_idx)915 static int get_srq_wqe_idx(struct hns_roce_srq *srq, u32 *wqe_idx)
916 {
917 struct hns_roce_idx_que *idx_que = &srq->idx_que;
918 u32 pos;
919
920 pos = find_first_zero_bit(idx_que->bitmap, srq->wqe_cnt);
921 if (unlikely(pos == srq->wqe_cnt))
922 return -ENOSPC;
923
924 bitmap_set(idx_que->bitmap, pos, 1);
925 *wqe_idx = pos;
926 return 0;
927 }
928
fill_wqe_idx(struct hns_roce_srq * srq,unsigned int wqe_idx)929 static void fill_wqe_idx(struct hns_roce_srq *srq, unsigned int wqe_idx)
930 {
931 struct hns_roce_idx_que *idx_que = &srq->idx_que;
932 unsigned int head;
933 __le32 *buf;
934
935 head = idx_que->head & (srq->wqe_cnt - 1);
936
937 buf = get_idx_buf(idx_que, head);
938 *buf = cpu_to_le32(wqe_idx);
939
940 idx_que->head++;
941 }
942
update_srq_db(struct hns_roce_srq * srq)943 static void update_srq_db(struct hns_roce_srq *srq)
944 {
945 struct hns_roce_dev *hr_dev = to_hr_dev(srq->ibsrq.device);
946 struct hns_roce_v2_db db;
947
948 hr_reg_write(&db, DB_TAG, srq->srqn);
949 hr_reg_write(&db, DB_CMD, HNS_ROCE_V2_SRQ_DB);
950 hr_reg_write(&db, DB_PI, srq->idx_que.head);
951
952 hns_roce_write64(hr_dev, (__le32 *)&db, srq->db_reg);
953 }
954
hns_roce_v2_post_srq_recv(struct ib_srq * ibsrq,const struct ib_recv_wr * wr,const struct ib_recv_wr ** bad_wr)955 static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq,
956 const struct ib_recv_wr *wr,
957 const struct ib_recv_wr **bad_wr)
958 {
959 struct hns_roce_srq *srq = to_hr_srq(ibsrq);
960 unsigned long flags;
961 int ret = 0;
962 u32 max_sge;
963 u32 wqe_idx;
964 void *wqe;
965 u32 nreq;
966
967 spin_lock_irqsave(&srq->lock, flags);
968
969 max_sge = srq->max_gs - srq->rsv_sge;
970 for (nreq = 0; wr; ++nreq, wr = wr->next) {
971 ret = check_post_srq_valid(srq, max_sge, wr);
972 if (ret) {
973 *bad_wr = wr;
974 break;
975 }
976
977 ret = get_srq_wqe_idx(srq, &wqe_idx);
978 if (unlikely(ret)) {
979 *bad_wr = wr;
980 break;
981 }
982
983 wqe = get_srq_wqe_buf(srq, wqe_idx);
984 fill_recv_sge_to_wqe(wr, wqe, max_sge, srq->rsv_sge);
985 fill_wqe_idx(srq, wqe_idx);
986 srq->wrid[wqe_idx] = wr->wr_id;
987 }
988
989 if (likely(nreq)) {
990 if (srq->cap_flags & HNS_ROCE_SRQ_CAP_RECORD_DB)
991 *srq->rdb.db_record = srq->idx_que.head &
992 V2_DB_PRODUCER_IDX_M;
993 else
994 update_srq_db(srq);
995 }
996
997 spin_unlock_irqrestore(&srq->lock, flags);
998
999 return ret;
1000 }
1001
hns_roce_v2_cmd_hw_reseted(struct hns_roce_dev * hr_dev,unsigned long instance_stage,unsigned long reset_stage)1002 static u32 hns_roce_v2_cmd_hw_reseted(struct hns_roce_dev *hr_dev,
1003 unsigned long instance_stage,
1004 unsigned long reset_stage)
1005 {
1006 /* When hardware reset has been completed once or more, we should stop
1007 * sending mailbox&cmq&doorbell to hardware. If now in .init_instance()
1008 * function, we should exit with error. If now at HNAE3_INIT_CLIENT
1009 * stage of soft reset process, we should exit with error, and then
1010 * HNAE3_INIT_CLIENT related process can rollback the operation like
1011 * notifing hardware to free resources, HNAE3_INIT_CLIENT related
1012 * process will exit with error to notify NIC driver to reschedule soft
1013 * reset process once again.
1014 */
1015 hr_dev->is_reset = true;
1016 hr_dev->dis_db = true;
1017
1018 if (reset_stage == HNS_ROCE_STATE_RST_INIT ||
1019 instance_stage == HNS_ROCE_STATE_INIT)
1020 return CMD_RST_PRC_EBUSY;
1021
1022 return CMD_RST_PRC_SUCCESS;
1023 }
1024
hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev * hr_dev,unsigned long instance_stage,unsigned long reset_stage)1025 static u32 hns_roce_v2_cmd_hw_resetting(struct hns_roce_dev *hr_dev,
1026 unsigned long instance_stage,
1027 unsigned long reset_stage)
1028 {
1029 #define HW_RESET_TIMEOUT_US 1000000
1030 #define HW_RESET_SLEEP_US 1000
1031
1032 struct hns_roce_v2_priv *priv = hr_dev->priv;
1033 struct hnae3_handle *handle = priv->handle;
1034 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1035 unsigned long val;
1036 int ret;
1037
1038 /* When hardware reset is detected, we should stop sending mailbox&cmq&
1039 * doorbell to hardware. If now in .init_instance() function, we should
1040 * exit with error. If now at HNAE3_INIT_CLIENT stage of soft reset
1041 * process, we should exit with error, and then HNAE3_INIT_CLIENT
1042 * related process can rollback the operation like notifing hardware to
1043 * free resources, HNAE3_INIT_CLIENT related process will exit with
1044 * error to notify NIC driver to reschedule soft reset process once
1045 * again.
1046 */
1047 hr_dev->dis_db = true;
1048
1049 ret = read_poll_timeout(ops->ae_dev_reset_cnt, val,
1050 val > hr_dev->reset_cnt, HW_RESET_SLEEP_US,
1051 HW_RESET_TIMEOUT_US, false, handle);
1052 if (!ret)
1053 hr_dev->is_reset = true;
1054
1055 if (!hr_dev->is_reset || reset_stage == HNS_ROCE_STATE_RST_INIT ||
1056 instance_stage == HNS_ROCE_STATE_INIT)
1057 return CMD_RST_PRC_EBUSY;
1058
1059 return CMD_RST_PRC_SUCCESS;
1060 }
1061
hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev * hr_dev)1062 static u32 hns_roce_v2_cmd_sw_resetting(struct hns_roce_dev *hr_dev)
1063 {
1064 struct hns_roce_v2_priv *priv = hr_dev->priv;
1065 struct hnae3_handle *handle = priv->handle;
1066 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1067
1068 /* When software reset is detected at .init_instance() function, we
1069 * should stop sending mailbox&cmq&doorbell to hardware, and exit
1070 * with error.
1071 */
1072 hr_dev->dis_db = true;
1073 if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt)
1074 hr_dev->is_reset = true;
1075
1076 return CMD_RST_PRC_EBUSY;
1077 }
1078
check_aedev_reset_status(struct hns_roce_dev * hr_dev,struct hnae3_handle * handle)1079 static u32 check_aedev_reset_status(struct hns_roce_dev *hr_dev,
1080 struct hnae3_handle *handle)
1081 {
1082 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1083 unsigned long instance_stage; /* the current instance stage */
1084 unsigned long reset_stage; /* the current reset stage */
1085 unsigned long reset_cnt;
1086 bool sw_resetting;
1087 bool hw_resetting;
1088
1089 /* Get information about reset from NIC driver or RoCE driver itself,
1090 * the meaning of the following variables from NIC driver are described
1091 * as below:
1092 * reset_cnt -- The count value of completed hardware reset.
1093 * hw_resetting -- Whether hardware device is resetting now.
1094 * sw_resetting -- Whether NIC's software reset process is running now.
1095 */
1096 instance_stage = handle->rinfo.instance_state;
1097 reset_stage = handle->rinfo.reset_state;
1098 reset_cnt = ops->ae_dev_reset_cnt(handle);
1099 if (reset_cnt != hr_dev->reset_cnt)
1100 return hns_roce_v2_cmd_hw_reseted(hr_dev, instance_stage,
1101 reset_stage);
1102
1103 hw_resetting = ops->get_cmdq_stat(handle);
1104 if (hw_resetting)
1105 return hns_roce_v2_cmd_hw_resetting(hr_dev, instance_stage,
1106 reset_stage);
1107
1108 sw_resetting = ops->ae_dev_resetting(handle);
1109 if (sw_resetting && instance_stage == HNS_ROCE_STATE_INIT)
1110 return hns_roce_v2_cmd_sw_resetting(hr_dev);
1111
1112 return CMD_RST_PRC_OTHERS;
1113 }
1114
check_device_is_in_reset(struct hns_roce_dev * hr_dev)1115 static bool check_device_is_in_reset(struct hns_roce_dev *hr_dev)
1116 {
1117 struct hns_roce_v2_priv *priv = hr_dev->priv;
1118 struct hnae3_handle *handle = priv->handle;
1119 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1120
1121 if (hr_dev->reset_cnt != ops->ae_dev_reset_cnt(handle))
1122 return true;
1123
1124 if (ops->get_hw_reset_stat(handle))
1125 return true;
1126
1127 if (ops->ae_dev_resetting(handle))
1128 return true;
1129
1130 return false;
1131 }
1132
v2_chk_mbox_is_avail(struct hns_roce_dev * hr_dev,bool * busy)1133 static bool v2_chk_mbox_is_avail(struct hns_roce_dev *hr_dev, bool *busy)
1134 {
1135 struct hns_roce_v2_priv *priv = hr_dev->priv;
1136 u32 status;
1137
1138 if (hr_dev->is_reset)
1139 status = CMD_RST_PRC_SUCCESS;
1140 else
1141 status = check_aedev_reset_status(hr_dev, priv->handle);
1142
1143 *busy = (status == CMD_RST_PRC_EBUSY);
1144
1145 return status == CMD_RST_PRC_OTHERS;
1146 }
1147
hns_roce_alloc_cmq_desc(struct hns_roce_dev * hr_dev,struct hns_roce_v2_cmq_ring * ring)1148 static int hns_roce_alloc_cmq_desc(struct hns_roce_dev *hr_dev,
1149 struct hns_roce_v2_cmq_ring *ring)
1150 {
1151 int size = ring->desc_num * sizeof(struct hns_roce_cmq_desc);
1152
1153 ring->desc = dma_alloc_coherent(hr_dev->dev, size,
1154 &ring->desc_dma_addr, GFP_KERNEL);
1155 if (!ring->desc)
1156 return -ENOMEM;
1157
1158 return 0;
1159 }
1160
hns_roce_free_cmq_desc(struct hns_roce_dev * hr_dev,struct hns_roce_v2_cmq_ring * ring)1161 static void hns_roce_free_cmq_desc(struct hns_roce_dev *hr_dev,
1162 struct hns_roce_v2_cmq_ring *ring)
1163 {
1164 dma_free_coherent(hr_dev->dev,
1165 ring->desc_num * sizeof(struct hns_roce_cmq_desc),
1166 ring->desc, ring->desc_dma_addr);
1167
1168 ring->desc_dma_addr = 0;
1169 }
1170
init_csq(struct hns_roce_dev * hr_dev,struct hns_roce_v2_cmq_ring * csq)1171 static int init_csq(struct hns_roce_dev *hr_dev,
1172 struct hns_roce_v2_cmq_ring *csq)
1173 {
1174 dma_addr_t dma;
1175 int ret;
1176
1177 csq->desc_num = CMD_CSQ_DESC_NUM;
1178 spin_lock_init(&csq->lock);
1179 csq->flag = TYPE_CSQ;
1180 csq->head = 0;
1181
1182 ret = hns_roce_alloc_cmq_desc(hr_dev, csq);
1183 if (ret)
1184 return ret;
1185
1186 dma = csq->desc_dma_addr;
1187 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_L_REG, lower_32_bits(dma));
1188 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_H_REG, upper_32_bits(dma));
1189 roce_write(hr_dev, ROCEE_TX_CMQ_DEPTH_REG,
1190 (u32)csq->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S);
1191
1192 /* Make sure to write CI first and then PI */
1193 roce_write(hr_dev, ROCEE_TX_CMQ_CI_REG, 0);
1194 roce_write(hr_dev, ROCEE_TX_CMQ_PI_REG, 0);
1195
1196 return 0;
1197 }
1198
hns_roce_v2_cmq_init(struct hns_roce_dev * hr_dev)1199 static int hns_roce_v2_cmq_init(struct hns_roce_dev *hr_dev)
1200 {
1201 struct hns_roce_v2_priv *priv = hr_dev->priv;
1202 int ret;
1203
1204 priv->cmq.tx_timeout = HNS_ROCE_CMQ_TX_TIMEOUT;
1205
1206 ret = init_csq(hr_dev, &priv->cmq.csq);
1207 if (ret)
1208 dev_err(hr_dev->dev, "failed to init CSQ, ret = %d.\n", ret);
1209
1210 return ret;
1211 }
1212
hns_roce_v2_cmq_exit(struct hns_roce_dev * hr_dev)1213 static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev)
1214 {
1215 struct hns_roce_v2_priv *priv = hr_dev->priv;
1216
1217 hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq);
1218 }
1219
hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc * desc,enum hns_roce_opcode_type opcode,bool is_read)1220 static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc,
1221 enum hns_roce_opcode_type opcode,
1222 bool is_read)
1223 {
1224 memset((void *)desc, 0, sizeof(struct hns_roce_cmq_desc));
1225 desc->opcode = cpu_to_le16(opcode);
1226 desc->flag = cpu_to_le16(HNS_ROCE_CMD_FLAG_IN);
1227 if (is_read)
1228 desc->flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_WR);
1229 else
1230 desc->flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);
1231 }
1232
hns_roce_cmq_csq_done(struct hns_roce_dev * hr_dev)1233 static int hns_roce_cmq_csq_done(struct hns_roce_dev *hr_dev)
1234 {
1235 u32 tail = roce_read(hr_dev, ROCEE_TX_CMQ_CI_REG);
1236 struct hns_roce_v2_priv *priv = hr_dev->priv;
1237
1238 return tail == priv->cmq.csq.head;
1239 }
1240
update_cmdq_status(struct hns_roce_dev * hr_dev)1241 static void update_cmdq_status(struct hns_roce_dev *hr_dev)
1242 {
1243 struct hns_roce_v2_priv *priv = hr_dev->priv;
1244 struct hnae3_handle *handle = priv->handle;
1245
1246 if (handle->rinfo.reset_state == HNS_ROCE_STATE_RST_INIT ||
1247 handle->rinfo.instance_state == HNS_ROCE_STATE_INIT)
1248 hr_dev->cmd.state = HNS_ROCE_CMDQ_STATE_FATAL_ERR;
1249 }
1250
hns_roce_cmd_err_convert_errno(u16 desc_ret)1251 static int hns_roce_cmd_err_convert_errno(u16 desc_ret)
1252 {
1253 struct hns_roce_cmd_errcode errcode_table[] = {
1254 {CMD_EXEC_SUCCESS, 0},
1255 {CMD_NO_AUTH, -EPERM},
1256 {CMD_NOT_EXIST, -EOPNOTSUPP},
1257 {CMD_CRQ_FULL, -EXFULL},
1258 {CMD_NEXT_ERR, -ENOSR},
1259 {CMD_NOT_EXEC, -ENOTBLK},
1260 {CMD_PARA_ERR, -EINVAL},
1261 {CMD_RESULT_ERR, -ERANGE},
1262 {CMD_TIMEOUT, -ETIME},
1263 {CMD_HILINK_ERR, -ENOLINK},
1264 {CMD_INFO_ILLEGAL, -ENXIO},
1265 {CMD_INVALID, -EBADR},
1266 };
1267 u16 i;
1268
1269 for (i = 0; i < ARRAY_SIZE(errcode_table); i++)
1270 if (desc_ret == errcode_table[i].return_status)
1271 return errcode_table[i].errno;
1272 return -EIO;
1273 }
1274
hns_roce_cmdq_tx_timeout(u16 opcode,u32 tx_timeout)1275 static u32 hns_roce_cmdq_tx_timeout(u16 opcode, u32 tx_timeout)
1276 {
1277 static const struct hns_roce_cmdq_tx_timeout_map cmdq_tx_timeout[] = {
1278 {HNS_ROCE_OPC_POST_MB, HNS_ROCE_OPC_POST_MB_TIMEOUT},
1279 };
1280 int i;
1281
1282 for (i = 0; i < ARRAY_SIZE(cmdq_tx_timeout); i++)
1283 if (cmdq_tx_timeout[i].opcode == opcode)
1284 return cmdq_tx_timeout[i].tx_timeout;
1285
1286 return tx_timeout;
1287 }
1288
hns_roce_wait_csq_done(struct hns_roce_dev * hr_dev,u16 opcode)1289 static void hns_roce_wait_csq_done(struct hns_roce_dev *hr_dev, u16 opcode)
1290 {
1291 struct hns_roce_v2_priv *priv = hr_dev->priv;
1292 u32 tx_timeout = hns_roce_cmdq_tx_timeout(opcode, priv->cmq.tx_timeout);
1293 u32 timeout = 0;
1294
1295 do {
1296 if (hns_roce_cmq_csq_done(hr_dev))
1297 break;
1298 udelay(1);
1299 } while (++timeout < tx_timeout);
1300 }
1301
__hns_roce_cmq_send(struct hns_roce_dev * hr_dev,struct hns_roce_cmq_desc * desc,int num)1302 static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
1303 struct hns_roce_cmq_desc *desc, int num)
1304 {
1305 struct hns_roce_v2_priv *priv = hr_dev->priv;
1306 struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
1307 u16 desc_ret;
1308 u32 tail;
1309 int ret;
1310 int i;
1311
1312 spin_lock_bh(&csq->lock);
1313
1314 tail = csq->head;
1315
1316 for (i = 0; i < num; i++) {
1317 csq->desc[csq->head++] = desc[i];
1318 if (csq->head == csq->desc_num)
1319 csq->head = 0;
1320 }
1321
1322 /* Write to hardware */
1323 roce_write(hr_dev, ROCEE_TX_CMQ_PI_REG, csq->head);
1324
1325 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_CMDS_CNT]);
1326
1327 hns_roce_wait_csq_done(hr_dev, le16_to_cpu(desc->opcode));
1328 if (hns_roce_cmq_csq_done(hr_dev)) {
1329 ret = 0;
1330 for (i = 0; i < num; i++) {
1331 /* check the result of hardware write back */
1332 desc[i] = csq->desc[tail++];
1333 if (tail == csq->desc_num)
1334 tail = 0;
1335
1336 desc_ret = le16_to_cpu(desc[i].retval);
1337 if (likely(desc_ret == CMD_EXEC_SUCCESS))
1338 continue;
1339
1340 dev_err_ratelimited(hr_dev->dev,
1341 "Cmdq IO error, opcode = 0x%x, return = 0x%x.\n",
1342 desc->opcode, desc_ret);
1343 ret = hns_roce_cmd_err_convert_errno(desc_ret);
1344 }
1345 } else {
1346 /* FW/HW reset or incorrect number of desc */
1347 tail = roce_read(hr_dev, ROCEE_TX_CMQ_CI_REG);
1348 dev_warn(hr_dev->dev, "CMDQ move tail from %u to %u.\n",
1349 csq->head, tail);
1350 csq->head = tail;
1351
1352 update_cmdq_status(hr_dev);
1353
1354 ret = -EAGAIN;
1355 }
1356
1357 spin_unlock_bh(&csq->lock);
1358
1359 if (ret)
1360 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_CMDS_ERR_CNT]);
1361
1362 return ret;
1363 }
1364
hns_roce_cmq_send(struct hns_roce_dev * hr_dev,struct hns_roce_cmq_desc * desc,int num)1365 static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
1366 struct hns_roce_cmq_desc *desc, int num)
1367 {
1368 bool busy;
1369 int ret;
1370
1371 if (hr_dev->cmd.state == HNS_ROCE_CMDQ_STATE_FATAL_ERR)
1372 return -EIO;
1373
1374 if (!v2_chk_mbox_is_avail(hr_dev, &busy))
1375 return busy ? -EBUSY : 0;
1376
1377 ret = __hns_roce_cmq_send(hr_dev, desc, num);
1378 if (ret) {
1379 if (!v2_chk_mbox_is_avail(hr_dev, &busy))
1380 return busy ? -EBUSY : 0;
1381 }
1382
1383 return ret;
1384 }
1385
config_hem_ba_to_hw(struct hns_roce_dev * hr_dev,dma_addr_t base_addr,u8 cmd,unsigned long tag)1386 static int config_hem_ba_to_hw(struct hns_roce_dev *hr_dev,
1387 dma_addr_t base_addr, u8 cmd, unsigned long tag)
1388 {
1389 struct hns_roce_cmd_mailbox *mbox;
1390 int ret;
1391
1392 mbox = hns_roce_alloc_cmd_mailbox(hr_dev);
1393 if (IS_ERR(mbox))
1394 return PTR_ERR(mbox);
1395
1396 ret = hns_roce_cmd_mbox(hr_dev, base_addr, mbox->dma, cmd, tag);
1397 hns_roce_free_cmd_mailbox(hr_dev, mbox);
1398 return ret;
1399 }
1400
hns_roce_cmq_query_hw_info(struct hns_roce_dev * hr_dev)1401 static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev)
1402 {
1403 struct hns_roce_query_version *resp;
1404 struct hns_roce_cmq_desc desc;
1405 int ret;
1406
1407 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_HW_VER, true);
1408 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1409 if (ret)
1410 return ret;
1411
1412 resp = (struct hns_roce_query_version *)desc.data;
1413 hr_dev->hw_rev = le16_to_cpu(resp->rocee_hw_version);
1414 hr_dev->vendor_id = hr_dev->pci_dev->vendor;
1415
1416 return 0;
1417 }
1418
func_clr_hw_resetting_state(struct hns_roce_dev * hr_dev,struct hnae3_handle * handle)1419 static void func_clr_hw_resetting_state(struct hns_roce_dev *hr_dev,
1420 struct hnae3_handle *handle)
1421 {
1422 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1423 unsigned long end;
1424
1425 hr_dev->dis_db = true;
1426
1427 dev_warn(hr_dev->dev,
1428 "func clear is pending, device in resetting state.\n");
1429 end = HNS_ROCE_V2_HW_RST_TIMEOUT;
1430 while (end) {
1431 if (!ops->get_hw_reset_stat(handle)) {
1432 hr_dev->is_reset = true;
1433 dev_info(hr_dev->dev,
1434 "func clear success after reset.\n");
1435 return;
1436 }
1437 msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT);
1438 end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT;
1439 }
1440
1441 dev_warn(hr_dev->dev, "func clear failed.\n");
1442 }
1443
func_clr_sw_resetting_state(struct hns_roce_dev * hr_dev,struct hnae3_handle * handle)1444 static void func_clr_sw_resetting_state(struct hns_roce_dev *hr_dev,
1445 struct hnae3_handle *handle)
1446 {
1447 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1448 unsigned long end;
1449
1450 hr_dev->dis_db = true;
1451
1452 dev_warn(hr_dev->dev,
1453 "func clear is pending, device in resetting state.\n");
1454 end = HNS_ROCE_V2_HW_RST_TIMEOUT;
1455 while (end) {
1456 if (ops->ae_dev_reset_cnt(handle) !=
1457 hr_dev->reset_cnt) {
1458 hr_dev->is_reset = true;
1459 dev_info(hr_dev->dev,
1460 "func clear success after sw reset\n");
1461 return;
1462 }
1463 msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT);
1464 end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT;
1465 }
1466
1467 dev_warn(hr_dev->dev, "func clear failed because of unfinished sw reset\n");
1468 }
1469
hns_roce_func_clr_rst_proc(struct hns_roce_dev * hr_dev,int retval,int flag)1470 static void hns_roce_func_clr_rst_proc(struct hns_roce_dev *hr_dev, int retval,
1471 int flag)
1472 {
1473 struct hns_roce_v2_priv *priv = hr_dev->priv;
1474 struct hnae3_handle *handle = priv->handle;
1475 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
1476
1477 if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt) {
1478 hr_dev->dis_db = true;
1479 hr_dev->is_reset = true;
1480 dev_info(hr_dev->dev, "func clear success after reset.\n");
1481 return;
1482 }
1483
1484 if (ops->get_hw_reset_stat(handle)) {
1485 func_clr_hw_resetting_state(hr_dev, handle);
1486 return;
1487 }
1488
1489 if (ops->ae_dev_resetting(handle) &&
1490 handle->rinfo.instance_state == HNS_ROCE_STATE_INIT) {
1491 func_clr_sw_resetting_state(hr_dev, handle);
1492 return;
1493 }
1494
1495 if (retval && !flag)
1496 dev_warn(hr_dev->dev,
1497 "func clear read failed, ret = %d.\n", retval);
1498
1499 dev_warn(hr_dev->dev, "func clear failed.\n");
1500 }
1501
__hns_roce_function_clear(struct hns_roce_dev * hr_dev,int vf_id)1502 static void __hns_roce_function_clear(struct hns_roce_dev *hr_dev, int vf_id)
1503 {
1504 bool fclr_write_fail_flag = false;
1505 struct hns_roce_func_clear *resp;
1506 struct hns_roce_cmq_desc desc;
1507 unsigned long end;
1508 int ret = 0;
1509
1510 if (check_device_is_in_reset(hr_dev))
1511 goto out;
1512
1513 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, false);
1514 resp = (struct hns_roce_func_clear *)desc.data;
1515 resp->rst_funcid_en = cpu_to_le32(vf_id);
1516
1517 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1518 if (ret) {
1519 fclr_write_fail_flag = true;
1520 dev_err(hr_dev->dev, "func clear write failed, ret = %d.\n",
1521 ret);
1522 goto out;
1523 }
1524
1525 msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL);
1526 end = HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS;
1527 while (end) {
1528 if (check_device_is_in_reset(hr_dev))
1529 goto out;
1530 msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT);
1531 end -= HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT;
1532
1533 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR,
1534 true);
1535
1536 resp->rst_funcid_en = cpu_to_le32(vf_id);
1537 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1538 if (ret)
1539 continue;
1540
1541 if (hr_reg_read(resp, FUNC_CLEAR_RST_FUN_DONE)) {
1542 if (vf_id == 0)
1543 hr_dev->is_reset = true;
1544 return;
1545 }
1546 }
1547
1548 out:
1549 hns_roce_func_clr_rst_proc(hr_dev, ret, fclr_write_fail_flag);
1550 }
1551
hns_roce_free_vf_resource(struct hns_roce_dev * hr_dev,int vf_id)1552 static int hns_roce_free_vf_resource(struct hns_roce_dev *hr_dev, int vf_id)
1553 {
1554 enum hns_roce_opcode_type opcode = HNS_ROCE_OPC_ALLOC_VF_RES;
1555 struct hns_roce_cmq_desc desc[2];
1556 struct hns_roce_cmq_req *req_a;
1557
1558 req_a = (struct hns_roce_cmq_req *)desc[0].data;
1559 hns_roce_cmq_setup_basic_desc(&desc[0], opcode, false);
1560 desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1561 hns_roce_cmq_setup_basic_desc(&desc[1], opcode, false);
1562 hr_reg_write(req_a, FUNC_RES_A_VF_ID, vf_id);
1563
1564 return hns_roce_cmq_send(hr_dev, desc, 2);
1565 }
1566
hns_roce_function_clear(struct hns_roce_dev * hr_dev)1567 static void hns_roce_function_clear(struct hns_roce_dev *hr_dev)
1568 {
1569 int ret;
1570 int i;
1571
1572 if (hr_dev->cmd.state == HNS_ROCE_CMDQ_STATE_FATAL_ERR)
1573 return;
1574
1575 for (i = hr_dev->func_num - 1; i >= 0; i--) {
1576 __hns_roce_function_clear(hr_dev, i);
1577
1578 if (i == 0)
1579 continue;
1580
1581 ret = hns_roce_free_vf_resource(hr_dev, i);
1582 if (ret)
1583 ibdev_err(&hr_dev->ib_dev,
1584 "failed to free vf resource, vf_id = %d, ret = %d.\n",
1585 i, ret);
1586 }
1587 }
1588
hns_roce_clear_extdb_list_info(struct hns_roce_dev * hr_dev)1589 static int hns_roce_clear_extdb_list_info(struct hns_roce_dev *hr_dev)
1590 {
1591 struct hns_roce_cmq_desc desc;
1592 int ret;
1593
1594 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CLEAR_EXTDB_LIST_INFO,
1595 false);
1596 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1597 if (ret)
1598 ibdev_err(&hr_dev->ib_dev,
1599 "failed to clear extended doorbell info, ret = %d.\n",
1600 ret);
1601
1602 return ret;
1603 }
1604
hns_roce_query_fw_ver(struct hns_roce_dev * hr_dev)1605 static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev)
1606 {
1607 struct hns_roce_query_fw_info *resp;
1608 struct hns_roce_cmq_desc desc;
1609 int ret;
1610
1611 hns_roce_cmq_setup_basic_desc(&desc, HNS_QUERY_FW_VER, true);
1612 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1613 if (ret)
1614 return ret;
1615
1616 resp = (struct hns_roce_query_fw_info *)desc.data;
1617 hr_dev->caps.fw_ver = (u64)(le32_to_cpu(resp->fw_ver));
1618
1619 return 0;
1620 }
1621
hns_roce_query_func_info(struct hns_roce_dev * hr_dev)1622 static int hns_roce_query_func_info(struct hns_roce_dev *hr_dev)
1623 {
1624 struct hns_roce_cmq_desc desc;
1625 int ret;
1626
1627 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {
1628 hr_dev->func_num = 1;
1629 return 0;
1630 }
1631
1632 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_FUNC_INFO,
1633 true);
1634 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1635 if (ret) {
1636 hr_dev->func_num = 1;
1637 return ret;
1638 }
1639
1640 hr_dev->func_num = le32_to_cpu(desc.func_info.own_func_num);
1641 hr_dev->cong_algo_tmpl_id = le32_to_cpu(desc.func_info.own_mac_id);
1642
1643 return 0;
1644 }
1645
hns_roce_hw_v2_query_counter(struct hns_roce_dev * hr_dev,u64 * stats,u32 port,int * num_counters)1646 static int hns_roce_hw_v2_query_counter(struct hns_roce_dev *hr_dev,
1647 u64 *stats, u32 port, int *num_counters)
1648 {
1649 #define CNT_PER_DESC 3
1650 struct hns_roce_cmq_desc *desc;
1651 int bd_idx, cnt_idx;
1652 __le64 *cnt_data;
1653 int desc_num;
1654 int ret;
1655 int i;
1656
1657 if (port > hr_dev->caps.num_ports)
1658 return -EINVAL;
1659
1660 desc_num = DIV_ROUND_UP(HNS_ROCE_HW_CNT_TOTAL, CNT_PER_DESC);
1661 desc = kcalloc(desc_num, sizeof(*desc), GFP_KERNEL);
1662 if (!desc)
1663 return -ENOMEM;
1664
1665 for (i = 0; i < desc_num; i++) {
1666 hns_roce_cmq_setup_basic_desc(&desc[i],
1667 HNS_ROCE_OPC_QUERY_COUNTER, true);
1668 if (i != desc_num - 1)
1669 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1670 }
1671
1672 ret = hns_roce_cmq_send(hr_dev, desc, desc_num);
1673 if (ret) {
1674 ibdev_err(&hr_dev->ib_dev,
1675 "failed to get counter, ret = %d.\n", ret);
1676 goto err_out;
1677 }
1678
1679 for (i = 0; i < HNS_ROCE_HW_CNT_TOTAL && i < *num_counters; i++) {
1680 bd_idx = i / CNT_PER_DESC;
1681 if (bd_idx != HNS_ROCE_HW_CNT_TOTAL / CNT_PER_DESC &&
1682 !(desc[bd_idx].flag & cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT)))
1683 break;
1684
1685 cnt_data = (__le64 *)&desc[bd_idx].data[0];
1686 cnt_idx = i % CNT_PER_DESC;
1687 stats[i] = le64_to_cpu(cnt_data[cnt_idx]);
1688 }
1689 *num_counters = i;
1690
1691 err_out:
1692 kfree(desc);
1693 return ret;
1694 }
1695
hns_roce_config_global_param(struct hns_roce_dev * hr_dev)1696 static int hns_roce_config_global_param(struct hns_roce_dev *hr_dev)
1697 {
1698 struct hns_roce_cmq_desc desc;
1699 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;
1700 u32 clock_cycles_of_1us;
1701
1702 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GLOBAL_PARAM,
1703 false);
1704
1705 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08)
1706 clock_cycles_of_1us = HNS_ROCE_1NS_CFG;
1707 else
1708 clock_cycles_of_1us = HNS_ROCE_1US_CFG;
1709
1710 hr_reg_write(req, CFG_GLOBAL_PARAM_1US_CYCLES, clock_cycles_of_1us);
1711 hr_reg_write(req, CFG_GLOBAL_PARAM_UDP_PORT, ROCE_V2_UDP_DPORT);
1712
1713 return hns_roce_cmq_send(hr_dev, &desc, 1);
1714 }
1715
load_func_res_caps(struct hns_roce_dev * hr_dev,bool is_vf)1716 static int load_func_res_caps(struct hns_roce_dev *hr_dev, bool is_vf)
1717 {
1718 struct hns_roce_cmq_desc desc[2];
1719 struct hns_roce_cmq_req *r_a = (struct hns_roce_cmq_req *)desc[0].data;
1720 struct hns_roce_cmq_req *r_b = (struct hns_roce_cmq_req *)desc[1].data;
1721 struct hns_roce_caps *caps = &hr_dev->caps;
1722 enum hns_roce_opcode_type opcode;
1723 u32 func_num;
1724 int ret;
1725
1726 if (is_vf) {
1727 opcode = HNS_ROCE_OPC_QUERY_VF_RES;
1728 func_num = 1;
1729 } else {
1730 opcode = HNS_ROCE_OPC_QUERY_PF_RES;
1731 func_num = hr_dev->func_num;
1732 }
1733
1734 hns_roce_cmq_setup_basic_desc(&desc[0], opcode, true);
1735 desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1736 hns_roce_cmq_setup_basic_desc(&desc[1], opcode, true);
1737
1738 ret = hns_roce_cmq_send(hr_dev, desc, 2);
1739 if (ret)
1740 return ret;
1741
1742 caps->qpc_bt_num = hr_reg_read(r_a, FUNC_RES_A_QPC_BT_NUM) / func_num;
1743 caps->srqc_bt_num = hr_reg_read(r_a, FUNC_RES_A_SRQC_BT_NUM) / func_num;
1744 caps->cqc_bt_num = hr_reg_read(r_a, FUNC_RES_A_CQC_BT_NUM) / func_num;
1745 caps->mpt_bt_num = hr_reg_read(r_a, FUNC_RES_A_MPT_BT_NUM) / func_num;
1746 caps->eqc_bt_num = hr_reg_read(r_a, FUNC_RES_A_EQC_BT_NUM) / func_num;
1747 caps->smac_bt_num = hr_reg_read(r_b, FUNC_RES_B_SMAC_NUM) / func_num;
1748 caps->sgid_bt_num = hr_reg_read(r_b, FUNC_RES_B_SGID_NUM) / func_num;
1749 caps->sccc_bt_num = hr_reg_read(r_b, FUNC_RES_B_SCCC_BT_NUM) / func_num;
1750
1751 if (is_vf) {
1752 caps->sl_num = hr_reg_read(r_b, FUNC_RES_V_QID_NUM) / func_num;
1753 caps->gmv_bt_num = hr_reg_read(r_b, FUNC_RES_V_GMV_BT_NUM) /
1754 func_num;
1755 } else {
1756 caps->sl_num = hr_reg_read(r_b, FUNC_RES_B_QID_NUM) / func_num;
1757 caps->gmv_bt_num = hr_reg_read(r_b, FUNC_RES_B_GMV_BT_NUM) /
1758 func_num;
1759 }
1760
1761 return 0;
1762 }
1763
load_pf_timer_res_caps(struct hns_roce_dev * hr_dev)1764 static int load_pf_timer_res_caps(struct hns_roce_dev *hr_dev)
1765 {
1766 struct hns_roce_cmq_desc desc;
1767 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;
1768 struct hns_roce_caps *caps = &hr_dev->caps;
1769 int ret;
1770
1771 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_PF_TIMER_RES,
1772 true);
1773
1774 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1775 if (ret)
1776 return ret;
1777
1778 caps->qpc_timer_bt_num = hr_reg_read(req, PF_TIMER_RES_QPC_ITEM_NUM);
1779 caps->cqc_timer_bt_num = hr_reg_read(req, PF_TIMER_RES_CQC_ITEM_NUM);
1780
1781 return 0;
1782 }
1783
hns_roce_query_pf_resource(struct hns_roce_dev * hr_dev)1784 static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev)
1785 {
1786 struct device *dev = hr_dev->dev;
1787 int ret;
1788
1789 ret = load_func_res_caps(hr_dev, false);
1790 if (ret) {
1791 dev_err(dev, "failed to load pf res caps, ret = %d.\n", ret);
1792 return ret;
1793 }
1794
1795 ret = load_pf_timer_res_caps(hr_dev);
1796 if (ret)
1797 dev_err(dev, "failed to load pf timer resource, ret = %d.\n",
1798 ret);
1799
1800 return ret;
1801 }
1802
hns_roce_query_vf_resource(struct hns_roce_dev * hr_dev)1803 static int hns_roce_query_vf_resource(struct hns_roce_dev *hr_dev)
1804 {
1805 struct device *dev = hr_dev->dev;
1806 int ret;
1807
1808 ret = load_func_res_caps(hr_dev, true);
1809 if (ret)
1810 dev_err(dev, "failed to load vf res caps, ret = %d.\n", ret);
1811
1812 return ret;
1813 }
1814
__hns_roce_set_vf_switch_param(struct hns_roce_dev * hr_dev,u32 vf_id)1815 static int __hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev,
1816 u32 vf_id)
1817 {
1818 struct hns_roce_vf_switch *swt;
1819 struct hns_roce_cmq_desc desc;
1820 int ret;
1821
1822 swt = (struct hns_roce_vf_switch *)desc.data;
1823 hns_roce_cmq_setup_basic_desc(&desc, HNS_SWITCH_PARAMETER_CFG, true);
1824 swt->rocee_sel |= cpu_to_le32(HNS_ICL_SWITCH_CMD_ROCEE_SEL);
1825 hr_reg_write(swt, VF_SWITCH_VF_ID, vf_id);
1826 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1827 if (ret)
1828 return ret;
1829
1830 desc.flag = cpu_to_le16(HNS_ROCE_CMD_FLAG_IN);
1831 desc.flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);
1832 hr_reg_enable(swt, VF_SWITCH_ALW_LPBK);
1833 hr_reg_clear(swt, VF_SWITCH_ALW_LCL_LPBK);
1834 hr_reg_enable(swt, VF_SWITCH_ALW_DST_OVRD);
1835
1836 return hns_roce_cmq_send(hr_dev, &desc, 1);
1837 }
1838
hns_roce_set_vf_switch_param(struct hns_roce_dev * hr_dev)1839 static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev)
1840 {
1841 u32 vf_id;
1842 int ret;
1843
1844 for (vf_id = 0; vf_id < hr_dev->func_num; vf_id++) {
1845 ret = __hns_roce_set_vf_switch_param(hr_dev, vf_id);
1846 if (ret)
1847 return ret;
1848 }
1849 return 0;
1850 }
1851
config_vf_hem_resource(struct hns_roce_dev * hr_dev,int vf_id)1852 static int config_vf_hem_resource(struct hns_roce_dev *hr_dev, int vf_id)
1853 {
1854 struct hns_roce_cmq_desc desc[2];
1855 struct hns_roce_cmq_req *r_a = (struct hns_roce_cmq_req *)desc[0].data;
1856 struct hns_roce_cmq_req *r_b = (struct hns_roce_cmq_req *)desc[1].data;
1857 enum hns_roce_opcode_type opcode = HNS_ROCE_OPC_ALLOC_VF_RES;
1858 struct hns_roce_caps *caps = &hr_dev->caps;
1859
1860 hns_roce_cmq_setup_basic_desc(&desc[0], opcode, false);
1861 desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1862 hns_roce_cmq_setup_basic_desc(&desc[1], opcode, false);
1863
1864 hr_reg_write(r_a, FUNC_RES_A_VF_ID, vf_id);
1865
1866 hr_reg_write(r_a, FUNC_RES_A_QPC_BT_NUM, caps->qpc_bt_num);
1867 hr_reg_write(r_a, FUNC_RES_A_QPC_BT_IDX, vf_id * caps->qpc_bt_num);
1868 hr_reg_write(r_a, FUNC_RES_A_SRQC_BT_NUM, caps->srqc_bt_num);
1869 hr_reg_write(r_a, FUNC_RES_A_SRQC_BT_IDX, vf_id * caps->srqc_bt_num);
1870 hr_reg_write(r_a, FUNC_RES_A_CQC_BT_NUM, caps->cqc_bt_num);
1871 hr_reg_write(r_a, FUNC_RES_A_CQC_BT_IDX, vf_id * caps->cqc_bt_num);
1872 hr_reg_write(r_a, FUNC_RES_A_MPT_BT_NUM, caps->mpt_bt_num);
1873 hr_reg_write(r_a, FUNC_RES_A_MPT_BT_IDX, vf_id * caps->mpt_bt_num);
1874 hr_reg_write(r_a, FUNC_RES_A_EQC_BT_NUM, caps->eqc_bt_num);
1875 hr_reg_write(r_a, FUNC_RES_A_EQC_BT_IDX, vf_id * caps->eqc_bt_num);
1876 hr_reg_write(r_b, FUNC_RES_V_QID_NUM, caps->sl_num);
1877 hr_reg_write(r_b, FUNC_RES_B_QID_IDX, vf_id * caps->sl_num);
1878 hr_reg_write(r_b, FUNC_RES_B_SCCC_BT_NUM, caps->sccc_bt_num);
1879 hr_reg_write(r_b, FUNC_RES_B_SCCC_BT_IDX, vf_id * caps->sccc_bt_num);
1880
1881 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {
1882 hr_reg_write(r_b, FUNC_RES_V_GMV_BT_NUM, caps->gmv_bt_num);
1883 hr_reg_write(r_b, FUNC_RES_B_GMV_BT_IDX,
1884 vf_id * caps->gmv_bt_num);
1885 } else {
1886 hr_reg_write(r_b, FUNC_RES_B_SGID_NUM, caps->sgid_bt_num);
1887 hr_reg_write(r_b, FUNC_RES_B_SGID_IDX,
1888 vf_id * caps->sgid_bt_num);
1889 hr_reg_write(r_b, FUNC_RES_B_SMAC_NUM, caps->smac_bt_num);
1890 hr_reg_write(r_b, FUNC_RES_B_SMAC_IDX,
1891 vf_id * caps->smac_bt_num);
1892 }
1893
1894 return hns_roce_cmq_send(hr_dev, desc, 2);
1895 }
1896
hns_roce_alloc_vf_resource(struct hns_roce_dev * hr_dev)1897 static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev)
1898 {
1899 u32 func_num = max_t(u32, 1, hr_dev->func_num);
1900 u32 vf_id;
1901 int ret;
1902
1903 for (vf_id = 0; vf_id < func_num; vf_id++) {
1904 ret = config_vf_hem_resource(hr_dev, vf_id);
1905 if (ret) {
1906 dev_err(hr_dev->dev,
1907 "failed to config vf-%u hem res, ret = %d.\n",
1908 vf_id, ret);
1909 return ret;
1910 }
1911 }
1912
1913 return 0;
1914 }
1915
hns_roce_v2_set_bt(struct hns_roce_dev * hr_dev)1916 static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev)
1917 {
1918 struct hns_roce_cmq_desc desc;
1919 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;
1920 struct hns_roce_caps *caps = &hr_dev->caps;
1921
1922 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_BT_ATTR, false);
1923
1924 hr_reg_write(req, CFG_BT_ATTR_QPC_BA_PGSZ,
1925 caps->qpc_ba_pg_sz + PG_SHIFT_OFFSET);
1926 hr_reg_write(req, CFG_BT_ATTR_QPC_BUF_PGSZ,
1927 caps->qpc_buf_pg_sz + PG_SHIFT_OFFSET);
1928 hr_reg_write(req, CFG_BT_ATTR_QPC_HOPNUM,
1929 to_hr_hem_hopnum(caps->qpc_hop_num, caps->num_qps));
1930
1931 hr_reg_write(req, CFG_BT_ATTR_SRQC_BA_PGSZ,
1932 caps->srqc_ba_pg_sz + PG_SHIFT_OFFSET);
1933 hr_reg_write(req, CFG_BT_ATTR_SRQC_BUF_PGSZ,
1934 caps->srqc_buf_pg_sz + PG_SHIFT_OFFSET);
1935 hr_reg_write(req, CFG_BT_ATTR_SRQC_HOPNUM,
1936 to_hr_hem_hopnum(caps->srqc_hop_num, caps->num_srqs));
1937
1938 hr_reg_write(req, CFG_BT_ATTR_CQC_BA_PGSZ,
1939 caps->cqc_ba_pg_sz + PG_SHIFT_OFFSET);
1940 hr_reg_write(req, CFG_BT_ATTR_CQC_BUF_PGSZ,
1941 caps->cqc_buf_pg_sz + PG_SHIFT_OFFSET);
1942 hr_reg_write(req, CFG_BT_ATTR_CQC_HOPNUM,
1943 to_hr_hem_hopnum(caps->cqc_hop_num, caps->num_cqs));
1944
1945 hr_reg_write(req, CFG_BT_ATTR_MPT_BA_PGSZ,
1946 caps->mpt_ba_pg_sz + PG_SHIFT_OFFSET);
1947 hr_reg_write(req, CFG_BT_ATTR_MPT_BUF_PGSZ,
1948 caps->mpt_buf_pg_sz + PG_SHIFT_OFFSET);
1949 hr_reg_write(req, CFG_BT_ATTR_MPT_HOPNUM,
1950 to_hr_hem_hopnum(caps->mpt_hop_num, caps->num_mtpts));
1951
1952 hr_reg_write(req, CFG_BT_ATTR_SCCC_BA_PGSZ,
1953 caps->sccc_ba_pg_sz + PG_SHIFT_OFFSET);
1954 hr_reg_write(req, CFG_BT_ATTR_SCCC_BUF_PGSZ,
1955 caps->sccc_buf_pg_sz + PG_SHIFT_OFFSET);
1956 hr_reg_write(req, CFG_BT_ATTR_SCCC_HOPNUM,
1957 to_hr_hem_hopnum(caps->sccc_hop_num, caps->num_qps));
1958
1959 return hns_roce_cmq_send(hr_dev, &desc, 1);
1960 }
1961
calc_pg_sz(u32 obj_num,u32 obj_size,u32 hop_num,u32 ctx_bt_num,u32 * buf_page_size,u32 * bt_page_size,u32 hem_type)1962 static void calc_pg_sz(u32 obj_num, u32 obj_size, u32 hop_num, u32 ctx_bt_num,
1963 u32 *buf_page_size, u32 *bt_page_size, u32 hem_type)
1964 {
1965 u64 obj_per_chunk;
1966 u64 bt_chunk_size = PAGE_SIZE;
1967 u64 buf_chunk_size = PAGE_SIZE;
1968 u64 obj_per_chunk_default = buf_chunk_size / obj_size;
1969
1970 *buf_page_size = 0;
1971 *bt_page_size = 0;
1972
1973 switch (hop_num) {
1974 case 3:
1975 obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *
1976 (bt_chunk_size / BA_BYTE_LEN) *
1977 (bt_chunk_size / BA_BYTE_LEN) *
1978 obj_per_chunk_default;
1979 break;
1980 case 2:
1981 obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *
1982 (bt_chunk_size / BA_BYTE_LEN) *
1983 obj_per_chunk_default;
1984 break;
1985 case 1:
1986 obj_per_chunk = ctx_bt_num * (bt_chunk_size / BA_BYTE_LEN) *
1987 obj_per_chunk_default;
1988 break;
1989 case HNS_ROCE_HOP_NUM_0:
1990 obj_per_chunk = ctx_bt_num * obj_per_chunk_default;
1991 break;
1992 default:
1993 pr_err("table %u not support hop_num = %u!\n", hem_type,
1994 hop_num);
1995 return;
1996 }
1997
1998 if (hem_type >= HEM_TYPE_MTT)
1999 *bt_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk));
2000 else
2001 *buf_page_size = ilog2(DIV_ROUND_UP(obj_num, obj_per_chunk));
2002 }
2003
set_hem_page_size(struct hns_roce_dev * hr_dev)2004 static void set_hem_page_size(struct hns_roce_dev *hr_dev)
2005 {
2006 struct hns_roce_caps *caps = &hr_dev->caps;
2007
2008 /* EQ */
2009 caps->eqe_ba_pg_sz = 0;
2010 caps->eqe_buf_pg_sz = 0;
2011
2012 /* Link Table */
2013 caps->llm_buf_pg_sz = 0;
2014
2015 /* MR */
2016 caps->mpt_ba_pg_sz = 0;
2017 caps->mpt_buf_pg_sz = 0;
2018 caps->pbl_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_16K;
2019 caps->pbl_buf_pg_sz = 0;
2020 calc_pg_sz(caps->num_mtpts, caps->mtpt_entry_sz, caps->mpt_hop_num,
2021 caps->mpt_bt_num, &caps->mpt_buf_pg_sz, &caps->mpt_ba_pg_sz,
2022 HEM_TYPE_MTPT);
2023
2024 /* QP */
2025 caps->qpc_ba_pg_sz = 0;
2026 caps->qpc_buf_pg_sz = 0;
2027 caps->qpc_timer_ba_pg_sz = 0;
2028 caps->qpc_timer_buf_pg_sz = 0;
2029 caps->sccc_ba_pg_sz = 0;
2030 caps->sccc_buf_pg_sz = 0;
2031 caps->mtt_ba_pg_sz = 0;
2032 caps->mtt_buf_pg_sz = 0;
2033 calc_pg_sz(caps->num_qps, caps->qpc_sz, caps->qpc_hop_num,
2034 caps->qpc_bt_num, &caps->qpc_buf_pg_sz, &caps->qpc_ba_pg_sz,
2035 HEM_TYPE_QPC);
2036
2037 if (caps->flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL)
2038 calc_pg_sz(caps->num_qps, caps->sccc_sz, caps->sccc_hop_num,
2039 caps->sccc_bt_num, &caps->sccc_buf_pg_sz,
2040 &caps->sccc_ba_pg_sz, HEM_TYPE_SCCC);
2041
2042 /* CQ */
2043 caps->cqc_ba_pg_sz = 0;
2044 caps->cqc_buf_pg_sz = 0;
2045 caps->cqc_timer_ba_pg_sz = 0;
2046 caps->cqc_timer_buf_pg_sz = 0;
2047 caps->cqe_ba_pg_sz = HNS_ROCE_BA_PG_SZ_SUPPORTED_256K;
2048 caps->cqe_buf_pg_sz = 0;
2049 calc_pg_sz(caps->num_cqs, caps->cqc_entry_sz, caps->cqc_hop_num,
2050 caps->cqc_bt_num, &caps->cqc_buf_pg_sz, &caps->cqc_ba_pg_sz,
2051 HEM_TYPE_CQC);
2052 calc_pg_sz(caps->max_cqes, caps->cqe_sz, caps->cqe_hop_num,
2053 1, &caps->cqe_buf_pg_sz, &caps->cqe_ba_pg_sz, HEM_TYPE_CQE);
2054
2055 /* SRQ */
2056 if (caps->flags & HNS_ROCE_CAP_FLAG_SRQ) {
2057 caps->srqc_ba_pg_sz = 0;
2058 caps->srqc_buf_pg_sz = 0;
2059 caps->srqwqe_ba_pg_sz = 0;
2060 caps->srqwqe_buf_pg_sz = 0;
2061 caps->idx_ba_pg_sz = 0;
2062 caps->idx_buf_pg_sz = 0;
2063 calc_pg_sz(caps->num_srqs, caps->srqc_entry_sz,
2064 caps->srqc_hop_num, caps->srqc_bt_num,
2065 &caps->srqc_buf_pg_sz, &caps->srqc_ba_pg_sz,
2066 HEM_TYPE_SRQC);
2067 calc_pg_sz(caps->num_srqwqe_segs, caps->mtt_entry_sz,
2068 caps->srqwqe_hop_num, 1, &caps->srqwqe_buf_pg_sz,
2069 &caps->srqwqe_ba_pg_sz, HEM_TYPE_SRQWQE);
2070 calc_pg_sz(caps->num_idx_segs, caps->idx_entry_sz,
2071 caps->idx_hop_num, 1, &caps->idx_buf_pg_sz,
2072 &caps->idx_ba_pg_sz, HEM_TYPE_IDX);
2073 }
2074
2075 /* GMV */
2076 caps->gmv_ba_pg_sz = 0;
2077 caps->gmv_buf_pg_sz = 0;
2078 }
2079
2080 /* Apply all loaded caps before setting to hardware */
apply_func_caps(struct hns_roce_dev * hr_dev)2081 static void apply_func_caps(struct hns_roce_dev *hr_dev)
2082 {
2083 #define MAX_GID_TBL_LEN 256
2084 struct hns_roce_caps *caps = &hr_dev->caps;
2085 struct hns_roce_v2_priv *priv = hr_dev->priv;
2086
2087 /* The following configurations don't need to be got from firmware. */
2088 caps->qpc_timer_entry_sz = HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ;
2089 caps->cqc_timer_entry_sz = HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ;
2090 caps->mtt_entry_sz = HNS_ROCE_V2_MTT_ENTRY_SZ;
2091
2092 caps->pbl_hop_num = HNS_ROCE_PBL_HOP_NUM;
2093 caps->qpc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
2094 caps->cqc_timer_hop_num = HNS_ROCE_HOP_NUM_0;
2095
2096 caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS;
2097 caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS;
2098
2099 if (!caps->num_comp_vectors)
2100 caps->num_comp_vectors =
2101 min_t(u32, caps->eqc_bt_num - HNS_ROCE_V2_AEQE_VEC_NUM,
2102 (u32)priv->handle->rinfo.num_vectors -
2103 (HNS_ROCE_V2_AEQE_VEC_NUM + HNS_ROCE_V2_ABNORMAL_VEC_NUM));
2104
2105 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {
2106 caps->eqe_hop_num = HNS_ROCE_V3_EQE_HOP_NUM;
2107 caps->ceqe_size = HNS_ROCE_V3_EQE_SIZE;
2108 caps->aeqe_size = HNS_ROCE_V3_EQE_SIZE;
2109
2110 /* The following configurations will be overwritten */
2111 caps->qpc_sz = HNS_ROCE_V3_QPC_SZ;
2112 caps->cqe_sz = HNS_ROCE_V3_CQE_SIZE;
2113 caps->sccc_sz = HNS_ROCE_V3_SCCC_SZ;
2114
2115 /* The following configurations are not got from firmware */
2116 caps->gmv_entry_sz = HNS_ROCE_V3_GMV_ENTRY_SZ;
2117
2118 caps->gmv_hop_num = HNS_ROCE_HOP_NUM_0;
2119
2120 /* It's meaningless to support excessively large gid_table_len,
2121 * as the type of sgid_index in kernel struct ib_global_route
2122 * and userspace struct ibv_global_route are u8/uint8_t (0-255).
2123 */
2124 caps->gid_table_len[0] = min_t(u32, MAX_GID_TBL_LEN,
2125 caps->gmv_bt_num *
2126 (HNS_HW_PAGE_SIZE / caps->gmv_entry_sz));
2127
2128 caps->gmv_entry_num = caps->gmv_bt_num * (HNS_HW_PAGE_SIZE /
2129 caps->gmv_entry_sz);
2130 } else {
2131 u32 func_num = max_t(u32, 1, hr_dev->func_num);
2132
2133 caps->eqe_hop_num = HNS_ROCE_V2_EQE_HOP_NUM;
2134 caps->ceqe_size = HNS_ROCE_CEQE_SIZE;
2135 caps->aeqe_size = HNS_ROCE_AEQE_SIZE;
2136 caps->gid_table_len[0] /= func_num;
2137 }
2138
2139 if (hr_dev->is_vf) {
2140 caps->default_aeq_arm_st = 0x3;
2141 caps->default_ceq_arm_st = 0x3;
2142 caps->default_ceq_max_cnt = 0x1;
2143 caps->default_ceq_period = 0x10;
2144 caps->default_aeq_max_cnt = 0x1;
2145 caps->default_aeq_period = 0x10;
2146 }
2147
2148 set_hem_page_size(hr_dev);
2149 }
2150
hns_roce_query_caps(struct hns_roce_dev * hr_dev)2151 static int hns_roce_query_caps(struct hns_roce_dev *hr_dev)
2152 {
2153 struct hns_roce_cmq_desc desc[HNS_ROCE_QUERY_PF_CAPS_CMD_NUM];
2154 struct hns_roce_caps *caps = &hr_dev->caps;
2155 struct hns_roce_query_pf_caps_a *resp_a;
2156 struct hns_roce_query_pf_caps_b *resp_b;
2157 struct hns_roce_query_pf_caps_c *resp_c;
2158 struct hns_roce_query_pf_caps_d *resp_d;
2159 struct hns_roce_query_pf_caps_e *resp_e;
2160 enum hns_roce_opcode_type cmd;
2161 int ctx_hop_num;
2162 int pbl_hop_num;
2163 int ret;
2164 int i;
2165
2166 cmd = hr_dev->is_vf ? HNS_ROCE_OPC_QUERY_VF_CAPS_NUM :
2167 HNS_ROCE_OPC_QUERY_PF_CAPS_NUM;
2168
2169 for (i = 0; i < HNS_ROCE_QUERY_PF_CAPS_CMD_NUM; i++) {
2170 hns_roce_cmq_setup_basic_desc(&desc[i], cmd, true);
2171 if (i < (HNS_ROCE_QUERY_PF_CAPS_CMD_NUM - 1))
2172 desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
2173 else
2174 desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
2175 }
2176
2177 ret = hns_roce_cmq_send(hr_dev, desc, HNS_ROCE_QUERY_PF_CAPS_CMD_NUM);
2178 if (ret)
2179 return ret;
2180
2181 resp_a = (struct hns_roce_query_pf_caps_a *)desc[0].data;
2182 resp_b = (struct hns_roce_query_pf_caps_b *)desc[1].data;
2183 resp_c = (struct hns_roce_query_pf_caps_c *)desc[2].data;
2184 resp_d = (struct hns_roce_query_pf_caps_d *)desc[3].data;
2185 resp_e = (struct hns_roce_query_pf_caps_e *)desc[4].data;
2186
2187 caps->local_ca_ack_delay = resp_a->local_ca_ack_delay;
2188 caps->max_sq_sg = le16_to_cpu(resp_a->max_sq_sg);
2189 caps->max_sq_inline = le16_to_cpu(resp_a->max_sq_inline);
2190 caps->max_rq_sg = le16_to_cpu(resp_a->max_rq_sg);
2191 caps->max_rq_sg = roundup_pow_of_two(caps->max_rq_sg);
2192 caps->max_srq_sges = le16_to_cpu(resp_a->max_srq_sges);
2193 caps->max_srq_sges = roundup_pow_of_two(caps->max_srq_sges);
2194 caps->num_aeq_vectors = resp_a->num_aeq_vectors;
2195 caps->num_other_vectors = resp_a->num_other_vectors;
2196 caps->max_sq_desc_sz = resp_a->max_sq_desc_sz;
2197 caps->max_rq_desc_sz = resp_a->max_rq_desc_sz;
2198
2199 caps->mtpt_entry_sz = resp_b->mtpt_entry_sz;
2200 caps->irrl_entry_sz = resp_b->irrl_entry_sz;
2201 caps->trrl_entry_sz = resp_b->trrl_entry_sz;
2202 caps->cqc_entry_sz = resp_b->cqc_entry_sz;
2203 caps->srqc_entry_sz = resp_b->srqc_entry_sz;
2204 caps->idx_entry_sz = resp_b->idx_entry_sz;
2205 caps->sccc_sz = resp_b->sccc_sz;
2206 caps->max_mtu = resp_b->max_mtu;
2207 caps->min_cqes = resp_b->min_cqes;
2208 caps->min_wqes = resp_b->min_wqes;
2209 caps->page_size_cap = le32_to_cpu(resp_b->page_size_cap);
2210 caps->pkey_table_len[0] = resp_b->pkey_table_len;
2211 caps->phy_num_uars = resp_b->phy_num_uars;
2212 ctx_hop_num = resp_b->ctx_hop_num;
2213 pbl_hop_num = resp_b->pbl_hop_num;
2214
2215 caps->num_pds = 1 << hr_reg_read(resp_c, PF_CAPS_C_NUM_PDS);
2216
2217 caps->flags = hr_reg_read(resp_c, PF_CAPS_C_CAP_FLAGS);
2218 caps->flags |= le16_to_cpu(resp_d->cap_flags_ex) <<
2219 HNS_ROCE_CAP_FLAGS_EX_SHIFT;
2220
2221 caps->num_cqs = 1 << hr_reg_read(resp_c, PF_CAPS_C_NUM_CQS);
2222 caps->gid_table_len[0] = hr_reg_read(resp_c, PF_CAPS_C_MAX_GID);
2223 caps->max_cqes = 1 << hr_reg_read(resp_c, PF_CAPS_C_CQ_DEPTH);
2224 caps->num_xrcds = 1 << hr_reg_read(resp_c, PF_CAPS_C_NUM_XRCDS);
2225 caps->num_mtpts = 1 << hr_reg_read(resp_c, PF_CAPS_C_NUM_MRWS);
2226 caps->num_qps = 1 << hr_reg_read(resp_c, PF_CAPS_C_NUM_QPS);
2227 caps->max_qp_init_rdma = hr_reg_read(resp_c, PF_CAPS_C_MAX_ORD);
2228 caps->max_qp_dest_rdma = caps->max_qp_init_rdma;
2229 caps->max_wqes = 1 << le16_to_cpu(resp_c->sq_depth);
2230
2231 caps->num_srqs = 1 << hr_reg_read(resp_d, PF_CAPS_D_NUM_SRQS);
2232 caps->cong_cap = hr_reg_read(resp_d, PF_CAPS_D_CONG_CAP);
2233 caps->max_srq_wrs = 1 << le16_to_cpu(resp_d->srq_depth);
2234 caps->ceqe_depth = 1 << hr_reg_read(resp_d, PF_CAPS_D_CEQ_DEPTH);
2235 caps->num_comp_vectors = hr_reg_read(resp_d, PF_CAPS_D_NUM_CEQS);
2236 caps->aeqe_depth = 1 << hr_reg_read(resp_d, PF_CAPS_D_AEQ_DEPTH);
2237 caps->default_cong_type = hr_reg_read(resp_d, PF_CAPS_D_DEFAULT_ALG);
2238 caps->reserved_pds = hr_reg_read(resp_d, PF_CAPS_D_RSV_PDS);
2239 caps->num_uars = 1 << hr_reg_read(resp_d, PF_CAPS_D_NUM_UARS);
2240 caps->reserved_qps = hr_reg_read(resp_d, PF_CAPS_D_RSV_QPS);
2241 caps->reserved_uars = hr_reg_read(resp_d, PF_CAPS_D_RSV_UARS);
2242
2243 caps->reserved_mrws = hr_reg_read(resp_e, PF_CAPS_E_RSV_MRWS);
2244 caps->chunk_sz = 1 << hr_reg_read(resp_e, PF_CAPS_E_CHUNK_SIZE_SHIFT);
2245 caps->reserved_cqs = hr_reg_read(resp_e, PF_CAPS_E_RSV_CQS);
2246 caps->reserved_xrcds = hr_reg_read(resp_e, PF_CAPS_E_RSV_XRCDS);
2247 caps->reserved_srqs = hr_reg_read(resp_e, PF_CAPS_E_RSV_SRQS);
2248 caps->reserved_lkey = hr_reg_read(resp_e, PF_CAPS_E_RSV_LKEYS);
2249
2250 caps->qpc_hop_num = ctx_hop_num;
2251 caps->sccc_hop_num = ctx_hop_num;
2252 caps->srqc_hop_num = ctx_hop_num;
2253 caps->cqc_hop_num = ctx_hop_num;
2254 caps->mpt_hop_num = ctx_hop_num;
2255 caps->mtt_hop_num = pbl_hop_num;
2256 caps->cqe_hop_num = pbl_hop_num;
2257 caps->srqwqe_hop_num = pbl_hop_num;
2258 caps->idx_hop_num = pbl_hop_num;
2259 caps->wqe_sq_hop_num = hr_reg_read(resp_d, PF_CAPS_D_SQWQE_HOP_NUM);
2260 caps->wqe_sge_hop_num = hr_reg_read(resp_d, PF_CAPS_D_EX_SGE_HOP_NUM);
2261 caps->wqe_rq_hop_num = hr_reg_read(resp_d, PF_CAPS_D_RQWQE_HOP_NUM);
2262
2263 if (!(caps->page_size_cap & PAGE_SIZE))
2264 caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED;
2265
2266 if (!hr_dev->is_vf) {
2267 caps->cqe_sz = resp_a->cqe_sz;
2268 caps->qpc_sz = le16_to_cpu(resp_b->qpc_sz);
2269 caps->default_aeq_arm_st =
2270 hr_reg_read(resp_d, PF_CAPS_D_AEQ_ARM_ST);
2271 caps->default_ceq_arm_st =
2272 hr_reg_read(resp_d, PF_CAPS_D_CEQ_ARM_ST);
2273 caps->default_ceq_max_cnt = le16_to_cpu(resp_e->ceq_max_cnt);
2274 caps->default_ceq_period = le16_to_cpu(resp_e->ceq_period);
2275 caps->default_aeq_max_cnt = le16_to_cpu(resp_e->aeq_max_cnt);
2276 caps->default_aeq_period = le16_to_cpu(resp_e->aeq_period);
2277 }
2278
2279 return 0;
2280 }
2281
config_hem_entry_size(struct hns_roce_dev * hr_dev,u32 type,u32 val)2282 static int config_hem_entry_size(struct hns_roce_dev *hr_dev, u32 type, u32 val)
2283 {
2284 struct hns_roce_cmq_desc desc;
2285 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;
2286
2287 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_ENTRY_SIZE,
2288 false);
2289
2290 hr_reg_write(req, CFG_HEM_ENTRY_SIZE_TYPE, type);
2291 hr_reg_write(req, CFG_HEM_ENTRY_SIZE_VALUE, val);
2292
2293 return hns_roce_cmq_send(hr_dev, &desc, 1);
2294 }
2295
hns_roce_config_entry_size(struct hns_roce_dev * hr_dev)2296 static int hns_roce_config_entry_size(struct hns_roce_dev *hr_dev)
2297 {
2298 struct hns_roce_caps *caps = &hr_dev->caps;
2299 int ret;
2300
2301 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08)
2302 return 0;
2303
2304 ret = config_hem_entry_size(hr_dev, HNS_ROCE_CFG_QPC_SIZE,
2305 caps->qpc_sz);
2306 if (ret) {
2307 dev_err(hr_dev->dev, "failed to cfg qpc sz, ret = %d.\n", ret);
2308 return ret;
2309 }
2310
2311 ret = config_hem_entry_size(hr_dev, HNS_ROCE_CFG_SCCC_SIZE,
2312 caps->sccc_sz);
2313 if (ret)
2314 dev_err(hr_dev->dev, "failed to cfg sccc sz, ret = %d.\n", ret);
2315
2316 return ret;
2317 }
2318
hns_roce_v2_vf_profile(struct hns_roce_dev * hr_dev)2319 static int hns_roce_v2_vf_profile(struct hns_roce_dev *hr_dev)
2320 {
2321 struct device *dev = hr_dev->dev;
2322 int ret;
2323
2324 hr_dev->func_num = 1;
2325
2326 ret = hns_roce_query_caps(hr_dev);
2327 if (ret) {
2328 dev_err(dev, "failed to query VF caps, ret = %d.\n", ret);
2329 return ret;
2330 }
2331
2332 ret = hns_roce_query_vf_resource(hr_dev);
2333 if (ret) {
2334 dev_err(dev, "failed to query VF resource, ret = %d.\n", ret);
2335 return ret;
2336 }
2337
2338 apply_func_caps(hr_dev);
2339
2340 ret = hns_roce_v2_set_bt(hr_dev);
2341 if (ret)
2342 dev_err(dev, "failed to config VF BA table, ret = %d.\n", ret);
2343
2344 return ret;
2345 }
2346
hns_roce_v2_pf_profile(struct hns_roce_dev * hr_dev)2347 static int hns_roce_v2_pf_profile(struct hns_roce_dev *hr_dev)
2348 {
2349 struct device *dev = hr_dev->dev;
2350 int ret;
2351
2352 ret = hns_roce_query_func_info(hr_dev);
2353 if (ret) {
2354 dev_err(dev, "failed to query func info, ret = %d.\n", ret);
2355 return ret;
2356 }
2357
2358 ret = hns_roce_config_global_param(hr_dev);
2359 if (ret) {
2360 dev_err(dev, "failed to config global param, ret = %d.\n", ret);
2361 return ret;
2362 }
2363
2364 ret = hns_roce_set_vf_switch_param(hr_dev);
2365 if (ret) {
2366 dev_err(dev, "failed to set switch param, ret = %d.\n", ret);
2367 return ret;
2368 }
2369
2370 ret = hns_roce_query_caps(hr_dev);
2371 if (ret) {
2372 dev_err(dev, "failed to query PF caps, ret = %d.\n", ret);
2373 return ret;
2374 }
2375
2376 ret = hns_roce_query_pf_resource(hr_dev);
2377 if (ret) {
2378 dev_err(dev, "failed to query pf resource, ret = %d.\n", ret);
2379 return ret;
2380 }
2381
2382 apply_func_caps(hr_dev);
2383
2384 ret = hns_roce_alloc_vf_resource(hr_dev);
2385 if (ret) {
2386 dev_err(dev, "failed to alloc vf resource, ret = %d.\n", ret);
2387 return ret;
2388 }
2389
2390 ret = hns_roce_v2_set_bt(hr_dev);
2391 if (ret) {
2392 dev_err(dev, "failed to config BA table, ret = %d.\n", ret);
2393 return ret;
2394 }
2395
2396 /* Configure the size of QPC, SCCC, etc. */
2397 return hns_roce_config_entry_size(hr_dev);
2398 }
2399
hns_roce_v2_profile(struct hns_roce_dev * hr_dev)2400 static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)
2401 {
2402 struct device *dev = hr_dev->dev;
2403 int ret;
2404
2405 ret = hns_roce_cmq_query_hw_info(hr_dev);
2406 if (ret) {
2407 dev_err(dev, "failed to query hardware info, ret = %d.\n", ret);
2408 return ret;
2409 }
2410
2411 ret = hns_roce_query_fw_ver(hr_dev);
2412 if (ret) {
2413 dev_err(dev, "failed to query firmware info, ret = %d.\n", ret);
2414 return ret;
2415 }
2416
2417 hr_dev->vendor_part_id = hr_dev->pci_dev->device;
2418 hr_dev->sys_image_guid = be64_to_cpu(hr_dev->ib_dev.node_guid);
2419
2420 if (hr_dev->is_vf)
2421 return hns_roce_v2_vf_profile(hr_dev);
2422 else
2423 return hns_roce_v2_pf_profile(hr_dev);
2424 }
2425
config_llm_table(struct hns_roce_buf * data_buf,void * cfg_buf)2426 static void config_llm_table(struct hns_roce_buf *data_buf, void *cfg_buf)
2427 {
2428 u32 i, next_ptr, page_num;
2429 __le64 *entry = cfg_buf;
2430 dma_addr_t addr;
2431 u64 val;
2432
2433 page_num = data_buf->npages;
2434 for (i = 0; i < page_num; i++) {
2435 addr = hns_roce_buf_page(data_buf, i);
2436 if (i == (page_num - 1))
2437 next_ptr = 0;
2438 else
2439 next_ptr = i + 1;
2440
2441 val = HNS_ROCE_EXT_LLM_ENTRY(addr, (u64)next_ptr);
2442 entry[i] = cpu_to_le64(val);
2443 }
2444 }
2445
set_llm_cfg_to_hw(struct hns_roce_dev * hr_dev,struct hns_roce_link_table * table)2446 static int set_llm_cfg_to_hw(struct hns_roce_dev *hr_dev,
2447 struct hns_roce_link_table *table)
2448 {
2449 struct hns_roce_cmq_desc desc[2];
2450 struct hns_roce_cmq_req *r_a = (struct hns_roce_cmq_req *)desc[0].data;
2451 struct hns_roce_cmq_req *r_b = (struct hns_roce_cmq_req *)desc[1].data;
2452 struct hns_roce_buf *buf = table->buf;
2453 enum hns_roce_opcode_type opcode;
2454 dma_addr_t addr;
2455
2456 opcode = HNS_ROCE_OPC_CFG_EXT_LLM;
2457 hns_roce_cmq_setup_basic_desc(&desc[0], opcode, false);
2458 desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
2459 hns_roce_cmq_setup_basic_desc(&desc[1], opcode, false);
2460
2461 hr_reg_write(r_a, CFG_LLM_A_BA_L, lower_32_bits(table->table.map));
2462 hr_reg_write(r_a, CFG_LLM_A_BA_H, upper_32_bits(table->table.map));
2463 hr_reg_write(r_a, CFG_LLM_A_DEPTH, buf->npages);
2464 hr_reg_write(r_a, CFG_LLM_A_PGSZ, to_hr_hw_page_shift(buf->page_shift));
2465 hr_reg_enable(r_a, CFG_LLM_A_INIT_EN);
2466
2467 addr = to_hr_hw_page_addr(hns_roce_buf_page(buf, 0));
2468 hr_reg_write(r_a, CFG_LLM_A_HEAD_BA_L, lower_32_bits(addr));
2469 hr_reg_write(r_a, CFG_LLM_A_HEAD_BA_H, upper_32_bits(addr));
2470 hr_reg_write(r_a, CFG_LLM_A_HEAD_NXTPTR, 1);
2471 hr_reg_write(r_a, CFG_LLM_A_HEAD_PTR, 0);
2472
2473 addr = to_hr_hw_page_addr(hns_roce_buf_page(buf, buf->npages - 1));
2474 hr_reg_write(r_b, CFG_LLM_B_TAIL_BA_L, lower_32_bits(addr));
2475 hr_reg_write(r_b, CFG_LLM_B_TAIL_BA_H, upper_32_bits(addr));
2476 hr_reg_write(r_b, CFG_LLM_B_TAIL_PTR, buf->npages - 1);
2477
2478 return hns_roce_cmq_send(hr_dev, desc, 2);
2479 }
2480
2481 static struct hns_roce_link_table *
alloc_link_table_buf(struct hns_roce_dev * hr_dev)2482 alloc_link_table_buf(struct hns_roce_dev *hr_dev)
2483 {
2484 u16 total_sl = hr_dev->caps.sl_num * hr_dev->func_num;
2485 struct hns_roce_v2_priv *priv = hr_dev->priv;
2486 struct hns_roce_link_table *link_tbl;
2487 u32 pg_shift, size, min_size;
2488
2489 link_tbl = &priv->ext_llm;
2490 pg_shift = hr_dev->caps.llm_buf_pg_sz + PAGE_SHIFT;
2491 size = hr_dev->caps.num_qps * hr_dev->func_num *
2492 HNS_ROCE_V2_EXT_LLM_ENTRY_SZ;
2493 min_size = HNS_ROCE_EXT_LLM_MIN_PAGES(total_sl) << pg_shift;
2494
2495 /* Alloc data table */
2496 size = max(size, min_size);
2497 link_tbl->buf = hns_roce_buf_alloc(hr_dev, size, pg_shift, 0);
2498 if (IS_ERR(link_tbl->buf))
2499 return ERR_PTR(-ENOMEM);
2500
2501 /* Alloc config table */
2502 size = link_tbl->buf->npages * sizeof(u64);
2503 link_tbl->table.buf = dma_alloc_coherent(hr_dev->dev, size,
2504 &link_tbl->table.map,
2505 GFP_KERNEL);
2506 if (!link_tbl->table.buf) {
2507 hns_roce_buf_free(hr_dev, link_tbl->buf);
2508 return ERR_PTR(-ENOMEM);
2509 }
2510
2511 return link_tbl;
2512 }
2513
free_link_table_buf(struct hns_roce_dev * hr_dev,struct hns_roce_link_table * tbl)2514 static void free_link_table_buf(struct hns_roce_dev *hr_dev,
2515 struct hns_roce_link_table *tbl)
2516 {
2517 if (tbl->buf) {
2518 u32 size = tbl->buf->npages * sizeof(u64);
2519
2520 dma_free_coherent(hr_dev->dev, size, tbl->table.buf,
2521 tbl->table.map);
2522 }
2523
2524 hns_roce_buf_free(hr_dev, tbl->buf);
2525 }
2526
hns_roce_init_link_table(struct hns_roce_dev * hr_dev)2527 static int hns_roce_init_link_table(struct hns_roce_dev *hr_dev)
2528 {
2529 struct hns_roce_link_table *link_tbl;
2530 int ret;
2531
2532 link_tbl = alloc_link_table_buf(hr_dev);
2533 if (IS_ERR(link_tbl))
2534 return -ENOMEM;
2535
2536 if (WARN_ON(link_tbl->buf->npages > HNS_ROCE_V2_EXT_LLM_MAX_DEPTH)) {
2537 ret = -EINVAL;
2538 goto err_alloc;
2539 }
2540
2541 config_llm_table(link_tbl->buf, link_tbl->table.buf);
2542 ret = set_llm_cfg_to_hw(hr_dev, link_tbl);
2543 if (ret)
2544 goto err_alloc;
2545
2546 return 0;
2547
2548 err_alloc:
2549 free_link_table_buf(hr_dev, link_tbl);
2550 return ret;
2551 }
2552
hns_roce_free_link_table(struct hns_roce_dev * hr_dev)2553 static void hns_roce_free_link_table(struct hns_roce_dev *hr_dev)
2554 {
2555 struct hns_roce_v2_priv *priv = hr_dev->priv;
2556
2557 free_link_table_buf(hr_dev, &priv->ext_llm);
2558 }
2559
free_dip_entry(struct hns_roce_dev * hr_dev)2560 static void free_dip_entry(struct hns_roce_dev *hr_dev)
2561 {
2562 struct hns_roce_dip *hr_dip;
2563 unsigned long idx;
2564
2565 xa_lock(&hr_dev->qp_table.dip_xa);
2566
2567 xa_for_each(&hr_dev->qp_table.dip_xa, idx, hr_dip) {
2568 __xa_erase(&hr_dev->qp_table.dip_xa, hr_dip->dip_idx);
2569 kfree(hr_dip);
2570 }
2571
2572 xa_unlock(&hr_dev->qp_table.dip_xa);
2573 }
2574
free_mr_init_pd(struct hns_roce_dev * hr_dev)2575 static struct ib_pd *free_mr_init_pd(struct hns_roce_dev *hr_dev)
2576 {
2577 struct hns_roce_v2_priv *priv = hr_dev->priv;
2578 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
2579 struct ib_device *ibdev = &hr_dev->ib_dev;
2580 struct hns_roce_pd *hr_pd;
2581 struct ib_pd *pd;
2582
2583 hr_pd = kzalloc(sizeof(*hr_pd), GFP_KERNEL);
2584 if (ZERO_OR_NULL_PTR(hr_pd))
2585 return NULL;
2586 pd = &hr_pd->ibpd;
2587 pd->device = ibdev;
2588
2589 if (hns_roce_alloc_pd(pd, NULL)) {
2590 ibdev_err(ibdev, "failed to create pd for free mr.\n");
2591 kfree(hr_pd);
2592 return NULL;
2593 }
2594 free_mr->rsv_pd = to_hr_pd(pd);
2595 free_mr->rsv_pd->ibpd.device = &hr_dev->ib_dev;
2596 free_mr->rsv_pd->ibpd.uobject = NULL;
2597 free_mr->rsv_pd->ibpd.__internal_mr = NULL;
2598 atomic_set(&free_mr->rsv_pd->ibpd.usecnt, 0);
2599
2600 return pd;
2601 }
2602
free_mr_init_cq(struct hns_roce_dev * hr_dev)2603 static struct ib_cq *free_mr_init_cq(struct hns_roce_dev *hr_dev)
2604 {
2605 struct hns_roce_v2_priv *priv = hr_dev->priv;
2606 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
2607 struct ib_device *ibdev = &hr_dev->ib_dev;
2608 struct ib_cq_init_attr cq_init_attr = {};
2609 struct hns_roce_cq *hr_cq;
2610 struct ib_cq *cq;
2611
2612 cq_init_attr.cqe = HNS_ROCE_FREE_MR_USED_CQE_NUM;
2613
2614 hr_cq = kzalloc(sizeof(*hr_cq), GFP_KERNEL);
2615 if (ZERO_OR_NULL_PTR(hr_cq))
2616 return NULL;
2617
2618 cq = &hr_cq->ib_cq;
2619 cq->device = ibdev;
2620
2621 if (hns_roce_create_cq(cq, &cq_init_attr, NULL)) {
2622 ibdev_err(ibdev, "failed to create cq for free mr.\n");
2623 kfree(hr_cq);
2624 return NULL;
2625 }
2626 free_mr->rsv_cq = to_hr_cq(cq);
2627 free_mr->rsv_cq->ib_cq.device = &hr_dev->ib_dev;
2628 free_mr->rsv_cq->ib_cq.uobject = NULL;
2629 free_mr->rsv_cq->ib_cq.comp_handler = NULL;
2630 free_mr->rsv_cq->ib_cq.event_handler = NULL;
2631 free_mr->rsv_cq->ib_cq.cq_context = NULL;
2632 atomic_set(&free_mr->rsv_cq->ib_cq.usecnt, 0);
2633
2634 return cq;
2635 }
2636
free_mr_init_qp(struct hns_roce_dev * hr_dev,struct ib_cq * cq,struct ib_qp_init_attr * init_attr,int i)2637 static int free_mr_init_qp(struct hns_roce_dev *hr_dev, struct ib_cq *cq,
2638 struct ib_qp_init_attr *init_attr, int i)
2639 {
2640 struct hns_roce_v2_priv *priv = hr_dev->priv;
2641 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
2642 struct ib_device *ibdev = &hr_dev->ib_dev;
2643 struct hns_roce_qp *hr_qp;
2644 struct ib_qp *qp;
2645 int ret;
2646
2647 hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL);
2648 if (ZERO_OR_NULL_PTR(hr_qp))
2649 return -ENOMEM;
2650
2651 qp = &hr_qp->ibqp;
2652 qp->device = ibdev;
2653
2654 ret = hns_roce_create_qp(qp, init_attr, NULL);
2655 if (ret) {
2656 ibdev_err(ibdev, "failed to create qp for free mr.\n");
2657 kfree(hr_qp);
2658 return ret;
2659 }
2660
2661 free_mr->rsv_qp[i] = hr_qp;
2662 free_mr->rsv_qp[i]->ibqp.recv_cq = cq;
2663 free_mr->rsv_qp[i]->ibqp.send_cq = cq;
2664
2665 return 0;
2666 }
2667
free_mr_exit(struct hns_roce_dev * hr_dev)2668 static void free_mr_exit(struct hns_roce_dev *hr_dev)
2669 {
2670 struct hns_roce_v2_priv *priv = hr_dev->priv;
2671 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
2672 struct ib_qp *qp;
2673 int i;
2674
2675 for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) {
2676 if (free_mr->rsv_qp[i]) {
2677 qp = &free_mr->rsv_qp[i]->ibqp;
2678 hns_roce_v2_destroy_qp(qp, NULL);
2679 kfree(free_mr->rsv_qp[i]);
2680 free_mr->rsv_qp[i] = NULL;
2681 }
2682 }
2683
2684 if (free_mr->rsv_cq) {
2685 hns_roce_destroy_cq(&free_mr->rsv_cq->ib_cq, NULL);
2686 kfree(free_mr->rsv_cq);
2687 free_mr->rsv_cq = NULL;
2688 }
2689
2690 if (free_mr->rsv_pd) {
2691 hns_roce_dealloc_pd(&free_mr->rsv_pd->ibpd, NULL);
2692 kfree(free_mr->rsv_pd);
2693 free_mr->rsv_pd = NULL;
2694 }
2695
2696 mutex_destroy(&free_mr->mutex);
2697 }
2698
free_mr_alloc_res(struct hns_roce_dev * hr_dev)2699 static int free_mr_alloc_res(struct hns_roce_dev *hr_dev)
2700 {
2701 struct hns_roce_v2_priv *priv = hr_dev->priv;
2702 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
2703 struct ib_qp_init_attr qp_init_attr = {};
2704 struct ib_pd *pd;
2705 struct ib_cq *cq;
2706 int ret;
2707 int i;
2708
2709 pd = free_mr_init_pd(hr_dev);
2710 if (!pd)
2711 return -ENOMEM;
2712
2713 cq = free_mr_init_cq(hr_dev);
2714 if (!cq) {
2715 ret = -ENOMEM;
2716 goto create_failed_cq;
2717 }
2718
2719 qp_init_attr.qp_type = IB_QPT_RC;
2720 qp_init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;
2721 qp_init_attr.send_cq = cq;
2722 qp_init_attr.recv_cq = cq;
2723 for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) {
2724 qp_init_attr.cap.max_send_wr = HNS_ROCE_FREE_MR_USED_SQWQE_NUM;
2725 qp_init_attr.cap.max_send_sge = HNS_ROCE_FREE_MR_USED_SQSGE_NUM;
2726 qp_init_attr.cap.max_recv_wr = HNS_ROCE_FREE_MR_USED_RQWQE_NUM;
2727 qp_init_attr.cap.max_recv_sge = HNS_ROCE_FREE_MR_USED_RQSGE_NUM;
2728
2729 ret = free_mr_init_qp(hr_dev, cq, &qp_init_attr, i);
2730 if (ret)
2731 goto create_failed_qp;
2732 }
2733
2734 return 0;
2735
2736 create_failed_qp:
2737 for (i--; i >= 0; i--) {
2738 hns_roce_v2_destroy_qp(&free_mr->rsv_qp[i]->ibqp, NULL);
2739 kfree(free_mr->rsv_qp[i]);
2740 }
2741 hns_roce_destroy_cq(cq, NULL);
2742 kfree(cq);
2743
2744 create_failed_cq:
2745 hns_roce_dealloc_pd(pd, NULL);
2746 kfree(pd);
2747
2748 return ret;
2749 }
2750
free_mr_modify_rsv_qp(struct hns_roce_dev * hr_dev,struct ib_qp_attr * attr,int sl_num)2751 static int free_mr_modify_rsv_qp(struct hns_roce_dev *hr_dev,
2752 struct ib_qp_attr *attr, int sl_num)
2753 {
2754 struct hns_roce_v2_priv *priv = hr_dev->priv;
2755 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
2756 struct ib_device *ibdev = &hr_dev->ib_dev;
2757 struct hns_roce_qp *hr_qp;
2758 int loopback;
2759 int mask;
2760 int ret;
2761
2762 hr_qp = to_hr_qp(&free_mr->rsv_qp[sl_num]->ibqp);
2763 hr_qp->free_mr_en = 1;
2764 hr_qp->ibqp.device = ibdev;
2765 hr_qp->ibqp.qp_type = IB_QPT_RC;
2766
2767 mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS;
2768 attr->qp_state = IB_QPS_INIT;
2769 attr->port_num = 1;
2770 attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE;
2771 ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_INIT,
2772 IB_QPS_INIT, NULL);
2773 if (ret) {
2774 ibdev_err_ratelimited(ibdev, "failed to modify qp to init, ret = %d.\n",
2775 ret);
2776 return ret;
2777 }
2778
2779 loopback = hr_dev->loop_idc;
2780 /* Set qpc lbi = 1 incidate loopback IO */
2781 hr_dev->loop_idc = 1;
2782
2783 mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU | IB_QP_DEST_QPN |
2784 IB_QP_RQ_PSN | IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER;
2785 attr->qp_state = IB_QPS_RTR;
2786 attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2787 attr->path_mtu = IB_MTU_256;
2788 attr->dest_qp_num = hr_qp->qpn;
2789 attr->rq_psn = HNS_ROCE_FREE_MR_USED_PSN;
2790
2791 rdma_ah_set_sl(&attr->ah_attr, (u8)sl_num);
2792
2793 ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_INIT,
2794 IB_QPS_RTR, NULL);
2795 hr_dev->loop_idc = loopback;
2796 if (ret) {
2797 ibdev_err(ibdev, "failed to modify qp to rtr, ret = %d.\n",
2798 ret);
2799 return ret;
2800 }
2801
2802 mask = IB_QP_STATE | IB_QP_SQ_PSN | IB_QP_RETRY_CNT | IB_QP_TIMEOUT |
2803 IB_QP_RNR_RETRY | IB_QP_MAX_QP_RD_ATOMIC;
2804 attr->qp_state = IB_QPS_RTS;
2805 attr->sq_psn = HNS_ROCE_FREE_MR_USED_PSN;
2806 attr->retry_cnt = HNS_ROCE_FREE_MR_USED_QP_RETRY_CNT;
2807 attr->timeout = HNS_ROCE_FREE_MR_USED_QP_TIMEOUT;
2808 ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, attr, mask, IB_QPS_RTR,
2809 IB_QPS_RTS, NULL);
2810 if (ret)
2811 ibdev_err(ibdev, "failed to modify qp to rts, ret = %d.\n",
2812 ret);
2813
2814 return ret;
2815 }
2816
free_mr_modify_qp(struct hns_roce_dev * hr_dev)2817 static int free_mr_modify_qp(struct hns_roce_dev *hr_dev)
2818 {
2819 struct hns_roce_v2_priv *priv = hr_dev->priv;
2820 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
2821 struct ib_qp_attr attr = {};
2822 int ret;
2823 int i;
2824
2825 rdma_ah_set_grh(&attr.ah_attr, NULL, 0, 0, 1, 0);
2826 rdma_ah_set_static_rate(&attr.ah_attr, 3);
2827 rdma_ah_set_port_num(&attr.ah_attr, 1);
2828
2829 for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) {
2830 ret = free_mr_modify_rsv_qp(hr_dev, &attr, i);
2831 if (ret)
2832 return ret;
2833 }
2834
2835 return 0;
2836 }
2837
free_mr_init(struct hns_roce_dev * hr_dev)2838 static int free_mr_init(struct hns_roce_dev *hr_dev)
2839 {
2840 struct hns_roce_v2_priv *priv = hr_dev->priv;
2841 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
2842 int ret;
2843
2844 mutex_init(&free_mr->mutex);
2845
2846 ret = free_mr_alloc_res(hr_dev);
2847 if (ret) {
2848 mutex_destroy(&free_mr->mutex);
2849 return ret;
2850 }
2851
2852 ret = free_mr_modify_qp(hr_dev);
2853 if (ret)
2854 goto err_modify_qp;
2855
2856 return 0;
2857
2858 err_modify_qp:
2859 free_mr_exit(hr_dev);
2860
2861 return ret;
2862 }
2863
get_hem_table(struct hns_roce_dev * hr_dev)2864 static int get_hem_table(struct hns_roce_dev *hr_dev)
2865 {
2866 unsigned int qpc_count;
2867 unsigned int cqc_count;
2868 unsigned int gmv_count;
2869 int ret;
2870 int i;
2871
2872 /* Alloc memory for source address table buffer space chunk */
2873 for (gmv_count = 0; gmv_count < hr_dev->caps.gmv_entry_num;
2874 gmv_count++) {
2875 ret = hns_roce_table_get(hr_dev, &hr_dev->gmv_table, gmv_count);
2876 if (ret)
2877 goto err_gmv_failed;
2878 }
2879
2880 if (hr_dev->is_vf)
2881 return 0;
2882
2883 /* Alloc memory for QPC Timer buffer space chunk */
2884 for (qpc_count = 0; qpc_count < hr_dev->caps.qpc_timer_bt_num;
2885 qpc_count++) {
2886 ret = hns_roce_table_get(hr_dev, &hr_dev->qpc_timer_table,
2887 qpc_count);
2888 if (ret) {
2889 dev_err(hr_dev->dev, "QPC Timer get failed\n");
2890 goto err_qpc_timer_failed;
2891 }
2892 }
2893
2894 /* Alloc memory for CQC Timer buffer space chunk */
2895 for (cqc_count = 0; cqc_count < hr_dev->caps.cqc_timer_bt_num;
2896 cqc_count++) {
2897 ret = hns_roce_table_get(hr_dev, &hr_dev->cqc_timer_table,
2898 cqc_count);
2899 if (ret) {
2900 dev_err(hr_dev->dev, "CQC Timer get failed\n");
2901 goto err_cqc_timer_failed;
2902 }
2903 }
2904
2905 return 0;
2906
2907 err_cqc_timer_failed:
2908 for (i = 0; i < cqc_count; i++)
2909 hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i);
2910
2911 err_qpc_timer_failed:
2912 for (i = 0; i < qpc_count; i++)
2913 hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i);
2914
2915 err_gmv_failed:
2916 for (i = 0; i < gmv_count; i++)
2917 hns_roce_table_put(hr_dev, &hr_dev->gmv_table, i);
2918
2919 return ret;
2920 }
2921
put_hem_table(struct hns_roce_dev * hr_dev)2922 static void put_hem_table(struct hns_roce_dev *hr_dev)
2923 {
2924 int i;
2925
2926 for (i = 0; i < hr_dev->caps.gmv_entry_num; i++)
2927 hns_roce_table_put(hr_dev, &hr_dev->gmv_table, i);
2928
2929 if (hr_dev->is_vf)
2930 return;
2931
2932 for (i = 0; i < hr_dev->caps.qpc_timer_bt_num; i++)
2933 hns_roce_table_put(hr_dev, &hr_dev->qpc_timer_table, i);
2934
2935 for (i = 0; i < hr_dev->caps.cqc_timer_bt_num; i++)
2936 hns_roce_table_put(hr_dev, &hr_dev->cqc_timer_table, i);
2937 }
2938
hns_roce_v2_init(struct hns_roce_dev * hr_dev)2939 static int hns_roce_v2_init(struct hns_roce_dev *hr_dev)
2940 {
2941 int ret;
2942
2943 /* The hns ROCEE requires the extdb info to be cleared before using */
2944 ret = hns_roce_clear_extdb_list_info(hr_dev);
2945 if (ret)
2946 return ret;
2947
2948 ret = get_hem_table(hr_dev);
2949 if (ret)
2950 return ret;
2951
2952 if (hr_dev->is_vf)
2953 return 0;
2954
2955 ret = hns_roce_init_link_table(hr_dev);
2956 if (ret) {
2957 dev_err(hr_dev->dev, "failed to init llm, ret = %d.\n", ret);
2958 goto err_llm_init_failed;
2959 }
2960
2961 return 0;
2962
2963 err_llm_init_failed:
2964 put_hem_table(hr_dev);
2965
2966 return ret;
2967 }
2968
hns_roce_v2_exit(struct hns_roce_dev * hr_dev)2969 static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev)
2970 {
2971 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08)
2972 free_mr_exit(hr_dev);
2973
2974 hns_roce_function_clear(hr_dev);
2975
2976 if (!hr_dev->is_vf)
2977 hns_roce_free_link_table(hr_dev);
2978
2979 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP09)
2980 free_dip_entry(hr_dev);
2981 }
2982
hns_roce_mbox_post(struct hns_roce_dev * hr_dev,struct hns_roce_mbox_msg * mbox_msg)2983 static int hns_roce_mbox_post(struct hns_roce_dev *hr_dev,
2984 struct hns_roce_mbox_msg *mbox_msg)
2985 {
2986 struct hns_roce_cmq_desc desc;
2987 struct hns_roce_post_mbox *mb = (struct hns_roce_post_mbox *)desc.data;
2988
2989 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_POST_MB, false);
2990
2991 mb->in_param_l = cpu_to_le32(mbox_msg->in_param);
2992 mb->in_param_h = cpu_to_le32(mbox_msg->in_param >> 32);
2993 mb->out_param_l = cpu_to_le32(mbox_msg->out_param);
2994 mb->out_param_h = cpu_to_le32(mbox_msg->out_param >> 32);
2995 mb->cmd_tag = cpu_to_le32(mbox_msg->tag << 8 | mbox_msg->cmd);
2996 mb->token_event_en = cpu_to_le32(mbox_msg->event_en << 16 |
2997 mbox_msg->token);
2998
2999 return hns_roce_cmq_send(hr_dev, &desc, 1);
3000 }
3001
v2_wait_mbox_complete(struct hns_roce_dev * hr_dev,u32 timeout,u8 * complete_status)3002 static int v2_wait_mbox_complete(struct hns_roce_dev *hr_dev, u32 timeout,
3003 u8 *complete_status)
3004 {
3005 struct hns_roce_mbox_status *mb_st;
3006 struct hns_roce_cmq_desc desc;
3007 unsigned long end;
3008 int ret = -EBUSY;
3009 u32 status;
3010 bool busy;
3011
3012 mb_st = (struct hns_roce_mbox_status *)desc.data;
3013 end = msecs_to_jiffies(timeout) + jiffies;
3014 while (v2_chk_mbox_is_avail(hr_dev, &busy)) {
3015 if (hr_dev->cmd.state == HNS_ROCE_CMDQ_STATE_FATAL_ERR)
3016 return -EIO;
3017
3018 status = 0;
3019 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_MB_ST,
3020 true);
3021 ret = __hns_roce_cmq_send(hr_dev, &desc, 1);
3022 if (!ret) {
3023 status = le32_to_cpu(mb_st->mb_status_hw_run);
3024 /* No pending message exists in ROCEE mbox. */
3025 if (!(status & MB_ST_HW_RUN_M))
3026 break;
3027 } else if (!v2_chk_mbox_is_avail(hr_dev, &busy)) {
3028 break;
3029 }
3030
3031 if (time_after(jiffies, end)) {
3032 dev_err_ratelimited(hr_dev->dev,
3033 "failed to wait mbox status 0x%x\n",
3034 status);
3035 return -ETIMEDOUT;
3036 }
3037
3038 cond_resched();
3039 ret = -EBUSY;
3040 }
3041
3042 if (!ret) {
3043 *complete_status = (u8)(status & MB_ST_COMPLETE_M);
3044 } else if (!v2_chk_mbox_is_avail(hr_dev, &busy)) {
3045 /* Ignore all errors if the mbox is unavailable. */
3046 ret = 0;
3047 *complete_status = MB_ST_COMPLETE_M;
3048 }
3049
3050 return ret;
3051 }
3052
v2_post_mbox(struct hns_roce_dev * hr_dev,struct hns_roce_mbox_msg * mbox_msg)3053 static int v2_post_mbox(struct hns_roce_dev *hr_dev,
3054 struct hns_roce_mbox_msg *mbox_msg)
3055 {
3056 u8 status = 0;
3057 int ret;
3058
3059 /* Waiting for the mbox to be idle */
3060 ret = v2_wait_mbox_complete(hr_dev, HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS,
3061 &status);
3062 if (unlikely(ret)) {
3063 dev_err_ratelimited(hr_dev->dev,
3064 "failed to check post mbox status = 0x%x, ret = %d.\n",
3065 status, ret);
3066 return ret;
3067 }
3068
3069 /* Post new message to mbox */
3070 ret = hns_roce_mbox_post(hr_dev, mbox_msg);
3071 if (ret)
3072 dev_err_ratelimited(hr_dev->dev,
3073 "failed to post mailbox, ret = %d.\n", ret);
3074
3075 return ret;
3076 }
3077
v2_poll_mbox_done(struct hns_roce_dev * hr_dev)3078 static int v2_poll_mbox_done(struct hns_roce_dev *hr_dev)
3079 {
3080 u8 status = 0;
3081 int ret;
3082
3083 ret = v2_wait_mbox_complete(hr_dev, HNS_ROCE_CMD_TIMEOUT_MSECS,
3084 &status);
3085 if (!ret) {
3086 if (status != MB_ST_COMPLETE_SUCC)
3087 return -EBUSY;
3088 } else {
3089 dev_err_ratelimited(hr_dev->dev,
3090 "failed to check mbox status = 0x%x, ret = %d.\n",
3091 status, ret);
3092 }
3093
3094 return ret;
3095 }
3096
copy_gid(void * dest,const union ib_gid * gid)3097 static void copy_gid(void *dest, const union ib_gid *gid)
3098 {
3099 #define GID_SIZE 4
3100 const union ib_gid *src = gid;
3101 __le32 (*p)[GID_SIZE] = dest;
3102 int i;
3103
3104 if (!gid)
3105 src = &zgid;
3106
3107 for (i = 0; i < GID_SIZE; i++)
3108 (*p)[i] = cpu_to_le32(*(u32 *)&src->raw[i * sizeof(u32)]);
3109 }
3110
config_sgid_table(struct hns_roce_dev * hr_dev,int gid_index,const union ib_gid * gid,enum hns_roce_sgid_type sgid_type)3111 static int config_sgid_table(struct hns_roce_dev *hr_dev,
3112 int gid_index, const union ib_gid *gid,
3113 enum hns_roce_sgid_type sgid_type)
3114 {
3115 struct hns_roce_cmq_desc desc;
3116 struct hns_roce_cfg_sgid_tb *sgid_tb =
3117 (struct hns_roce_cfg_sgid_tb *)desc.data;
3118
3119 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SGID_TB, false);
3120
3121 hr_reg_write(sgid_tb, CFG_SGID_TB_TABLE_IDX, gid_index);
3122 hr_reg_write(sgid_tb, CFG_SGID_TB_VF_SGID_TYPE, sgid_type);
3123
3124 copy_gid(&sgid_tb->vf_sgid_l, gid);
3125
3126 return hns_roce_cmq_send(hr_dev, &desc, 1);
3127 }
3128
config_gmv_table(struct hns_roce_dev * hr_dev,int gid_index,const union ib_gid * gid,enum hns_roce_sgid_type sgid_type,const struct ib_gid_attr * attr)3129 static int config_gmv_table(struct hns_roce_dev *hr_dev,
3130 int gid_index, const union ib_gid *gid,
3131 enum hns_roce_sgid_type sgid_type,
3132 const struct ib_gid_attr *attr)
3133 {
3134 struct hns_roce_cmq_desc desc[2];
3135 struct hns_roce_cfg_gmv_tb_a *tb_a =
3136 (struct hns_roce_cfg_gmv_tb_a *)desc[0].data;
3137 struct hns_roce_cfg_gmv_tb_b *tb_b =
3138 (struct hns_roce_cfg_gmv_tb_b *)desc[1].data;
3139
3140 u16 vlan_id = VLAN_CFI_MASK;
3141 u8 mac[ETH_ALEN] = {};
3142 int ret;
3143
3144 if (gid) {
3145 ret = rdma_read_gid_l2_fields(attr, &vlan_id, mac);
3146 if (ret)
3147 return ret;
3148 }
3149
3150 hns_roce_cmq_setup_basic_desc(&desc[0], HNS_ROCE_OPC_CFG_GMV_TBL, false);
3151 desc[0].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
3152
3153 hns_roce_cmq_setup_basic_desc(&desc[1], HNS_ROCE_OPC_CFG_GMV_TBL, false);
3154
3155 copy_gid(&tb_a->vf_sgid_l, gid);
3156
3157 hr_reg_write(tb_a, GMV_TB_A_VF_SGID_TYPE, sgid_type);
3158 hr_reg_write(tb_a, GMV_TB_A_VF_VLAN_EN, vlan_id < VLAN_CFI_MASK);
3159 hr_reg_write(tb_a, GMV_TB_A_VF_VLAN_ID, vlan_id);
3160
3161 tb_b->vf_smac_l = cpu_to_le32(*(u32 *)mac);
3162
3163 hr_reg_write(tb_b, GMV_TB_B_SMAC_H, *(u16 *)&mac[4]);
3164 hr_reg_write(tb_b, GMV_TB_B_SGID_IDX, gid_index);
3165
3166 return hns_roce_cmq_send(hr_dev, desc, 2);
3167 }
3168
hns_roce_v2_set_gid(struct hns_roce_dev * hr_dev,int gid_index,const union ib_gid * gid,const struct ib_gid_attr * attr)3169 static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, int gid_index,
3170 const union ib_gid *gid,
3171 const struct ib_gid_attr *attr)
3172 {
3173 enum hns_roce_sgid_type sgid_type = GID_TYPE_FLAG_ROCE_V1;
3174 int ret;
3175
3176 if (gid) {
3177 if (attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {
3178 if (ipv6_addr_v4mapped((void *)gid))
3179 sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV4;
3180 else
3181 sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV6;
3182 } else if (attr->gid_type == IB_GID_TYPE_ROCE) {
3183 sgid_type = GID_TYPE_FLAG_ROCE_V1;
3184 }
3185 }
3186
3187 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
3188 ret = config_gmv_table(hr_dev, gid_index, gid, sgid_type, attr);
3189 else
3190 ret = config_sgid_table(hr_dev, gid_index, gid, sgid_type);
3191
3192 if (ret)
3193 ibdev_err(&hr_dev->ib_dev, "failed to set gid, ret = %d!\n",
3194 ret);
3195
3196 return ret;
3197 }
3198
hns_roce_v2_set_mac(struct hns_roce_dev * hr_dev,u8 phy_port,const u8 * addr)3199 static int hns_roce_v2_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port,
3200 const u8 *addr)
3201 {
3202 struct hns_roce_cmq_desc desc;
3203 struct hns_roce_cfg_smac_tb *smac_tb =
3204 (struct hns_roce_cfg_smac_tb *)desc.data;
3205 u16 reg_smac_h;
3206 u32 reg_smac_l;
3207
3208 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SMAC_TB, false);
3209
3210 reg_smac_l = *(u32 *)(&addr[0]);
3211 reg_smac_h = *(u16 *)(&addr[4]);
3212
3213 hr_reg_write(smac_tb, CFG_SMAC_TB_IDX, phy_port);
3214 hr_reg_write(smac_tb, CFG_SMAC_TB_VF_SMAC_H, reg_smac_h);
3215 smac_tb->vf_smac_l = cpu_to_le32(reg_smac_l);
3216
3217 return hns_roce_cmq_send(hr_dev, &desc, 1);
3218 }
3219
set_mtpt_pbl(struct hns_roce_dev * hr_dev,struct hns_roce_v2_mpt_entry * mpt_entry,struct hns_roce_mr * mr)3220 static int set_mtpt_pbl(struct hns_roce_dev *hr_dev,
3221 struct hns_roce_v2_mpt_entry *mpt_entry,
3222 struct hns_roce_mr *mr)
3223 {
3224 u64 pages[HNS_ROCE_V2_MAX_INNER_MTPT_NUM] = { 0 };
3225 struct ib_device *ibdev = &hr_dev->ib_dev;
3226 dma_addr_t pbl_ba;
3227 int ret;
3228 int i;
3229
3230 ret = hns_roce_mtr_find(hr_dev, &mr->pbl_mtr, 0, pages,
3231 min_t(int, ARRAY_SIZE(pages), mr->npages));
3232 if (ret) {
3233 ibdev_err(ibdev, "failed to find PBL mtr, ret = %d.\n", ret);
3234 return ret;
3235 }
3236
3237 /* Aligned to the hardware address access unit */
3238 for (i = 0; i < ARRAY_SIZE(pages); i++)
3239 pages[i] >>= MPT_PBL_BUF_ADDR_S;
3240
3241 pbl_ba = hns_roce_get_mtr_ba(&mr->pbl_mtr);
3242
3243 mpt_entry->pbl_size = cpu_to_le32(mr->npages);
3244 mpt_entry->pbl_ba_l = cpu_to_le32(pbl_ba >> MPT_PBL_BA_ADDR_S);
3245 hr_reg_write(mpt_entry, MPT_PBL_BA_H,
3246 upper_32_bits(pbl_ba >> MPT_PBL_BA_ADDR_S));
3247
3248 mpt_entry->pa0_l = cpu_to_le32(lower_32_bits(pages[0]));
3249 hr_reg_write(mpt_entry, MPT_PA0_H, upper_32_bits(pages[0]));
3250
3251 mpt_entry->pa1_l = cpu_to_le32(lower_32_bits(pages[1]));
3252 hr_reg_write(mpt_entry, MPT_PA1_H, upper_32_bits(pages[1]));
3253 hr_reg_write(mpt_entry, MPT_PBL_BUF_PG_SZ,
3254 to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift));
3255
3256 return 0;
3257 }
3258
hns_roce_v2_write_mtpt(struct hns_roce_dev * hr_dev,void * mb_buf,struct hns_roce_mr * mr)3259 static int hns_roce_v2_write_mtpt(struct hns_roce_dev *hr_dev,
3260 void *mb_buf, struct hns_roce_mr *mr)
3261 {
3262 struct hns_roce_v2_mpt_entry *mpt_entry;
3263
3264 mpt_entry = mb_buf;
3265 memset(mpt_entry, 0, sizeof(*mpt_entry));
3266
3267 hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_VALID);
3268 hr_reg_write(mpt_entry, MPT_PD, mr->pd);
3269
3270 hr_reg_write_bool(mpt_entry, MPT_BIND_EN,
3271 mr->access & IB_ACCESS_MW_BIND);
3272 hr_reg_write_bool(mpt_entry, MPT_ATOMIC_EN,
3273 mr->access & IB_ACCESS_REMOTE_ATOMIC);
3274 hr_reg_write_bool(mpt_entry, MPT_RR_EN,
3275 mr->access & IB_ACCESS_REMOTE_READ);
3276 hr_reg_write_bool(mpt_entry, MPT_RW_EN,
3277 mr->access & IB_ACCESS_REMOTE_WRITE);
3278 hr_reg_write_bool(mpt_entry, MPT_LW_EN,
3279 mr->access & IB_ACCESS_LOCAL_WRITE);
3280
3281 mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size));
3282 mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size));
3283 mpt_entry->lkey = cpu_to_le32(mr->key);
3284 mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova));
3285 mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova));
3286
3287 if (mr->type != MR_TYPE_MR)
3288 hr_reg_enable(mpt_entry, MPT_PA);
3289
3290 if (mr->type == MR_TYPE_DMA)
3291 return 0;
3292
3293 if (mr->pbl_hop_num != HNS_ROCE_HOP_NUM_0)
3294 hr_reg_write(mpt_entry, MPT_PBL_HOP_NUM, mr->pbl_hop_num);
3295
3296 hr_reg_write(mpt_entry, MPT_PBL_BA_PG_SZ,
3297 to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.ba_pg_shift));
3298 hr_reg_enable(mpt_entry, MPT_INNER_PA_VLD);
3299
3300 return set_mtpt_pbl(hr_dev, mpt_entry, mr);
3301 }
3302
hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev * hr_dev,struct hns_roce_mr * mr,int flags,void * mb_buf)3303 static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev,
3304 struct hns_roce_mr *mr, int flags,
3305 void *mb_buf)
3306 {
3307 struct hns_roce_v2_mpt_entry *mpt_entry = mb_buf;
3308 u32 mr_access_flags = mr->access;
3309 int ret = 0;
3310
3311 hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_VALID);
3312 hr_reg_write(mpt_entry, MPT_PD, mr->pd);
3313
3314 if (flags & IB_MR_REREG_ACCESS) {
3315 hr_reg_write(mpt_entry, MPT_BIND_EN,
3316 (mr_access_flags & IB_ACCESS_MW_BIND ? 1 : 0));
3317 hr_reg_write(mpt_entry, MPT_ATOMIC_EN,
3318 mr_access_flags & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
3319 hr_reg_write(mpt_entry, MPT_RR_EN,
3320 mr_access_flags & IB_ACCESS_REMOTE_READ ? 1 : 0);
3321 hr_reg_write(mpt_entry, MPT_RW_EN,
3322 mr_access_flags & IB_ACCESS_REMOTE_WRITE ? 1 : 0);
3323 hr_reg_write(mpt_entry, MPT_LW_EN,
3324 mr_access_flags & IB_ACCESS_LOCAL_WRITE ? 1 : 0);
3325 }
3326
3327 if (flags & IB_MR_REREG_TRANS) {
3328 mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova));
3329 mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova));
3330 mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size));
3331 mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size));
3332
3333 ret = set_mtpt_pbl(hr_dev, mpt_entry, mr);
3334 }
3335
3336 return ret;
3337 }
3338
hns_roce_v2_frmr_write_mtpt(void * mb_buf,struct hns_roce_mr * mr)3339 static int hns_roce_v2_frmr_write_mtpt(void *mb_buf, struct hns_roce_mr *mr)
3340 {
3341 dma_addr_t pbl_ba = hns_roce_get_mtr_ba(&mr->pbl_mtr);
3342 struct hns_roce_v2_mpt_entry *mpt_entry;
3343
3344 mpt_entry = mb_buf;
3345 memset(mpt_entry, 0, sizeof(*mpt_entry));
3346
3347 hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_FREE);
3348 hr_reg_write(mpt_entry, MPT_PD, mr->pd);
3349
3350 hr_reg_enable(mpt_entry, MPT_RA_EN);
3351 hr_reg_enable(mpt_entry, MPT_R_INV_EN);
3352
3353 hr_reg_enable(mpt_entry, MPT_FRE);
3354 hr_reg_clear(mpt_entry, MPT_MR_MW);
3355 hr_reg_enable(mpt_entry, MPT_BPD);
3356 hr_reg_clear(mpt_entry, MPT_PA);
3357
3358 hr_reg_write(mpt_entry, MPT_PBL_HOP_NUM, 1);
3359 hr_reg_write(mpt_entry, MPT_PBL_BA_PG_SZ,
3360 to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.ba_pg_shift));
3361 hr_reg_write(mpt_entry, MPT_PBL_BUF_PG_SZ,
3362 to_hr_hw_page_shift(mr->pbl_mtr.hem_cfg.buf_pg_shift));
3363
3364 mpt_entry->pbl_size = cpu_to_le32(mr->npages);
3365
3366 mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(pbl_ba >>
3367 MPT_PBL_BA_ADDR_S));
3368 hr_reg_write(mpt_entry, MPT_PBL_BA_H,
3369 upper_32_bits(pbl_ba >> MPT_PBL_BA_ADDR_S));
3370
3371 return 0;
3372 }
3373
hns_roce_v2_mw_write_mtpt(void * mb_buf,struct hns_roce_mw * mw)3374 static int hns_roce_v2_mw_write_mtpt(void *mb_buf, struct hns_roce_mw *mw)
3375 {
3376 struct hns_roce_v2_mpt_entry *mpt_entry;
3377
3378 mpt_entry = mb_buf;
3379 memset(mpt_entry, 0, sizeof(*mpt_entry));
3380
3381 hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_FREE);
3382 hr_reg_write(mpt_entry, MPT_PD, mw->pdn);
3383
3384 hr_reg_enable(mpt_entry, MPT_R_INV_EN);
3385 hr_reg_enable(mpt_entry, MPT_LW_EN);
3386
3387 hr_reg_enable(mpt_entry, MPT_MR_MW);
3388 hr_reg_enable(mpt_entry, MPT_BPD);
3389 hr_reg_clear(mpt_entry, MPT_PA);
3390 hr_reg_write(mpt_entry, MPT_BQP,
3391 mw->ibmw.type == IB_MW_TYPE_1 ? 0 : 1);
3392
3393 mpt_entry->lkey = cpu_to_le32(mw->rkey);
3394
3395 hr_reg_write(mpt_entry, MPT_PBL_HOP_NUM,
3396 mw->pbl_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 :
3397 mw->pbl_hop_num);
3398 hr_reg_write(mpt_entry, MPT_PBL_BA_PG_SZ,
3399 mw->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
3400 hr_reg_write(mpt_entry, MPT_PBL_BUF_PG_SZ,
3401 mw->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
3402
3403 return 0;
3404 }
3405
free_mr_post_send_lp_wqe(struct hns_roce_qp * hr_qp)3406 static int free_mr_post_send_lp_wqe(struct hns_roce_qp *hr_qp)
3407 {
3408 struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp->ibqp.device);
3409 struct ib_device *ibdev = &hr_dev->ib_dev;
3410 const struct ib_send_wr *bad_wr;
3411 struct ib_rdma_wr rdma_wr = {};
3412 struct ib_send_wr *send_wr;
3413 int ret;
3414
3415 send_wr = &rdma_wr.wr;
3416 send_wr->opcode = IB_WR_RDMA_WRITE;
3417
3418 ret = hns_roce_v2_post_send(&hr_qp->ibqp, send_wr, &bad_wr);
3419 if (ret) {
3420 ibdev_err_ratelimited(ibdev, "failed to post wqe for free mr, ret = %d.\n",
3421 ret);
3422 return ret;
3423 }
3424
3425 return 0;
3426 }
3427
3428 static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries,
3429 struct ib_wc *wc);
3430
free_mr_send_cmd_to_hw(struct hns_roce_dev * hr_dev)3431 static void free_mr_send_cmd_to_hw(struct hns_roce_dev *hr_dev)
3432 {
3433 struct hns_roce_v2_priv *priv = hr_dev->priv;
3434 struct hns_roce_v2_free_mr *free_mr = &priv->free_mr;
3435 struct ib_wc wc[ARRAY_SIZE(free_mr->rsv_qp)];
3436 struct ib_device *ibdev = &hr_dev->ib_dev;
3437 struct hns_roce_qp *hr_qp;
3438 unsigned long end;
3439 int cqe_cnt = 0;
3440 int npolled;
3441 int ret;
3442 int i;
3443
3444 /*
3445 * If the device initialization is not complete or in the uninstall
3446 * process, then there is no need to execute free mr.
3447 */
3448 if (priv->handle->rinfo.reset_state == HNS_ROCE_STATE_RST_INIT ||
3449 priv->handle->rinfo.instance_state == HNS_ROCE_STATE_INIT ||
3450 hr_dev->state == HNS_ROCE_DEVICE_STATE_UNINIT)
3451 return;
3452
3453 mutex_lock(&free_mr->mutex);
3454
3455 for (i = 0; i < ARRAY_SIZE(free_mr->rsv_qp); i++) {
3456 hr_qp = free_mr->rsv_qp[i];
3457
3458 ret = free_mr_post_send_lp_wqe(hr_qp);
3459 if (ret) {
3460 ibdev_err_ratelimited(ibdev,
3461 "failed to send wqe (qp:0x%lx) for free mr, ret = %d.\n",
3462 hr_qp->qpn, ret);
3463 break;
3464 }
3465
3466 cqe_cnt++;
3467 }
3468
3469 end = msecs_to_jiffies(HNS_ROCE_V2_FREE_MR_TIMEOUT) + jiffies;
3470 while (cqe_cnt) {
3471 npolled = hns_roce_v2_poll_cq(&free_mr->rsv_cq->ib_cq, cqe_cnt, wc);
3472 if (npolled < 0) {
3473 ibdev_err_ratelimited(ibdev,
3474 "failed to poll cqe for free mr, remain %d cqe.\n",
3475 cqe_cnt);
3476 goto out;
3477 }
3478
3479 if (time_after(jiffies, end)) {
3480 ibdev_err_ratelimited(ibdev,
3481 "failed to poll cqe for free mr and timeout, remain %d cqe.\n",
3482 cqe_cnt);
3483 goto out;
3484 }
3485 cqe_cnt -= npolled;
3486 }
3487
3488 out:
3489 mutex_unlock(&free_mr->mutex);
3490 }
3491
hns_roce_v2_dereg_mr(struct hns_roce_dev * hr_dev)3492 static void hns_roce_v2_dereg_mr(struct hns_roce_dev *hr_dev)
3493 {
3494 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08)
3495 free_mr_send_cmd_to_hw(hr_dev);
3496 }
3497
get_cqe_v2(struct hns_roce_cq * hr_cq,int n)3498 static void *get_cqe_v2(struct hns_roce_cq *hr_cq, int n)
3499 {
3500 return hns_roce_buf_offset(hr_cq->mtr.kmem, n * hr_cq->cqe_size);
3501 }
3502
get_sw_cqe_v2(struct hns_roce_cq * hr_cq,unsigned int n)3503 static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, unsigned int n)
3504 {
3505 struct hns_roce_v2_cqe *cqe = get_cqe_v2(hr_cq, n & hr_cq->ib_cq.cqe);
3506
3507 /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */
3508 return (hr_reg_read(cqe, CQE_OWNER) ^ !!(n & hr_cq->cq_depth)) ? cqe :
3509 NULL;
3510 }
3511
update_cq_db(struct hns_roce_dev * hr_dev,struct hns_roce_cq * hr_cq)3512 static inline void update_cq_db(struct hns_roce_dev *hr_dev,
3513 struct hns_roce_cq *hr_cq)
3514 {
3515 if (likely(hr_cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB)) {
3516 *hr_cq->set_ci_db = hr_cq->cons_index & V2_CQ_DB_CONS_IDX_M;
3517 } else {
3518 struct hns_roce_v2_db cq_db = {};
3519
3520 hr_reg_write(&cq_db, DB_TAG, hr_cq->cqn);
3521 hr_reg_write(&cq_db, DB_CMD, HNS_ROCE_V2_CQ_DB);
3522 hr_reg_write(&cq_db, DB_CQ_CI, hr_cq->cons_index);
3523 hr_reg_write(&cq_db, DB_CQ_CMD_SN, 1);
3524
3525 hns_roce_write64(hr_dev, (__le32 *)&cq_db, hr_cq->db_reg);
3526 }
3527 }
3528
__hns_roce_v2_cq_clean(struct hns_roce_cq * hr_cq,u32 qpn,struct hns_roce_srq * srq)3529 static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
3530 struct hns_roce_srq *srq)
3531 {
3532 struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device);
3533 struct hns_roce_v2_cqe *cqe, *dest;
3534 u32 prod_index;
3535 int nfreed = 0;
3536 int wqe_index;
3537 u8 owner_bit;
3538
3539 for (prod_index = hr_cq->cons_index; get_sw_cqe_v2(hr_cq, prod_index);
3540 ++prod_index) {
3541 if (prod_index > hr_cq->cons_index + hr_cq->ib_cq.cqe)
3542 break;
3543 }
3544
3545 /*
3546 * Now backwards through the CQ, removing CQ entries
3547 * that match our QP by overwriting them with next entries.
3548 */
3549 while ((int) --prod_index - (int) hr_cq->cons_index >= 0) {
3550 cqe = get_cqe_v2(hr_cq, prod_index & hr_cq->ib_cq.cqe);
3551 if (hr_reg_read(cqe, CQE_LCL_QPN) == qpn) {
3552 if (srq && hr_reg_read(cqe, CQE_S_R)) {
3553 wqe_index = hr_reg_read(cqe, CQE_WQE_IDX);
3554 hns_roce_free_srq_wqe(srq, wqe_index);
3555 }
3556 ++nfreed;
3557 } else if (nfreed) {
3558 dest = get_cqe_v2(hr_cq, (prod_index + nfreed) &
3559 hr_cq->ib_cq.cqe);
3560 owner_bit = hr_reg_read(dest, CQE_OWNER);
3561 memcpy(dest, cqe, hr_cq->cqe_size);
3562 hr_reg_write(dest, CQE_OWNER, owner_bit);
3563 }
3564 }
3565
3566 if (nfreed) {
3567 hr_cq->cons_index += nfreed;
3568 update_cq_db(hr_dev, hr_cq);
3569 }
3570 }
3571
hns_roce_v2_cq_clean(struct hns_roce_cq * hr_cq,u32 qpn,struct hns_roce_srq * srq)3572 static void hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
3573 struct hns_roce_srq *srq)
3574 {
3575 spin_lock_irq(&hr_cq->lock);
3576 __hns_roce_v2_cq_clean(hr_cq, qpn, srq);
3577 spin_unlock_irq(&hr_cq->lock);
3578 }
3579
hns_roce_v2_write_cqc(struct hns_roce_dev * hr_dev,struct hns_roce_cq * hr_cq,void * mb_buf,u64 * mtts,dma_addr_t dma_handle)3580 static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev,
3581 struct hns_roce_cq *hr_cq, void *mb_buf,
3582 u64 *mtts, dma_addr_t dma_handle)
3583 {
3584 struct hns_roce_v2_cq_context *cq_context;
3585
3586 cq_context = mb_buf;
3587 memset(cq_context, 0, sizeof(*cq_context));
3588
3589 hr_reg_write(cq_context, CQC_CQ_ST, V2_CQ_STATE_VALID);
3590 hr_reg_write(cq_context, CQC_ARM_ST, NO_ARMED);
3591 hr_reg_write(cq_context, CQC_SHIFT, ilog2(hr_cq->cq_depth));
3592 hr_reg_write(cq_context, CQC_CEQN, hr_cq->vector);
3593 hr_reg_write(cq_context, CQC_CQN, hr_cq->cqn);
3594
3595 if (hr_cq->cqe_size == HNS_ROCE_V3_CQE_SIZE)
3596 hr_reg_write(cq_context, CQC_CQE_SIZE, CQE_SIZE_64B);
3597
3598 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_STASH)
3599 hr_reg_enable(cq_context, CQC_STASH);
3600
3601 hr_reg_write(cq_context, CQC_CQE_CUR_BLK_ADDR_L,
3602 to_hr_hw_page_addr(mtts[0]));
3603 hr_reg_write(cq_context, CQC_CQE_CUR_BLK_ADDR_H,
3604 upper_32_bits(to_hr_hw_page_addr(mtts[0])));
3605 hr_reg_write(cq_context, CQC_CQE_HOP_NUM, hr_dev->caps.cqe_hop_num ==
3606 HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev->caps.cqe_hop_num);
3607 hr_reg_write(cq_context, CQC_CQE_NEX_BLK_ADDR_L,
3608 to_hr_hw_page_addr(mtts[1]));
3609 hr_reg_write(cq_context, CQC_CQE_NEX_BLK_ADDR_H,
3610 upper_32_bits(to_hr_hw_page_addr(mtts[1])));
3611 hr_reg_write(cq_context, CQC_CQE_BAR_PG_SZ,
3612 to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.ba_pg_shift));
3613 hr_reg_write(cq_context, CQC_CQE_BUF_PG_SZ,
3614 to_hr_hw_page_shift(hr_cq->mtr.hem_cfg.buf_pg_shift));
3615 hr_reg_write(cq_context, CQC_CQE_BA_L, dma_handle >> CQC_CQE_BA_L_S);
3616 hr_reg_write(cq_context, CQC_CQE_BA_H, dma_handle >> CQC_CQE_BA_H_S);
3617 hr_reg_write_bool(cq_context, CQC_DB_RECORD_EN,
3618 hr_cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB);
3619 hr_reg_write(cq_context, CQC_CQE_DB_RECORD_ADDR_L,
3620 ((u32)hr_cq->db.dma) >> 1);
3621 hr_reg_write(cq_context, CQC_CQE_DB_RECORD_ADDR_H,
3622 hr_cq->db.dma >> CQC_CQE_DB_RECORD_ADDR_H_S);
3623 hr_reg_write(cq_context, CQC_CQ_MAX_CNT,
3624 HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM);
3625 hr_reg_write(cq_context, CQC_CQ_PERIOD,
3626 HNS_ROCE_V2_CQ_DEFAULT_INTERVAL);
3627 }
3628
hns_roce_v2_req_notify_cq(struct ib_cq * ibcq,enum ib_cq_notify_flags flags)3629 static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq,
3630 enum ib_cq_notify_flags flags)
3631 {
3632 struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device);
3633 struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
3634 struct hns_roce_v2_db cq_db = {};
3635 u32 notify_flag;
3636
3637 /*
3638 * flags = 0, then notify_flag : next
3639 * flags = 1, then notify flag : solocited
3640 */
3641 notify_flag = (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
3642 V2_CQ_DB_REQ_NOT : V2_CQ_DB_REQ_NOT_SOL;
3643
3644 hr_reg_write(&cq_db, DB_TAG, hr_cq->cqn);
3645 hr_reg_write(&cq_db, DB_CMD, HNS_ROCE_V2_CQ_DB_NOTIFY);
3646 hr_reg_write(&cq_db, DB_CQ_CI, hr_cq->cons_index);
3647 hr_reg_write(&cq_db, DB_CQ_CMD_SN, hr_cq->arm_sn);
3648 hr_reg_write(&cq_db, DB_CQ_NOTIFY, notify_flag);
3649
3650 hns_roce_write64(hr_dev, (__le32 *)&cq_db, hr_cq->db_reg);
3651
3652 return 0;
3653 }
3654
sw_comp(struct hns_roce_qp * hr_qp,struct hns_roce_wq * wq,int num_entries,struct ib_wc * wc)3655 static int sw_comp(struct hns_roce_qp *hr_qp, struct hns_roce_wq *wq,
3656 int num_entries, struct ib_wc *wc)
3657 {
3658 unsigned int left;
3659 int npolled = 0;
3660
3661 left = wq->head - wq->tail;
3662 if (left == 0)
3663 return 0;
3664
3665 left = min_t(unsigned int, (unsigned int)num_entries, left);
3666 while (npolled < left) {
3667 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
3668 wc->status = IB_WC_WR_FLUSH_ERR;
3669 wc->vendor_err = 0;
3670 wc->qp = &hr_qp->ibqp;
3671
3672 wq->tail++;
3673 wc++;
3674 npolled++;
3675 }
3676
3677 return npolled;
3678 }
3679
hns_roce_v2_sw_poll_cq(struct hns_roce_cq * hr_cq,int num_entries,struct ib_wc * wc)3680 static int hns_roce_v2_sw_poll_cq(struct hns_roce_cq *hr_cq, int num_entries,
3681 struct ib_wc *wc)
3682 {
3683 struct hns_roce_qp *hr_qp;
3684 int npolled = 0;
3685
3686 list_for_each_entry(hr_qp, &hr_cq->sq_list, sq_node) {
3687 npolled += sw_comp(hr_qp, &hr_qp->sq,
3688 num_entries - npolled, wc + npolled);
3689 if (npolled >= num_entries)
3690 goto out;
3691 }
3692
3693 list_for_each_entry(hr_qp, &hr_cq->rq_list, rq_node) {
3694 npolled += sw_comp(hr_qp, &hr_qp->rq,
3695 num_entries - npolled, wc + npolled);
3696 if (npolled >= num_entries)
3697 goto out;
3698 }
3699
3700 out:
3701 return npolled;
3702 }
3703
get_cqe_status(struct hns_roce_dev * hr_dev,struct hns_roce_qp * qp,struct hns_roce_cq * cq,struct hns_roce_v2_cqe * cqe,struct ib_wc * wc)3704 static void get_cqe_status(struct hns_roce_dev *hr_dev, struct hns_roce_qp *qp,
3705 struct hns_roce_cq *cq, struct hns_roce_v2_cqe *cqe,
3706 struct ib_wc *wc)
3707 {
3708 static const struct {
3709 u32 cqe_status;
3710 enum ib_wc_status wc_status;
3711 } map[] = {
3712 { HNS_ROCE_CQE_V2_SUCCESS, IB_WC_SUCCESS },
3713 { HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR, IB_WC_LOC_LEN_ERR },
3714 { HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR, IB_WC_LOC_QP_OP_ERR },
3715 { HNS_ROCE_CQE_V2_LOCAL_PROT_ERR, IB_WC_LOC_PROT_ERR },
3716 { HNS_ROCE_CQE_V2_WR_FLUSH_ERR, IB_WC_WR_FLUSH_ERR },
3717 { HNS_ROCE_CQE_V2_MW_BIND_ERR, IB_WC_MW_BIND_ERR },
3718 { HNS_ROCE_CQE_V2_BAD_RESP_ERR, IB_WC_BAD_RESP_ERR },
3719 { HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR, IB_WC_LOC_ACCESS_ERR },
3720 { HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR, IB_WC_REM_INV_REQ_ERR },
3721 { HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR, IB_WC_REM_ACCESS_ERR },
3722 { HNS_ROCE_CQE_V2_REMOTE_OP_ERR, IB_WC_REM_OP_ERR },
3723 { HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR,
3724 IB_WC_RETRY_EXC_ERR },
3725 { HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR, IB_WC_RNR_RETRY_EXC_ERR },
3726 { HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR, IB_WC_REM_ABORT_ERR },
3727 { HNS_ROCE_CQE_V2_GENERAL_ERR, IB_WC_GENERAL_ERR}
3728 };
3729
3730 u32 cqe_status = hr_reg_read(cqe, CQE_STATUS);
3731 int i;
3732
3733 wc->status = IB_WC_GENERAL_ERR;
3734 for (i = 0; i < ARRAY_SIZE(map); i++)
3735 if (cqe_status == map[i].cqe_status) {
3736 wc->status = map[i].wc_status;
3737 break;
3738 }
3739
3740 if (likely(wc->status == IB_WC_SUCCESS ||
3741 wc->status == IB_WC_WR_FLUSH_ERR))
3742 return;
3743
3744 ibdev_err_ratelimited(&hr_dev->ib_dev, "error cqe status 0x%x:\n",
3745 cqe_status);
3746 print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_NONE, 16, 4, cqe,
3747 cq->cqe_size, false);
3748 wc->vendor_err = hr_reg_read(cqe, CQE_SUB_STATUS);
3749
3750 /*
3751 * For hns ROCEE, GENERAL_ERR is an error type that is not defined in
3752 * the standard protocol, the driver must ignore it and needn't to set
3753 * the QP to an error state.
3754 */
3755 if (cqe_status == HNS_ROCE_CQE_V2_GENERAL_ERR)
3756 return;
3757
3758 flush_cqe(hr_dev, qp);
3759 }
3760
get_cur_qp(struct hns_roce_cq * hr_cq,struct hns_roce_v2_cqe * cqe,struct hns_roce_qp ** cur_qp)3761 static int get_cur_qp(struct hns_roce_cq *hr_cq, struct hns_roce_v2_cqe *cqe,
3762 struct hns_roce_qp **cur_qp)
3763 {
3764 struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device);
3765 struct hns_roce_qp *hr_qp = *cur_qp;
3766 u32 qpn;
3767
3768 qpn = hr_reg_read(cqe, CQE_LCL_QPN);
3769
3770 if (!hr_qp || qpn != hr_qp->qpn) {
3771 hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
3772 if (unlikely(!hr_qp)) {
3773 ibdev_err(&hr_dev->ib_dev,
3774 "CQ %06lx with entry for unknown QPN %06x\n",
3775 hr_cq->cqn, qpn);
3776 return -EINVAL;
3777 }
3778 *cur_qp = hr_qp;
3779 }
3780
3781 return 0;
3782 }
3783
3784 /*
3785 * mapped-value = 1 + real-value
3786 * The ib wc opcode's real value is start from 0, In order to distinguish
3787 * between initialized and uninitialized map values, we plus 1 to the actual
3788 * value when defining the mapping, so that the validity can be identified by
3789 * checking whether the mapped value is greater than 0.
3790 */
3791 #define HR_WC_OP_MAP(hr_key, ib_key) \
3792 [HNS_ROCE_V2_WQE_OP_ ## hr_key] = 1 + IB_WC_ ## ib_key
3793
3794 static const u32 wc_send_op_map[] = {
3795 HR_WC_OP_MAP(SEND, SEND),
3796 HR_WC_OP_MAP(SEND_WITH_INV, SEND),
3797 HR_WC_OP_MAP(SEND_WITH_IMM, SEND),
3798 HR_WC_OP_MAP(RDMA_READ, RDMA_READ),
3799 HR_WC_OP_MAP(RDMA_WRITE, RDMA_WRITE),
3800 HR_WC_OP_MAP(RDMA_WRITE_WITH_IMM, RDMA_WRITE),
3801 HR_WC_OP_MAP(ATOM_CMP_AND_SWAP, COMP_SWAP),
3802 HR_WC_OP_MAP(ATOM_FETCH_AND_ADD, FETCH_ADD),
3803 HR_WC_OP_MAP(ATOM_MSK_CMP_AND_SWAP, MASKED_COMP_SWAP),
3804 HR_WC_OP_MAP(ATOM_MSK_FETCH_AND_ADD, MASKED_FETCH_ADD),
3805 HR_WC_OP_MAP(FAST_REG_PMR, REG_MR),
3806 HR_WC_OP_MAP(BIND_MW, REG_MR),
3807 };
3808
to_ib_wc_send_op(u32 hr_opcode)3809 static int to_ib_wc_send_op(u32 hr_opcode)
3810 {
3811 if (hr_opcode >= ARRAY_SIZE(wc_send_op_map))
3812 return -EINVAL;
3813
3814 return wc_send_op_map[hr_opcode] ? wc_send_op_map[hr_opcode] - 1 :
3815 -EINVAL;
3816 }
3817
3818 static const u32 wc_recv_op_map[] = {
3819 HR_WC_OP_MAP(RDMA_WRITE_WITH_IMM, WITH_IMM),
3820 HR_WC_OP_MAP(SEND, RECV),
3821 HR_WC_OP_MAP(SEND_WITH_IMM, WITH_IMM),
3822 HR_WC_OP_MAP(SEND_WITH_INV, RECV),
3823 };
3824
to_ib_wc_recv_op(u32 hr_opcode)3825 static int to_ib_wc_recv_op(u32 hr_opcode)
3826 {
3827 if (hr_opcode >= ARRAY_SIZE(wc_recv_op_map))
3828 return -EINVAL;
3829
3830 return wc_recv_op_map[hr_opcode] ? wc_recv_op_map[hr_opcode] - 1 :
3831 -EINVAL;
3832 }
3833
fill_send_wc(struct ib_wc * wc,struct hns_roce_v2_cqe * cqe)3834 static void fill_send_wc(struct ib_wc *wc, struct hns_roce_v2_cqe *cqe)
3835 {
3836 u32 hr_opcode;
3837 int ib_opcode;
3838
3839 wc->wc_flags = 0;
3840
3841 hr_opcode = hr_reg_read(cqe, CQE_OPCODE);
3842 switch (hr_opcode) {
3843 case HNS_ROCE_V2_WQE_OP_RDMA_READ:
3844 wc->byte_len = le32_to_cpu(cqe->byte_cnt);
3845 break;
3846 case HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM:
3847 case HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM:
3848 wc->wc_flags |= IB_WC_WITH_IMM;
3849 break;
3850 case HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP:
3851 case HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD:
3852 case HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP:
3853 case HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD:
3854 wc->byte_len = 8;
3855 break;
3856 default:
3857 break;
3858 }
3859
3860 ib_opcode = to_ib_wc_send_op(hr_opcode);
3861 if (ib_opcode < 0)
3862 wc->status = IB_WC_GENERAL_ERR;
3863 else
3864 wc->opcode = ib_opcode;
3865 }
3866
fill_recv_wc(struct ib_wc * wc,struct hns_roce_v2_cqe * cqe)3867 static int fill_recv_wc(struct ib_wc *wc, struct hns_roce_v2_cqe *cqe)
3868 {
3869 u32 hr_opcode;
3870 int ib_opcode;
3871
3872 wc->byte_len = le32_to_cpu(cqe->byte_cnt);
3873
3874 hr_opcode = hr_reg_read(cqe, CQE_OPCODE);
3875 switch (hr_opcode) {
3876 case HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM:
3877 case HNS_ROCE_V2_OPCODE_SEND_WITH_IMM:
3878 wc->wc_flags = IB_WC_WITH_IMM;
3879 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(cqe->immtdata));
3880 break;
3881 case HNS_ROCE_V2_OPCODE_SEND_WITH_INV:
3882 wc->wc_flags = IB_WC_WITH_INVALIDATE;
3883 wc->ex.invalidate_rkey = le32_to_cpu(cqe->rkey);
3884 break;
3885 default:
3886 wc->wc_flags = 0;
3887 }
3888
3889 ib_opcode = to_ib_wc_recv_op(hr_opcode);
3890 if (ib_opcode < 0)
3891 wc->status = IB_WC_GENERAL_ERR;
3892 else
3893 wc->opcode = ib_opcode;
3894
3895 wc->sl = hr_reg_read(cqe, CQE_SL);
3896 wc->src_qp = hr_reg_read(cqe, CQE_RMT_QPN);
3897 wc->slid = 0;
3898 wc->wc_flags |= hr_reg_read(cqe, CQE_GRH) ? IB_WC_GRH : 0;
3899 wc->port_num = hr_reg_read(cqe, CQE_PORTN);
3900 wc->pkey_index = 0;
3901
3902 if (hr_reg_read(cqe, CQE_VID_VLD)) {
3903 wc->vlan_id = hr_reg_read(cqe, CQE_VID);
3904 wc->wc_flags |= IB_WC_WITH_VLAN;
3905 } else {
3906 wc->vlan_id = 0xffff;
3907 }
3908
3909 wc->network_hdr_type = hr_reg_read(cqe, CQE_PORT_TYPE);
3910
3911 return 0;
3912 }
3913
hns_roce_v2_poll_one(struct hns_roce_cq * hr_cq,struct hns_roce_qp ** cur_qp,struct ib_wc * wc)3914 static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,
3915 struct hns_roce_qp **cur_qp, struct ib_wc *wc)
3916 {
3917 struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device);
3918 struct hns_roce_qp *qp = *cur_qp;
3919 struct hns_roce_srq *srq = NULL;
3920 struct hns_roce_v2_cqe *cqe;
3921 struct hns_roce_wq *wq;
3922 int is_send;
3923 u16 wqe_idx;
3924 int ret;
3925
3926 cqe = get_sw_cqe_v2(hr_cq, hr_cq->cons_index);
3927 if (!cqe)
3928 return -EAGAIN;
3929
3930 ++hr_cq->cons_index;
3931 /* Memory barrier */
3932 rmb();
3933
3934 ret = get_cur_qp(hr_cq, cqe, &qp);
3935 if (ret)
3936 return ret;
3937
3938 wc->qp = &qp->ibqp;
3939 wc->vendor_err = 0;
3940
3941 wqe_idx = hr_reg_read(cqe, CQE_WQE_IDX);
3942
3943 is_send = !hr_reg_read(cqe, CQE_S_R);
3944 if (is_send) {
3945 wq = &qp->sq;
3946
3947 /* If sg_signal_bit is set, tail pointer will be updated to
3948 * the WQE corresponding to the current CQE.
3949 */
3950 if (qp->sq_signal_bits)
3951 wq->tail += (wqe_idx - (u16)wq->tail) &
3952 (wq->wqe_cnt - 1);
3953
3954 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
3955 ++wq->tail;
3956
3957 fill_send_wc(wc, cqe);
3958 } else {
3959 if (qp->ibqp.srq) {
3960 srq = to_hr_srq(qp->ibqp.srq);
3961 wc->wr_id = srq->wrid[wqe_idx];
3962 hns_roce_free_srq_wqe(srq, wqe_idx);
3963 } else {
3964 wq = &qp->rq;
3965 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
3966 ++wq->tail;
3967 }
3968
3969 ret = fill_recv_wc(wc, cqe);
3970 }
3971
3972 get_cqe_status(hr_dev, qp, hr_cq, cqe, wc);
3973 if (unlikely(wc->status != IB_WC_SUCCESS))
3974 return 0;
3975
3976 return ret;
3977 }
3978
hns_roce_v2_poll_cq(struct ib_cq * ibcq,int num_entries,struct ib_wc * wc)3979 static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries,
3980 struct ib_wc *wc)
3981 {
3982 struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device);
3983 struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
3984 struct hns_roce_qp *cur_qp = NULL;
3985 unsigned long flags;
3986 int npolled;
3987
3988 spin_lock_irqsave(&hr_cq->lock, flags);
3989
3990 /*
3991 * When the device starts to reset, the state is RST_DOWN. At this time,
3992 * there may still be some valid CQEs in the hardware that are not
3993 * polled. Therefore, it is not allowed to switch to the software mode
3994 * immediately. When the state changes to UNINIT, CQE no longer exists
3995 * in the hardware, and then switch to software mode.
3996 */
3997 if (hr_dev->state == HNS_ROCE_DEVICE_STATE_UNINIT) {
3998 npolled = hns_roce_v2_sw_poll_cq(hr_cq, num_entries, wc);
3999 goto out;
4000 }
4001
4002 for (npolled = 0; npolled < num_entries; ++npolled) {
4003 if (hns_roce_v2_poll_one(hr_cq, &cur_qp, wc + npolled))
4004 break;
4005 }
4006
4007 if (npolled)
4008 update_cq_db(hr_dev, hr_cq);
4009
4010 out:
4011 spin_unlock_irqrestore(&hr_cq->lock, flags);
4012
4013 return npolled;
4014 }
4015
get_op_for_set_hem(struct hns_roce_dev * hr_dev,u32 type,u32 step_idx,u8 * mbox_cmd)4016 static int get_op_for_set_hem(struct hns_roce_dev *hr_dev, u32 type,
4017 u32 step_idx, u8 *mbox_cmd)
4018 {
4019 u8 cmd;
4020
4021 switch (type) {
4022 case HEM_TYPE_QPC:
4023 cmd = HNS_ROCE_CMD_WRITE_QPC_BT0;
4024 break;
4025 case HEM_TYPE_MTPT:
4026 cmd = HNS_ROCE_CMD_WRITE_MPT_BT0;
4027 break;
4028 case HEM_TYPE_CQC:
4029 cmd = HNS_ROCE_CMD_WRITE_CQC_BT0;
4030 break;
4031 case HEM_TYPE_SRQC:
4032 cmd = HNS_ROCE_CMD_WRITE_SRQC_BT0;
4033 break;
4034 case HEM_TYPE_SCCC:
4035 cmd = HNS_ROCE_CMD_WRITE_SCCC_BT0;
4036 break;
4037 case HEM_TYPE_QPC_TIMER:
4038 cmd = HNS_ROCE_CMD_WRITE_QPC_TIMER_BT0;
4039 break;
4040 case HEM_TYPE_CQC_TIMER:
4041 cmd = HNS_ROCE_CMD_WRITE_CQC_TIMER_BT0;
4042 break;
4043 default:
4044 dev_warn(hr_dev->dev, "failed to check hem type %u.\n", type);
4045 return -EINVAL;
4046 }
4047
4048 *mbox_cmd = cmd + step_idx;
4049
4050 return 0;
4051 }
4052
config_gmv_ba_to_hw(struct hns_roce_dev * hr_dev,unsigned long obj,dma_addr_t base_addr)4053 static int config_gmv_ba_to_hw(struct hns_roce_dev *hr_dev, unsigned long obj,
4054 dma_addr_t base_addr)
4055 {
4056 struct hns_roce_cmq_desc desc;
4057 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;
4058 u32 idx = obj / (HNS_HW_PAGE_SIZE / hr_dev->caps.gmv_entry_sz);
4059 u64 addr = to_hr_hw_page_addr(base_addr);
4060
4061 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT, false);
4062
4063 hr_reg_write(req, CFG_GMV_BT_BA_L, lower_32_bits(addr));
4064 hr_reg_write(req, CFG_GMV_BT_BA_H, upper_32_bits(addr));
4065 hr_reg_write(req, CFG_GMV_BT_IDX, idx);
4066
4067 return hns_roce_cmq_send(hr_dev, &desc, 1);
4068 }
4069
set_hem_to_hw(struct hns_roce_dev * hr_dev,int obj,dma_addr_t base_addr,u32 hem_type,u32 step_idx)4070 static int set_hem_to_hw(struct hns_roce_dev *hr_dev, int obj,
4071 dma_addr_t base_addr, u32 hem_type, u32 step_idx)
4072 {
4073 int ret;
4074 u8 cmd;
4075
4076 if (unlikely(hem_type == HEM_TYPE_GMV))
4077 return config_gmv_ba_to_hw(hr_dev, obj, base_addr);
4078
4079 if (unlikely(hem_type == HEM_TYPE_SCCC && step_idx))
4080 return 0;
4081
4082 ret = get_op_for_set_hem(hr_dev, hem_type, step_idx, &cmd);
4083 if (ret < 0)
4084 return ret;
4085
4086 return config_hem_ba_to_hw(hr_dev, base_addr, cmd, obj);
4087 }
4088
hns_roce_v2_set_hem(struct hns_roce_dev * hr_dev,struct hns_roce_hem_table * table,int obj,u32 step_idx)4089 static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev,
4090 struct hns_roce_hem_table *table, int obj,
4091 u32 step_idx)
4092 {
4093 struct hns_roce_hem_mhop mhop;
4094 struct hns_roce_hem *hem;
4095 unsigned long mhop_obj = obj;
4096 int i, j, k;
4097 int ret = 0;
4098 u64 hem_idx = 0;
4099 u64 l1_idx = 0;
4100 u64 bt_ba = 0;
4101 u32 chunk_ba_num;
4102 u32 hop_num;
4103
4104 if (!hns_roce_check_whether_mhop(hr_dev, table->type))
4105 return 0;
4106
4107 hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
4108 i = mhop.l0_idx;
4109 j = mhop.l1_idx;
4110 k = mhop.l2_idx;
4111 hop_num = mhop.hop_num;
4112 chunk_ba_num = mhop.bt_chunk_size / 8;
4113
4114 if (hop_num == 2) {
4115 hem_idx = i * chunk_ba_num * chunk_ba_num + j * chunk_ba_num +
4116 k;
4117 l1_idx = i * chunk_ba_num + j;
4118 } else if (hop_num == 1) {
4119 hem_idx = i * chunk_ba_num + j;
4120 } else if (hop_num == HNS_ROCE_HOP_NUM_0) {
4121 hem_idx = i;
4122 }
4123
4124 if (table->type == HEM_TYPE_SCCC)
4125 obj = mhop.l0_idx;
4126
4127 if (check_whether_last_step(hop_num, step_idx)) {
4128 hem = table->hem[hem_idx];
4129
4130 ret = set_hem_to_hw(hr_dev, obj, hem->dma, table->type, step_idx);
4131 } else {
4132 if (step_idx == 0)
4133 bt_ba = table->bt_l0_dma_addr[i];
4134 else if (step_idx == 1 && hop_num == 2)
4135 bt_ba = table->bt_l1_dma_addr[l1_idx];
4136
4137 ret = set_hem_to_hw(hr_dev, obj, bt_ba, table->type, step_idx);
4138 }
4139
4140 return ret;
4141 }
4142
hns_roce_v2_clear_hem(struct hns_roce_dev * hr_dev,struct hns_roce_hem_table * table,int tag,u32 step_idx)4143 static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev,
4144 struct hns_roce_hem_table *table,
4145 int tag, u32 step_idx)
4146 {
4147 struct hns_roce_cmd_mailbox *mailbox;
4148 struct device *dev = hr_dev->dev;
4149 u8 cmd = 0xff;
4150 int ret;
4151
4152 if (!hns_roce_check_whether_mhop(hr_dev, table->type))
4153 return 0;
4154
4155 switch (table->type) {
4156 case HEM_TYPE_QPC:
4157 cmd = HNS_ROCE_CMD_DESTROY_QPC_BT0;
4158 break;
4159 case HEM_TYPE_MTPT:
4160 cmd = HNS_ROCE_CMD_DESTROY_MPT_BT0;
4161 break;
4162 case HEM_TYPE_CQC:
4163 cmd = HNS_ROCE_CMD_DESTROY_CQC_BT0;
4164 break;
4165 case HEM_TYPE_SRQC:
4166 cmd = HNS_ROCE_CMD_DESTROY_SRQC_BT0;
4167 break;
4168 case HEM_TYPE_SCCC:
4169 case HEM_TYPE_QPC_TIMER:
4170 case HEM_TYPE_CQC_TIMER:
4171 case HEM_TYPE_GMV:
4172 return 0;
4173 default:
4174 dev_warn(dev, "table %u not to be destroyed by mailbox!\n",
4175 table->type);
4176 return 0;
4177 }
4178
4179 cmd += step_idx;
4180
4181 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
4182 if (IS_ERR(mailbox))
4183 return PTR_ERR(mailbox);
4184
4185 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, cmd, tag);
4186
4187 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
4188 return ret;
4189 }
4190
hns_roce_v2_qp_modify(struct hns_roce_dev * hr_dev,struct hns_roce_v2_qp_context * context,struct hns_roce_v2_qp_context * qpc_mask,struct hns_roce_qp * hr_qp)4191 static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev,
4192 struct hns_roce_v2_qp_context *context,
4193 struct hns_roce_v2_qp_context *qpc_mask,
4194 struct hns_roce_qp *hr_qp)
4195 {
4196 struct hns_roce_cmd_mailbox *mailbox;
4197 int qpc_size;
4198 int ret;
4199
4200 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
4201 if (IS_ERR(mailbox))
4202 return PTR_ERR(mailbox);
4203
4204 /* The qpc size of HIP08 is only 256B, which is half of HIP09 */
4205 qpc_size = hr_dev->caps.qpc_sz;
4206 memcpy(mailbox->buf, context, qpc_size);
4207 memcpy(mailbox->buf + qpc_size, qpc_mask, qpc_size);
4208
4209 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0,
4210 HNS_ROCE_CMD_MODIFY_QPC, hr_qp->qpn);
4211
4212 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
4213
4214 return ret;
4215 }
4216
set_access_flags(struct hns_roce_qp * hr_qp,struct hns_roce_v2_qp_context * context,struct hns_roce_v2_qp_context * qpc_mask,const struct ib_qp_attr * attr,int attr_mask)4217 static void set_access_flags(struct hns_roce_qp *hr_qp,
4218 struct hns_roce_v2_qp_context *context,
4219 struct hns_roce_v2_qp_context *qpc_mask,
4220 const struct ib_qp_attr *attr, int attr_mask)
4221 {
4222 u8 dest_rd_atomic;
4223 u32 access_flags;
4224
4225 dest_rd_atomic = (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) ?
4226 attr->max_dest_rd_atomic : hr_qp->resp_depth;
4227
4228 access_flags = (attr_mask & IB_QP_ACCESS_FLAGS) ?
4229 attr->qp_access_flags : hr_qp->atomic_rd_en;
4230
4231 if (!dest_rd_atomic)
4232 access_flags &= IB_ACCESS_REMOTE_WRITE;
4233
4234 hr_reg_write_bool(context, QPC_RRE,
4235 access_flags & IB_ACCESS_REMOTE_READ);
4236 hr_reg_clear(qpc_mask, QPC_RRE);
4237
4238 hr_reg_write_bool(context, QPC_RWE,
4239 access_flags & IB_ACCESS_REMOTE_WRITE);
4240 hr_reg_clear(qpc_mask, QPC_RWE);
4241
4242 hr_reg_write_bool(context, QPC_ATE,
4243 access_flags & IB_ACCESS_REMOTE_ATOMIC);
4244 hr_reg_clear(qpc_mask, QPC_ATE);
4245 hr_reg_write_bool(context, QPC_EXT_ATE,
4246 access_flags & IB_ACCESS_REMOTE_ATOMIC);
4247 hr_reg_clear(qpc_mask, QPC_EXT_ATE);
4248 }
4249
set_qpc_wqe_cnt(struct hns_roce_qp * hr_qp,struct hns_roce_v2_qp_context * context)4250 static void set_qpc_wqe_cnt(struct hns_roce_qp *hr_qp,
4251 struct hns_roce_v2_qp_context *context)
4252 {
4253 hr_reg_write(context, QPC_SGE_SHIFT,
4254 to_hr_hem_entries_shift(hr_qp->sge.sge_cnt,
4255 hr_qp->sge.sge_shift));
4256
4257 hr_reg_write(context, QPC_SQ_SHIFT, ilog2(hr_qp->sq.wqe_cnt));
4258
4259 hr_reg_write(context, QPC_RQ_SHIFT, ilog2(hr_qp->rq.wqe_cnt));
4260 }
4261
get_cqn(struct ib_cq * ib_cq)4262 static inline int get_cqn(struct ib_cq *ib_cq)
4263 {
4264 return ib_cq ? to_hr_cq(ib_cq)->cqn : 0;
4265 }
4266
get_pdn(struct ib_pd * ib_pd)4267 static inline int get_pdn(struct ib_pd *ib_pd)
4268 {
4269 return ib_pd ? to_hr_pd(ib_pd)->pdn : 0;
4270 }
4271
modify_qp_reset_to_init(struct ib_qp * ibqp,struct hns_roce_v2_qp_context * context,struct hns_roce_v2_qp_context * qpc_mask)4272 static void modify_qp_reset_to_init(struct ib_qp *ibqp,
4273 struct hns_roce_v2_qp_context *context,
4274 struct hns_roce_v2_qp_context *qpc_mask)
4275 {
4276 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4277 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4278
4279 /*
4280 * In v2 engine, software pass context and context mask to hardware
4281 * when modifying qp. If software need modify some fields in context,
4282 * we should set all bits of the relevant fields in context mask to
4283 * 0 at the same time, else set them to 0x1.
4284 */
4285 hr_reg_write(context, QPC_TST, to_hr_qp_type(ibqp->qp_type));
4286
4287 hr_reg_write(context, QPC_PD, get_pdn(ibqp->pd));
4288
4289 hr_reg_write(context, QPC_RQWS, ilog2(hr_qp->rq.max_gs));
4290
4291 set_qpc_wqe_cnt(hr_qp, context);
4292
4293 /* No VLAN need to set 0xFFF */
4294 hr_reg_write(context, QPC_VLAN_ID, 0xfff);
4295
4296 if (ibqp->qp_type == IB_QPT_XRC_TGT) {
4297 context->qkey_xrcd = cpu_to_le32(hr_qp->xrcdn);
4298
4299 hr_reg_enable(context, QPC_XRC_QP_TYPE);
4300 }
4301
4302 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
4303 hr_reg_enable(context, QPC_RQ_RECORD_EN);
4304
4305 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_OWNER_DB)
4306 hr_reg_enable(context, QPC_OWNER_MODE);
4307
4308 hr_reg_write(context, QPC_RQ_DB_RECORD_ADDR_L,
4309 lower_32_bits(hr_qp->rdb.dma) >> 1);
4310 hr_reg_write(context, QPC_RQ_DB_RECORD_ADDR_H,
4311 upper_32_bits(hr_qp->rdb.dma));
4312
4313 hr_reg_write(context, QPC_RX_CQN, get_cqn(ibqp->recv_cq));
4314
4315 if (ibqp->srq) {
4316 hr_reg_enable(context, QPC_SRQ_EN);
4317 hr_reg_write(context, QPC_SRQN, to_hr_srq(ibqp->srq)->srqn);
4318 }
4319
4320 hr_reg_enable(context, QPC_FRE);
4321
4322 hr_reg_write(context, QPC_TX_CQN, get_cqn(ibqp->send_cq));
4323
4324 if (hr_dev->caps.qpc_sz < HNS_ROCE_V3_QPC_SZ)
4325 return;
4326
4327 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_STASH)
4328 hr_reg_enable(&context->ext, QPCEX_STASH);
4329 }
4330
modify_qp_init_to_init(struct ib_qp * ibqp,struct hns_roce_v2_qp_context * context,struct hns_roce_v2_qp_context * qpc_mask)4331 static void modify_qp_init_to_init(struct ib_qp *ibqp,
4332 struct hns_roce_v2_qp_context *context,
4333 struct hns_roce_v2_qp_context *qpc_mask)
4334 {
4335 /*
4336 * In v2 engine, software pass context and context mask to hardware
4337 * when modifying qp. If software need modify some fields in context,
4338 * we should set all bits of the relevant fields in context mask to
4339 * 0 at the same time, else set them to 0x1.
4340 */
4341 hr_reg_write(context, QPC_TST, to_hr_qp_type(ibqp->qp_type));
4342 hr_reg_clear(qpc_mask, QPC_TST);
4343
4344 hr_reg_write(context, QPC_PD, get_pdn(ibqp->pd));
4345 hr_reg_clear(qpc_mask, QPC_PD);
4346
4347 hr_reg_write(context, QPC_RX_CQN, get_cqn(ibqp->recv_cq));
4348 hr_reg_clear(qpc_mask, QPC_RX_CQN);
4349
4350 hr_reg_write(context, QPC_TX_CQN, get_cqn(ibqp->send_cq));
4351 hr_reg_clear(qpc_mask, QPC_TX_CQN);
4352
4353 if (ibqp->srq) {
4354 hr_reg_enable(context, QPC_SRQ_EN);
4355 hr_reg_clear(qpc_mask, QPC_SRQ_EN);
4356 hr_reg_write(context, QPC_SRQN, to_hr_srq(ibqp->srq)->srqn);
4357 hr_reg_clear(qpc_mask, QPC_SRQN);
4358 }
4359 }
4360
config_qp_rq_buf(struct hns_roce_dev * hr_dev,struct hns_roce_qp * hr_qp,struct hns_roce_v2_qp_context * context,struct hns_roce_v2_qp_context * qpc_mask)4361 static int config_qp_rq_buf(struct hns_roce_dev *hr_dev,
4362 struct hns_roce_qp *hr_qp,
4363 struct hns_roce_v2_qp_context *context,
4364 struct hns_roce_v2_qp_context *qpc_mask)
4365 {
4366 u64 mtts[MTT_MIN_COUNT] = { 0 };
4367 u64 wqe_sge_ba;
4368 int ret;
4369
4370 /* Search qp buf's mtts */
4371 ret = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->rq.offset, mtts,
4372 MTT_MIN_COUNT);
4373 if (hr_qp->rq.wqe_cnt && ret) {
4374 ibdev_err(&hr_dev->ib_dev,
4375 "failed to find QP(0x%lx) RQ WQE buf, ret = %d.\n",
4376 hr_qp->qpn, ret);
4377 return ret;
4378 }
4379
4380 wqe_sge_ba = hns_roce_get_mtr_ba(&hr_qp->mtr);
4381
4382 context->wqe_sge_ba = cpu_to_le32(wqe_sge_ba >> 3);
4383 qpc_mask->wqe_sge_ba = 0;
4384
4385 /*
4386 * In v2 engine, software pass context and context mask to hardware
4387 * when modifying qp. If software need modify some fields in context,
4388 * we should set all bits of the relevant fields in context mask to
4389 * 0 at the same time, else set them to 0x1.
4390 */
4391 hr_reg_write(context, QPC_WQE_SGE_BA_H, wqe_sge_ba >> (32 + 3));
4392 hr_reg_clear(qpc_mask, QPC_WQE_SGE_BA_H);
4393
4394 hr_reg_write(context, QPC_SQ_HOP_NUM,
4395 to_hr_hem_hopnum(hr_dev->caps.wqe_sq_hop_num,
4396 hr_qp->sq.wqe_cnt));
4397 hr_reg_clear(qpc_mask, QPC_SQ_HOP_NUM);
4398
4399 hr_reg_write(context, QPC_SGE_HOP_NUM,
4400 to_hr_hem_hopnum(hr_dev->caps.wqe_sge_hop_num,
4401 hr_qp->sge.sge_cnt));
4402 hr_reg_clear(qpc_mask, QPC_SGE_HOP_NUM);
4403
4404 hr_reg_write(context, QPC_RQ_HOP_NUM,
4405 to_hr_hem_hopnum(hr_dev->caps.wqe_rq_hop_num,
4406 hr_qp->rq.wqe_cnt));
4407
4408 hr_reg_clear(qpc_mask, QPC_RQ_HOP_NUM);
4409
4410 hr_reg_write(context, QPC_WQE_SGE_BA_PG_SZ,
4411 to_hr_hw_page_shift(hr_qp->mtr.hem_cfg.ba_pg_shift));
4412 hr_reg_clear(qpc_mask, QPC_WQE_SGE_BA_PG_SZ);
4413
4414 hr_reg_write(context, QPC_WQE_SGE_BUF_PG_SZ,
4415 to_hr_hw_page_shift(hr_qp->mtr.hem_cfg.buf_pg_shift));
4416 hr_reg_clear(qpc_mask, QPC_WQE_SGE_BUF_PG_SZ);
4417
4418 context->rq_cur_blk_addr = cpu_to_le32(to_hr_hw_page_addr(mtts[0]));
4419 qpc_mask->rq_cur_blk_addr = 0;
4420
4421 hr_reg_write(context, QPC_RQ_CUR_BLK_ADDR_H,
4422 upper_32_bits(to_hr_hw_page_addr(mtts[0])));
4423 hr_reg_clear(qpc_mask, QPC_RQ_CUR_BLK_ADDR_H);
4424
4425 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {
4426 context->rq_nxt_blk_addr =
4427 cpu_to_le32(to_hr_hw_page_addr(mtts[1]));
4428 qpc_mask->rq_nxt_blk_addr = 0;
4429 hr_reg_write(context, QPC_RQ_NXT_BLK_ADDR_H,
4430 upper_32_bits(to_hr_hw_page_addr(mtts[1])));
4431 hr_reg_clear(qpc_mask, QPC_RQ_NXT_BLK_ADDR_H);
4432 }
4433
4434 return 0;
4435 }
4436
config_qp_sq_buf(struct hns_roce_dev * hr_dev,struct hns_roce_qp * hr_qp,struct hns_roce_v2_qp_context * context,struct hns_roce_v2_qp_context * qpc_mask)4437 static int config_qp_sq_buf(struct hns_roce_dev *hr_dev,
4438 struct hns_roce_qp *hr_qp,
4439 struct hns_roce_v2_qp_context *context,
4440 struct hns_roce_v2_qp_context *qpc_mask)
4441 {
4442 struct ib_device *ibdev = &hr_dev->ib_dev;
4443 u64 sge_cur_blk = 0;
4444 u64 sq_cur_blk = 0;
4445 int ret;
4446
4447 /* search qp buf's mtts */
4448 ret = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, hr_qp->sq.offset,
4449 &sq_cur_blk, 1);
4450 if (ret) {
4451 ibdev_err(ibdev, "failed to find QP(0x%lx) SQ WQE buf, ret = %d.\n",
4452 hr_qp->qpn, ret);
4453 return ret;
4454 }
4455 if (hr_qp->sge.sge_cnt > 0) {
4456 ret = hns_roce_mtr_find(hr_dev, &hr_qp->mtr,
4457 hr_qp->sge.offset, &sge_cur_blk, 1);
4458 if (ret) {
4459 ibdev_err(ibdev, "failed to find QP(0x%lx) SGE buf, ret = %d.\n",
4460 hr_qp->qpn, ret);
4461 return ret;
4462 }
4463 }
4464
4465 /*
4466 * In v2 engine, software pass context and context mask to hardware
4467 * when modifying qp. If software need modify some fields in context,
4468 * we should set all bits of the relevant fields in context mask to
4469 * 0 at the same time, else set them to 0x1.
4470 */
4471 hr_reg_write(context, QPC_SQ_CUR_BLK_ADDR_L,
4472 lower_32_bits(to_hr_hw_page_addr(sq_cur_blk)));
4473 hr_reg_write(context, QPC_SQ_CUR_BLK_ADDR_H,
4474 upper_32_bits(to_hr_hw_page_addr(sq_cur_blk)));
4475 hr_reg_clear(qpc_mask, QPC_SQ_CUR_BLK_ADDR_L);
4476 hr_reg_clear(qpc_mask, QPC_SQ_CUR_BLK_ADDR_H);
4477
4478 hr_reg_write(context, QPC_SQ_CUR_SGE_BLK_ADDR_L,
4479 lower_32_bits(to_hr_hw_page_addr(sge_cur_blk)));
4480 hr_reg_write(context, QPC_SQ_CUR_SGE_BLK_ADDR_H,
4481 upper_32_bits(to_hr_hw_page_addr(sge_cur_blk)));
4482 hr_reg_clear(qpc_mask, QPC_SQ_CUR_SGE_BLK_ADDR_L);
4483 hr_reg_clear(qpc_mask, QPC_SQ_CUR_SGE_BLK_ADDR_H);
4484
4485 hr_reg_write(context, QPC_RX_SQ_CUR_BLK_ADDR_L,
4486 lower_32_bits(to_hr_hw_page_addr(sq_cur_blk)));
4487 hr_reg_write(context, QPC_RX_SQ_CUR_BLK_ADDR_H,
4488 upper_32_bits(to_hr_hw_page_addr(sq_cur_blk)));
4489 hr_reg_clear(qpc_mask, QPC_RX_SQ_CUR_BLK_ADDR_L);
4490 hr_reg_clear(qpc_mask, QPC_RX_SQ_CUR_BLK_ADDR_H);
4491
4492 return 0;
4493 }
4494
get_mtu(struct ib_qp * ibqp,const struct ib_qp_attr * attr)4495 static inline enum ib_mtu get_mtu(struct ib_qp *ibqp,
4496 const struct ib_qp_attr *attr)
4497 {
4498 if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_UD)
4499 return IB_MTU_4096;
4500
4501 return attr->path_mtu;
4502 }
4503
modify_qp_init_to_rtr(struct ib_qp * ibqp,const struct ib_qp_attr * attr,int attr_mask,struct hns_roce_v2_qp_context * context,struct hns_roce_v2_qp_context * qpc_mask,struct ib_udata * udata)4504 static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
4505 const struct ib_qp_attr *attr, int attr_mask,
4506 struct hns_roce_v2_qp_context *context,
4507 struct hns_roce_v2_qp_context *qpc_mask,
4508 struct ib_udata *udata)
4509 {
4510 struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(udata,
4511 struct hns_roce_ucontext, ibucontext);
4512 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4513 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4514 struct ib_device *ibdev = &hr_dev->ib_dev;
4515 dma_addr_t trrl_ba;
4516 dma_addr_t irrl_ba;
4517 enum ib_mtu ib_mtu;
4518 const u8 *smac;
4519 u8 lp_pktn_ini;
4520 u64 *mtts;
4521 u8 *dmac;
4522 u32 port;
4523 int mtu;
4524 int ret;
4525
4526 ret = config_qp_rq_buf(hr_dev, hr_qp, context, qpc_mask);
4527 if (ret) {
4528 ibdev_err(ibdev, "failed to config rq buf, ret = %d.\n", ret);
4529 return ret;
4530 }
4531
4532 /* Search IRRL's mtts */
4533 mtts = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table,
4534 hr_qp->qpn, &irrl_ba);
4535 if (!mtts) {
4536 ibdev_err(ibdev, "failed to find qp irrl_table.\n");
4537 return -EINVAL;
4538 }
4539
4540 /* Search TRRL's mtts */
4541 mtts = hns_roce_table_find(hr_dev, &hr_dev->qp_table.trrl_table,
4542 hr_qp->qpn, &trrl_ba);
4543 if (!mtts) {
4544 ibdev_err(ibdev, "failed to find qp trrl_table.\n");
4545 return -EINVAL;
4546 }
4547
4548 if (attr_mask & IB_QP_ALT_PATH) {
4549 ibdev_err(ibdev, "INIT2RTR attr_mask (0x%x) error.\n",
4550 attr_mask);
4551 return -EINVAL;
4552 }
4553
4554 hr_reg_write(context, QPC_TRRL_BA_L, trrl_ba >> QPC_TRRL_BA_L_S);
4555 hr_reg_clear(qpc_mask, QPC_TRRL_BA_L);
4556 context->trrl_ba = cpu_to_le32(trrl_ba >> QPC_TRRL_BA_M_S);
4557 qpc_mask->trrl_ba = 0;
4558 hr_reg_write(context, QPC_TRRL_BA_H, trrl_ba >> QPC_TRRL_BA_H_S);
4559 hr_reg_clear(qpc_mask, QPC_TRRL_BA_H);
4560
4561 context->irrl_ba = cpu_to_le32(irrl_ba >> QPC_IRRL_BA_L_S);
4562 qpc_mask->irrl_ba = 0;
4563 hr_reg_write(context, QPC_IRRL_BA_H, irrl_ba >> QPC_IRRL_BA_H_S);
4564 hr_reg_clear(qpc_mask, QPC_IRRL_BA_H);
4565
4566 hr_reg_enable(context, QPC_RMT_E2E);
4567 hr_reg_clear(qpc_mask, QPC_RMT_E2E);
4568
4569 hr_reg_write(context, QPC_SIG_TYPE, hr_qp->sq_signal_bits);
4570 hr_reg_clear(qpc_mask, QPC_SIG_TYPE);
4571
4572 port = (attr_mask & IB_QP_PORT) ? (attr->port_num - 1) : hr_qp->port;
4573
4574 smac = (const u8 *)hr_dev->dev_addr[port];
4575 dmac = (u8 *)attr->ah_attr.roce.dmac;
4576 /* when dmac equals smac or loop_idc is 1, it should loopback */
4577 if (ether_addr_equal_unaligned(dmac, smac) ||
4578 hr_dev->loop_idc == 0x1) {
4579 hr_reg_write(context, QPC_LBI, hr_dev->loop_idc);
4580 hr_reg_clear(qpc_mask, QPC_LBI);
4581 }
4582
4583 if (attr_mask & IB_QP_DEST_QPN) {
4584 hr_reg_write(context, QPC_DQPN, attr->dest_qp_num);
4585 hr_reg_clear(qpc_mask, QPC_DQPN);
4586 }
4587
4588 memcpy(&context->dmac, dmac, sizeof(u32));
4589 hr_reg_write(context, QPC_DMAC_H, *((u16 *)(&dmac[4])));
4590 qpc_mask->dmac = 0;
4591 hr_reg_clear(qpc_mask, QPC_DMAC_H);
4592
4593 ib_mtu = get_mtu(ibqp, attr);
4594 hr_qp->path_mtu = ib_mtu;
4595
4596 mtu = ib_mtu_enum_to_int(ib_mtu);
4597 if (WARN_ON(mtu <= 0))
4598 return -EINVAL;
4599 #define MIN_LP_MSG_LEN 1024
4600 /* mtu * (2 ^ lp_pktn_ini) should be in the range of 1024 to mtu */
4601 lp_pktn_ini = ilog2(max(mtu, MIN_LP_MSG_LEN) / mtu);
4602
4603 if (attr_mask & IB_QP_PATH_MTU) {
4604 hr_reg_write(context, QPC_MTU, ib_mtu);
4605 hr_reg_clear(qpc_mask, QPC_MTU);
4606 }
4607
4608 hr_reg_write(context, QPC_LP_PKTN_INI, lp_pktn_ini);
4609 hr_reg_clear(qpc_mask, QPC_LP_PKTN_INI);
4610
4611 /* ACK_REQ_FREQ should be larger than or equal to LP_PKTN_INI */
4612 hr_reg_write(context, QPC_ACK_REQ_FREQ, lp_pktn_ini);
4613 hr_reg_clear(qpc_mask, QPC_ACK_REQ_FREQ);
4614
4615 hr_reg_clear(qpc_mask, QPC_RX_REQ_PSN_ERR);
4616 hr_reg_clear(qpc_mask, QPC_RX_REQ_MSN);
4617 hr_reg_clear(qpc_mask, QPC_RX_REQ_LAST_OPTYPE);
4618
4619 context->rq_rnr_timer = 0;
4620 qpc_mask->rq_rnr_timer = 0;
4621
4622 hr_reg_clear(qpc_mask, QPC_TRRL_HEAD_MAX);
4623 hr_reg_clear(qpc_mask, QPC_TRRL_TAIL_MAX);
4624
4625 #define MAX_LP_SGEN 3
4626 /* rocee send 2^lp_sgen_ini segs every time */
4627 hr_reg_write(context, QPC_LP_SGEN_INI, MAX_LP_SGEN);
4628 hr_reg_clear(qpc_mask, QPC_LP_SGEN_INI);
4629
4630 if (udata && ibqp->qp_type == IB_QPT_RC &&
4631 (uctx->config & HNS_ROCE_RQ_INLINE_FLAGS)) {
4632 hr_reg_write_bool(context, QPC_RQIE,
4633 hr_dev->caps.flags &
4634 HNS_ROCE_CAP_FLAG_RQ_INLINE);
4635 hr_reg_clear(qpc_mask, QPC_RQIE);
4636 }
4637
4638 if (udata &&
4639 (ibqp->qp_type == IB_QPT_RC || ibqp->qp_type == IB_QPT_XRC_TGT) &&
4640 (uctx->config & HNS_ROCE_CQE_INLINE_FLAGS)) {
4641 hr_reg_write_bool(context, QPC_CQEIE,
4642 hr_dev->caps.flags &
4643 HNS_ROCE_CAP_FLAG_CQE_INLINE);
4644 hr_reg_clear(qpc_mask, QPC_CQEIE);
4645
4646 hr_reg_write(context, QPC_CQEIS, 0);
4647 hr_reg_clear(qpc_mask, QPC_CQEIS);
4648 }
4649
4650 return 0;
4651 }
4652
modify_qp_rtr_to_rts(struct ib_qp * ibqp,int attr_mask,struct hns_roce_v2_qp_context * context,struct hns_roce_v2_qp_context * qpc_mask)4653 static int modify_qp_rtr_to_rts(struct ib_qp *ibqp, int attr_mask,
4654 struct hns_roce_v2_qp_context *context,
4655 struct hns_roce_v2_qp_context *qpc_mask)
4656 {
4657 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4658 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4659 struct ib_device *ibdev = &hr_dev->ib_dev;
4660 int ret;
4661
4662 /* Not support alternate path and path migration */
4663 if (attr_mask & (IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE)) {
4664 ibdev_err(ibdev, "RTR2RTS attr_mask (0x%x)error\n", attr_mask);
4665 return -EINVAL;
4666 }
4667
4668 ret = config_qp_sq_buf(hr_dev, hr_qp, context, qpc_mask);
4669 if (ret) {
4670 ibdev_err(ibdev, "failed to config sq buf, ret = %d.\n", ret);
4671 return ret;
4672 }
4673
4674 /*
4675 * Set some fields in context to zero, Because the default values
4676 * of all fields in context are zero, we need not set them to 0 again.
4677 * but we should set the relevant fields of context mask to 0.
4678 */
4679 hr_reg_clear(qpc_mask, QPC_IRRL_SGE_IDX);
4680
4681 hr_reg_clear(qpc_mask, QPC_RX_ACK_MSN);
4682
4683 hr_reg_clear(qpc_mask, QPC_ACK_LAST_OPTYPE);
4684 hr_reg_clear(qpc_mask, QPC_IRRL_PSN_VLD);
4685 hr_reg_clear(qpc_mask, QPC_IRRL_PSN);
4686
4687 hr_reg_clear(qpc_mask, QPC_IRRL_TAIL_REAL);
4688
4689 hr_reg_clear(qpc_mask, QPC_RETRY_MSG_MSN);
4690
4691 hr_reg_clear(qpc_mask, QPC_RNR_RETRY_FLAG);
4692
4693 hr_reg_clear(qpc_mask, QPC_CHECK_FLG);
4694
4695 hr_reg_clear(qpc_mask, QPC_V2_IRRL_HEAD);
4696
4697 return 0;
4698 }
4699
alloc_dip_entry(struct xarray * dip_xa,u32 qpn)4700 static int alloc_dip_entry(struct xarray *dip_xa, u32 qpn)
4701 {
4702 struct hns_roce_dip *hr_dip;
4703 int ret;
4704
4705 hr_dip = xa_load(dip_xa, qpn);
4706 if (hr_dip)
4707 return 0;
4708
4709 hr_dip = kzalloc(sizeof(*hr_dip), GFP_KERNEL);
4710 if (!hr_dip)
4711 return -ENOMEM;
4712
4713 ret = xa_err(xa_store(dip_xa, qpn, hr_dip, GFP_KERNEL));
4714 if (ret)
4715 kfree(hr_dip);
4716
4717 return ret;
4718 }
4719
get_dip_ctx_idx(struct ib_qp * ibqp,const struct ib_qp_attr * attr,u32 * dip_idx)4720 static int get_dip_ctx_idx(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
4721 u32 *dip_idx)
4722 {
4723 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
4724 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4725 struct xarray *dip_xa = &hr_dev->qp_table.dip_xa;
4726 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4727 struct hns_roce_dip *hr_dip;
4728 unsigned long idx;
4729 int ret = 0;
4730
4731 ret = alloc_dip_entry(dip_xa, ibqp->qp_num);
4732 if (ret)
4733 return ret;
4734
4735 xa_lock(dip_xa);
4736
4737 xa_for_each(dip_xa, idx, hr_dip) {
4738 if (hr_dip->qp_cnt &&
4739 !memcmp(grh->dgid.raw, hr_dip->dgid, GID_LEN_V2)) {
4740 *dip_idx = hr_dip->dip_idx;
4741 hr_dip->qp_cnt++;
4742 hr_qp->dip = hr_dip;
4743 goto out;
4744 }
4745 }
4746
4747 /* If no dgid is found, a new dip and a mapping between dgid and
4748 * dip_idx will be created.
4749 */
4750 xa_for_each(dip_xa, idx, hr_dip) {
4751 if (hr_dip->qp_cnt)
4752 continue;
4753
4754 *dip_idx = idx;
4755 memcpy(hr_dip->dgid, grh->dgid.raw, sizeof(grh->dgid.raw));
4756 hr_dip->dip_idx = idx;
4757 hr_dip->qp_cnt++;
4758 hr_qp->dip = hr_dip;
4759 break;
4760 }
4761
4762 /* This should never happen. */
4763 if (WARN_ON_ONCE(!hr_qp->dip))
4764 ret = -ENOSPC;
4765
4766 out:
4767 xa_unlock(dip_xa);
4768 return ret;
4769 }
4770
4771 enum {
4772 CONG_DCQCN,
4773 CONG_WINDOW,
4774 };
4775
4776 enum {
4777 UNSUPPORT_CONG_LEVEL,
4778 SUPPORT_CONG_LEVEL,
4779 };
4780
4781 enum {
4782 CONG_LDCP,
4783 CONG_HC3,
4784 };
4785
4786 enum {
4787 DIP_INVALID,
4788 DIP_VALID,
4789 };
4790
4791 enum {
4792 WND_LIMIT,
4793 WND_UNLIMIT,
4794 };
4795
check_cong_type(struct ib_qp * ibqp,struct hns_roce_congestion_algorithm * cong_alg)4796 static int check_cong_type(struct ib_qp *ibqp,
4797 struct hns_roce_congestion_algorithm *cong_alg)
4798 {
4799 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4800
4801 /* different congestion types match different configurations */
4802 switch (hr_qp->cong_type) {
4803 case CONG_TYPE_DCQCN:
4804 cong_alg->alg_sel = CONG_DCQCN;
4805 cong_alg->alg_sub_sel = UNSUPPORT_CONG_LEVEL;
4806 cong_alg->dip_vld = DIP_INVALID;
4807 cong_alg->wnd_mode_sel = WND_LIMIT;
4808 break;
4809 case CONG_TYPE_LDCP:
4810 cong_alg->alg_sel = CONG_WINDOW;
4811 cong_alg->alg_sub_sel = CONG_LDCP;
4812 cong_alg->dip_vld = DIP_INVALID;
4813 cong_alg->wnd_mode_sel = WND_UNLIMIT;
4814 break;
4815 case CONG_TYPE_HC3:
4816 cong_alg->alg_sel = CONG_WINDOW;
4817 cong_alg->alg_sub_sel = CONG_HC3;
4818 cong_alg->dip_vld = DIP_INVALID;
4819 cong_alg->wnd_mode_sel = WND_LIMIT;
4820 break;
4821 case CONG_TYPE_DIP:
4822 cong_alg->alg_sel = CONG_DCQCN;
4823 cong_alg->alg_sub_sel = UNSUPPORT_CONG_LEVEL;
4824 cong_alg->dip_vld = DIP_VALID;
4825 cong_alg->wnd_mode_sel = WND_LIMIT;
4826 break;
4827 default:
4828 hr_qp->cong_type = CONG_TYPE_DCQCN;
4829 cong_alg->alg_sel = CONG_DCQCN;
4830 cong_alg->alg_sub_sel = UNSUPPORT_CONG_LEVEL;
4831 cong_alg->dip_vld = DIP_INVALID;
4832 cong_alg->wnd_mode_sel = WND_LIMIT;
4833 break;
4834 }
4835
4836 return 0;
4837 }
4838
fill_cong_field(struct ib_qp * ibqp,const struct ib_qp_attr * attr,struct hns_roce_v2_qp_context * context,struct hns_roce_v2_qp_context * qpc_mask)4839 static int fill_cong_field(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
4840 struct hns_roce_v2_qp_context *context,
4841 struct hns_roce_v2_qp_context *qpc_mask)
4842 {
4843 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
4844 struct hns_roce_congestion_algorithm cong_field;
4845 struct ib_device *ibdev = ibqp->device;
4846 struct hns_roce_dev *hr_dev = to_hr_dev(ibdev);
4847 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4848 u32 dip_idx = 0;
4849 int ret;
4850
4851 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 ||
4852 grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE)
4853 return 0;
4854
4855 ret = check_cong_type(ibqp, &cong_field);
4856 if (ret)
4857 return ret;
4858
4859 hr_reg_write(context, QPC_CONG_ALGO_TMPL_ID, hr_dev->cong_algo_tmpl_id +
4860 hr_qp->cong_type * HNS_ROCE_CONG_SIZE);
4861 hr_reg_clear(qpc_mask, QPC_CONG_ALGO_TMPL_ID);
4862 hr_reg_write(&context->ext, QPCEX_CONG_ALG_SEL, cong_field.alg_sel);
4863 hr_reg_clear(&qpc_mask->ext, QPCEX_CONG_ALG_SEL);
4864 hr_reg_write(&context->ext, QPCEX_CONG_ALG_SUB_SEL,
4865 cong_field.alg_sub_sel);
4866 hr_reg_clear(&qpc_mask->ext, QPCEX_CONG_ALG_SUB_SEL);
4867 hr_reg_write(&context->ext, QPCEX_DIP_CTX_IDX_VLD, cong_field.dip_vld);
4868 hr_reg_clear(&qpc_mask->ext, QPCEX_DIP_CTX_IDX_VLD);
4869 hr_reg_write(&context->ext, QPCEX_SQ_RQ_NOT_FORBID_EN,
4870 cong_field.wnd_mode_sel);
4871 hr_reg_clear(&qpc_mask->ext, QPCEX_SQ_RQ_NOT_FORBID_EN);
4872
4873 /* if dip is disabled, there is no need to set dip idx */
4874 if (cong_field.dip_vld == 0)
4875 return 0;
4876
4877 ret = get_dip_ctx_idx(ibqp, attr, &dip_idx);
4878 if (ret) {
4879 ibdev_err(ibdev, "failed to fill cong field, ret = %d.\n", ret);
4880 return ret;
4881 }
4882
4883 hr_reg_write(&context->ext, QPCEX_DIP_CTX_IDX, dip_idx);
4884 hr_reg_write(&qpc_mask->ext, QPCEX_DIP_CTX_IDX, 0);
4885
4886 return 0;
4887 }
4888
hns_roce_hw_v2_get_dscp(struct hns_roce_dev * hr_dev,u8 dscp,u8 * tc_mode,u8 * priority)4889 static int hns_roce_hw_v2_get_dscp(struct hns_roce_dev *hr_dev, u8 dscp,
4890 u8 *tc_mode, u8 *priority)
4891 {
4892 struct hns_roce_v2_priv *priv = hr_dev->priv;
4893 struct hnae3_handle *handle = priv->handle;
4894 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
4895
4896 if (!ops->get_dscp_prio)
4897 return -EOPNOTSUPP;
4898
4899 return ops->get_dscp_prio(handle, dscp, tc_mode, priority);
4900 }
4901
check_sl_valid(struct hns_roce_dev * hr_dev,u8 sl)4902 bool check_sl_valid(struct hns_roce_dev *hr_dev, u8 sl)
4903 {
4904 u32 max_sl;
4905
4906 max_sl = min_t(u32, MAX_SERVICE_LEVEL, hr_dev->caps.sl_num - 1);
4907 if (unlikely(sl > max_sl)) {
4908 ibdev_err_ratelimited(&hr_dev->ib_dev,
4909 "failed to set SL(%u). Shouldn't be larger than %u.\n",
4910 sl, max_sl);
4911 return false;
4912 }
4913
4914 return true;
4915 }
4916
hns_roce_set_sl(struct ib_qp * ibqp,const struct ib_qp_attr * attr,struct hns_roce_v2_qp_context * context,struct hns_roce_v2_qp_context * qpc_mask)4917 static int hns_roce_set_sl(struct ib_qp *ibqp,
4918 const struct ib_qp_attr *attr,
4919 struct hns_roce_v2_qp_context *context,
4920 struct hns_roce_v2_qp_context *qpc_mask)
4921 {
4922 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
4923 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4924 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4925 struct ib_device *ibdev = &hr_dev->ib_dev;
4926 int ret;
4927
4928 ret = hns_roce_hw_v2_get_dscp(hr_dev, get_tclass(&attr->ah_attr.grh),
4929 &hr_qp->tc_mode, &hr_qp->priority);
4930 if (ret && ret != -EOPNOTSUPP &&
4931 grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {
4932 ibdev_err_ratelimited(ibdev,
4933 "failed to get dscp, ret = %d.\n", ret);
4934 return ret;
4935 }
4936
4937 if (hr_qp->tc_mode == HNAE3_TC_MAP_MODE_DSCP &&
4938 grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)
4939 hr_qp->sl = hr_qp->priority;
4940 else
4941 hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr);
4942
4943 if (!check_sl_valid(hr_dev, hr_qp->sl))
4944 return -EINVAL;
4945
4946 hr_reg_write(context, QPC_SL, hr_qp->sl);
4947 hr_reg_clear(qpc_mask, QPC_SL);
4948
4949 return 0;
4950 }
4951
hns_roce_v2_set_path(struct ib_qp * ibqp,const struct ib_qp_attr * attr,int attr_mask,struct hns_roce_v2_qp_context * context,struct hns_roce_v2_qp_context * qpc_mask)4952 static int hns_roce_v2_set_path(struct ib_qp *ibqp,
4953 const struct ib_qp_attr *attr,
4954 int attr_mask,
4955 struct hns_roce_v2_qp_context *context,
4956 struct hns_roce_v2_qp_context *qpc_mask)
4957 {
4958 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
4959 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4960 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4961 struct ib_device *ibdev = &hr_dev->ib_dev;
4962 const struct ib_gid_attr *gid_attr = NULL;
4963 u8 sl = rdma_ah_get_sl(&attr->ah_attr);
4964 int is_roce_protocol;
4965 u16 vlan_id = 0xffff;
4966 bool is_udp = false;
4967 u8 ib_port;
4968 u8 hr_port;
4969 int ret;
4970
4971 /*
4972 * If free_mr_en of qp is set, it means that this qp comes from
4973 * free mr. This qp will perform the loopback operation.
4974 * In the loopback scenario, only sl needs to be set.
4975 */
4976 if (hr_qp->free_mr_en) {
4977 if (!check_sl_valid(hr_dev, sl))
4978 return -EINVAL;
4979 hr_reg_write(context, QPC_SL, sl);
4980 hr_reg_clear(qpc_mask, QPC_SL);
4981 hr_qp->sl = sl;
4982 return 0;
4983 }
4984
4985 ib_port = (attr_mask & IB_QP_PORT) ? attr->port_num : hr_qp->port + 1;
4986 hr_port = ib_port - 1;
4987 is_roce_protocol = rdma_cap_eth_ah(&hr_dev->ib_dev, ib_port) &&
4988 rdma_ah_get_ah_flags(&attr->ah_attr) & IB_AH_GRH;
4989
4990 if (is_roce_protocol) {
4991 gid_attr = attr->ah_attr.grh.sgid_attr;
4992 ret = rdma_read_gid_l2_fields(gid_attr, &vlan_id, NULL);
4993 if (ret)
4994 return ret;
4995
4996 is_udp = (gid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP);
4997 }
4998
4999 /* Only HIP08 needs to set the vlan_en bits in QPC */
5000 if (vlan_id < VLAN_N_VID &&
5001 hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {
5002 hr_reg_enable(context, QPC_RQ_VLAN_EN);
5003 hr_reg_clear(qpc_mask, QPC_RQ_VLAN_EN);
5004 hr_reg_enable(context, QPC_SQ_VLAN_EN);
5005 hr_reg_clear(qpc_mask, QPC_SQ_VLAN_EN);
5006 }
5007
5008 hr_reg_write(context, QPC_VLAN_ID, vlan_id);
5009 hr_reg_clear(qpc_mask, QPC_VLAN_ID);
5010
5011 if (grh->sgid_index >= hr_dev->caps.gid_table_len[hr_port]) {
5012 ibdev_err(ibdev, "sgid_index(%u) too large. max is %d\n",
5013 grh->sgid_index, hr_dev->caps.gid_table_len[hr_port]);
5014 return -EINVAL;
5015 }
5016
5017 if (attr->ah_attr.type != RDMA_AH_ATTR_TYPE_ROCE) {
5018 ibdev_err(ibdev, "ah attr is not RDMA roce type\n");
5019 return -EINVAL;
5020 }
5021
5022 hr_reg_write(context, QPC_UDPSPN,
5023 is_udp ? rdma_get_udp_sport(grh->flow_label, ibqp->qp_num,
5024 attr->dest_qp_num) :
5025 0);
5026
5027 hr_reg_clear(qpc_mask, QPC_UDPSPN);
5028
5029 hr_reg_write(context, QPC_GMV_IDX, grh->sgid_index);
5030
5031 hr_reg_clear(qpc_mask, QPC_GMV_IDX);
5032
5033 hr_reg_write(context, QPC_HOPLIMIT, grh->hop_limit);
5034 hr_reg_clear(qpc_mask, QPC_HOPLIMIT);
5035
5036 ret = fill_cong_field(ibqp, attr, context, qpc_mask);
5037 if (ret)
5038 return ret;
5039
5040 hr_reg_write(context, QPC_TC, get_tclass(&attr->ah_attr.grh));
5041 hr_reg_clear(qpc_mask, QPC_TC);
5042
5043 hr_reg_write(context, QPC_FL, grh->flow_label);
5044 hr_reg_clear(qpc_mask, QPC_FL);
5045 memcpy(context->dgid, grh->dgid.raw, sizeof(grh->dgid.raw));
5046 memset(qpc_mask->dgid, 0, sizeof(grh->dgid.raw));
5047
5048 return hns_roce_set_sl(ibqp, attr, context, qpc_mask);
5049 }
5050
check_qp_state(enum ib_qp_state cur_state,enum ib_qp_state new_state)5051 static bool check_qp_state(enum ib_qp_state cur_state,
5052 enum ib_qp_state new_state)
5053 {
5054 static const bool sm[][IB_QPS_ERR + 1] = {
5055 [IB_QPS_RESET] = { [IB_QPS_RESET] = true,
5056 [IB_QPS_INIT] = true },
5057 [IB_QPS_INIT] = { [IB_QPS_RESET] = true,
5058 [IB_QPS_INIT] = true,
5059 [IB_QPS_RTR] = true,
5060 [IB_QPS_ERR] = true },
5061 [IB_QPS_RTR] = { [IB_QPS_RESET] = true,
5062 [IB_QPS_RTS] = true,
5063 [IB_QPS_ERR] = true },
5064 [IB_QPS_RTS] = { [IB_QPS_RESET] = true,
5065 [IB_QPS_RTS] = true,
5066 [IB_QPS_ERR] = true },
5067 [IB_QPS_SQD] = {},
5068 [IB_QPS_SQE] = {},
5069 [IB_QPS_ERR] = { [IB_QPS_RESET] = true,
5070 [IB_QPS_ERR] = true }
5071 };
5072
5073 return sm[cur_state][new_state];
5074 }
5075
hns_roce_v2_set_abs_fields(struct ib_qp * ibqp,const struct ib_qp_attr * attr,int attr_mask,enum ib_qp_state cur_state,enum ib_qp_state new_state,struct hns_roce_v2_qp_context * context,struct hns_roce_v2_qp_context * qpc_mask,struct ib_udata * udata)5076 static int hns_roce_v2_set_abs_fields(struct ib_qp *ibqp,
5077 const struct ib_qp_attr *attr,
5078 int attr_mask,
5079 enum ib_qp_state cur_state,
5080 enum ib_qp_state new_state,
5081 struct hns_roce_v2_qp_context *context,
5082 struct hns_roce_v2_qp_context *qpc_mask,
5083 struct ib_udata *udata)
5084 {
5085 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
5086 int ret = 0;
5087
5088 if (!check_qp_state(cur_state, new_state))
5089 return -EINVAL;
5090
5091 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
5092 memset(qpc_mask, 0, hr_dev->caps.qpc_sz);
5093 modify_qp_reset_to_init(ibqp, context, qpc_mask);
5094 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
5095 modify_qp_init_to_init(ibqp, context, qpc_mask);
5096 } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
5097 ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context,
5098 qpc_mask, udata);
5099 } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
5100 ret = modify_qp_rtr_to_rts(ibqp, attr_mask, context, qpc_mask);
5101 }
5102
5103 return ret;
5104 }
5105
check_qp_timeout_cfg_range(struct hns_roce_dev * hr_dev,u8 * timeout)5106 static bool check_qp_timeout_cfg_range(struct hns_roce_dev *hr_dev, u8 *timeout)
5107 {
5108 #define QP_ACK_TIMEOUT_MAX_HIP08 20
5109 #define QP_ACK_TIMEOUT_MAX 31
5110
5111 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {
5112 if (*timeout > QP_ACK_TIMEOUT_MAX_HIP08) {
5113 ibdev_warn(&hr_dev->ib_dev,
5114 "local ACK timeout shall be 0 to 20.\n");
5115 return false;
5116 }
5117 *timeout += HNS_ROCE_V2_QP_ACK_TIMEOUT_OFS_HIP08;
5118 } else if (hr_dev->pci_dev->revision > PCI_REVISION_ID_HIP08) {
5119 if (*timeout > QP_ACK_TIMEOUT_MAX) {
5120 ibdev_warn(&hr_dev->ib_dev,
5121 "local ACK timeout shall be 0 to 31.\n");
5122 return false;
5123 }
5124 }
5125
5126 return true;
5127 }
5128
hns_roce_v2_set_opt_fields(struct ib_qp * ibqp,const struct ib_qp_attr * attr,int attr_mask,struct hns_roce_v2_qp_context * context,struct hns_roce_v2_qp_context * qpc_mask)5129 static int hns_roce_v2_set_opt_fields(struct ib_qp *ibqp,
5130 const struct ib_qp_attr *attr,
5131 int attr_mask,
5132 struct hns_roce_v2_qp_context *context,
5133 struct hns_roce_v2_qp_context *qpc_mask)
5134 {
5135 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
5136 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
5137 int ret = 0;
5138 u8 timeout;
5139
5140 if (attr_mask & IB_QP_AV) {
5141 ret = hns_roce_v2_set_path(ibqp, attr, attr_mask, context,
5142 qpc_mask);
5143 if (ret)
5144 return ret;
5145 }
5146
5147 if (attr_mask & IB_QP_TIMEOUT) {
5148 timeout = attr->timeout;
5149 if (check_qp_timeout_cfg_range(hr_dev, &timeout)) {
5150 hr_reg_write(context, QPC_AT, timeout);
5151 hr_reg_clear(qpc_mask, QPC_AT);
5152 }
5153 }
5154
5155 if (attr_mask & IB_QP_RETRY_CNT) {
5156 hr_reg_write(context, QPC_RETRY_NUM_INIT, attr->retry_cnt);
5157 hr_reg_clear(qpc_mask, QPC_RETRY_NUM_INIT);
5158
5159 hr_reg_write(context, QPC_RETRY_CNT, attr->retry_cnt);
5160 hr_reg_clear(qpc_mask, QPC_RETRY_CNT);
5161 }
5162
5163 if (attr_mask & IB_QP_RNR_RETRY) {
5164 hr_reg_write(context, QPC_RNR_NUM_INIT, attr->rnr_retry);
5165 hr_reg_clear(qpc_mask, QPC_RNR_NUM_INIT);
5166
5167 hr_reg_write(context, QPC_RNR_CNT, attr->rnr_retry);
5168 hr_reg_clear(qpc_mask, QPC_RNR_CNT);
5169 }
5170
5171 if (attr_mask & IB_QP_SQ_PSN) {
5172 hr_reg_write(context, QPC_SQ_CUR_PSN, attr->sq_psn);
5173 hr_reg_clear(qpc_mask, QPC_SQ_CUR_PSN);
5174
5175 hr_reg_write(context, QPC_SQ_MAX_PSN, attr->sq_psn);
5176 hr_reg_clear(qpc_mask, QPC_SQ_MAX_PSN);
5177
5178 hr_reg_write(context, QPC_RETRY_MSG_PSN_L, attr->sq_psn);
5179 hr_reg_clear(qpc_mask, QPC_RETRY_MSG_PSN_L);
5180
5181 hr_reg_write(context, QPC_RETRY_MSG_PSN_H,
5182 attr->sq_psn >> RETRY_MSG_PSN_SHIFT);
5183 hr_reg_clear(qpc_mask, QPC_RETRY_MSG_PSN_H);
5184
5185 hr_reg_write(context, QPC_RETRY_MSG_FPKT_PSN, attr->sq_psn);
5186 hr_reg_clear(qpc_mask, QPC_RETRY_MSG_FPKT_PSN);
5187
5188 hr_reg_write(context, QPC_RX_ACK_EPSN, attr->sq_psn);
5189 hr_reg_clear(qpc_mask, QPC_RX_ACK_EPSN);
5190 }
5191
5192 if ((attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) &&
5193 attr->max_dest_rd_atomic) {
5194 hr_reg_write(context, QPC_RR_MAX,
5195 fls(attr->max_dest_rd_atomic - 1));
5196 hr_reg_clear(qpc_mask, QPC_RR_MAX);
5197 }
5198
5199 if ((attr_mask & IB_QP_MAX_QP_RD_ATOMIC) && attr->max_rd_atomic) {
5200 hr_reg_write(context, QPC_SR_MAX, fls(attr->max_rd_atomic - 1));
5201 hr_reg_clear(qpc_mask, QPC_SR_MAX);
5202 }
5203
5204 if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC))
5205 set_access_flags(hr_qp, context, qpc_mask, attr, attr_mask);
5206
5207 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
5208 hr_reg_write(context, QPC_MIN_RNR_TIME,
5209 hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 ?
5210 HNS_ROCE_RNR_TIMER_10NS : attr->min_rnr_timer);
5211 hr_reg_clear(qpc_mask, QPC_MIN_RNR_TIME);
5212 }
5213
5214 if (attr_mask & IB_QP_RQ_PSN) {
5215 hr_reg_write(context, QPC_RX_REQ_EPSN, attr->rq_psn);
5216 hr_reg_clear(qpc_mask, QPC_RX_REQ_EPSN);
5217
5218 hr_reg_write(context, QPC_RAQ_PSN, attr->rq_psn - 1);
5219 hr_reg_clear(qpc_mask, QPC_RAQ_PSN);
5220 }
5221
5222 if (attr_mask & IB_QP_QKEY) {
5223 context->qkey_xrcd = cpu_to_le32(attr->qkey);
5224 qpc_mask->qkey_xrcd = 0;
5225 hr_qp->qkey = attr->qkey;
5226 }
5227
5228 return ret;
5229 }
5230
hns_roce_v2_record_opt_fields(struct ib_qp * ibqp,const struct ib_qp_attr * attr,int attr_mask)5231 static void hns_roce_v2_record_opt_fields(struct ib_qp *ibqp,
5232 const struct ib_qp_attr *attr,
5233 int attr_mask)
5234 {
5235 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
5236 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
5237
5238 if (attr_mask & IB_QP_ACCESS_FLAGS)
5239 hr_qp->atomic_rd_en = attr->qp_access_flags;
5240
5241 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
5242 hr_qp->resp_depth = attr->max_dest_rd_atomic;
5243 if (attr_mask & IB_QP_PORT) {
5244 hr_qp->port = attr->port_num - 1;
5245 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port];
5246 }
5247 }
5248
clear_qp(struct hns_roce_qp * hr_qp)5249 static void clear_qp(struct hns_roce_qp *hr_qp)
5250 {
5251 struct ib_qp *ibqp = &hr_qp->ibqp;
5252
5253 if (ibqp->send_cq)
5254 hns_roce_v2_cq_clean(to_hr_cq(ibqp->send_cq),
5255 hr_qp->qpn, NULL);
5256
5257 if (ibqp->recv_cq && ibqp->recv_cq != ibqp->send_cq)
5258 hns_roce_v2_cq_clean(to_hr_cq(ibqp->recv_cq),
5259 hr_qp->qpn, ibqp->srq ?
5260 to_hr_srq(ibqp->srq) : NULL);
5261
5262 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
5263 *hr_qp->rdb.db_record = 0;
5264
5265 hr_qp->rq.head = 0;
5266 hr_qp->rq.tail = 0;
5267 hr_qp->sq.head = 0;
5268 hr_qp->sq.tail = 0;
5269 hr_qp->next_sge = 0;
5270 }
5271
v2_set_flushed_fields(struct ib_qp * ibqp,struct hns_roce_v2_qp_context * context,struct hns_roce_v2_qp_context * qpc_mask)5272 static void v2_set_flushed_fields(struct ib_qp *ibqp,
5273 struct hns_roce_v2_qp_context *context,
5274 struct hns_roce_v2_qp_context *qpc_mask)
5275 {
5276 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
5277 unsigned long sq_flag = 0;
5278 unsigned long rq_flag = 0;
5279
5280 if (ibqp->qp_type == IB_QPT_XRC_TGT)
5281 return;
5282
5283 spin_lock_irqsave(&hr_qp->sq.lock, sq_flag);
5284 hr_reg_write(context, QPC_SQ_PRODUCER_IDX, hr_qp->sq.head);
5285 hr_reg_clear(qpc_mask, QPC_SQ_PRODUCER_IDX);
5286 hr_qp->state = IB_QPS_ERR;
5287 spin_unlock_irqrestore(&hr_qp->sq.lock, sq_flag);
5288
5289 if (ibqp->srq || ibqp->qp_type == IB_QPT_XRC_INI) /* no RQ */
5290 return;
5291
5292 spin_lock_irqsave(&hr_qp->rq.lock, rq_flag);
5293 hr_reg_write(context, QPC_RQ_PRODUCER_IDX, hr_qp->rq.head);
5294 hr_reg_clear(qpc_mask, QPC_RQ_PRODUCER_IDX);
5295 spin_unlock_irqrestore(&hr_qp->rq.lock, rq_flag);
5296 }
5297
hns_roce_v2_modify_qp(struct ib_qp * ibqp,const struct ib_qp_attr * attr,int attr_mask,enum ib_qp_state cur_state,enum ib_qp_state new_state,struct ib_udata * udata)5298 static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
5299 const struct ib_qp_attr *attr,
5300 int attr_mask, enum ib_qp_state cur_state,
5301 enum ib_qp_state new_state, struct ib_udata *udata)
5302 {
5303 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
5304 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
5305 struct hns_roce_v2_qp_context ctx[2];
5306 struct hns_roce_v2_qp_context *context = ctx;
5307 struct hns_roce_v2_qp_context *qpc_mask = ctx + 1;
5308 struct ib_device *ibdev = &hr_dev->ib_dev;
5309 int ret;
5310
5311 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
5312 return -EOPNOTSUPP;
5313
5314 /*
5315 * In v2 engine, software pass context and context mask to hardware
5316 * when modifying qp. If software need modify some fields in context,
5317 * we should set all bits of the relevant fields in context mask to
5318 * 0 at the same time, else set them to 0x1.
5319 */
5320 memset(context, 0, hr_dev->caps.qpc_sz);
5321 memset(qpc_mask, 0xff, hr_dev->caps.qpc_sz);
5322
5323 ret = hns_roce_v2_set_abs_fields(ibqp, attr, attr_mask, cur_state,
5324 new_state, context, qpc_mask, udata);
5325 if (ret)
5326 goto out;
5327
5328 /* When QP state is err, SQ and RQ WQE should be flushed */
5329 if (new_state == IB_QPS_ERR)
5330 v2_set_flushed_fields(ibqp, context, qpc_mask);
5331
5332 /* Configure the optional fields */
5333 ret = hns_roce_v2_set_opt_fields(ibqp, attr, attr_mask, context,
5334 qpc_mask);
5335 if (ret)
5336 goto out;
5337
5338 hr_reg_write_bool(context, QPC_INV_CREDIT,
5339 to_hr_qp_type(hr_qp->ibqp.qp_type) == SERV_TYPE_XRC ||
5340 ibqp->srq);
5341 hr_reg_clear(qpc_mask, QPC_INV_CREDIT);
5342
5343 /* Every status migrate must change state */
5344 hr_reg_write(context, QPC_QP_ST, new_state);
5345 hr_reg_clear(qpc_mask, QPC_QP_ST);
5346
5347 /* SW pass context to HW */
5348 ret = hns_roce_v2_qp_modify(hr_dev, context, qpc_mask, hr_qp);
5349 if (ret) {
5350 ibdev_err_ratelimited(ibdev, "failed to modify QP, ret = %d.\n", ret);
5351 goto out;
5352 }
5353
5354 hr_qp->state = new_state;
5355
5356 hns_roce_v2_record_opt_fields(ibqp, attr, attr_mask);
5357
5358 if (new_state == IB_QPS_RESET && !ibqp->uobject)
5359 clear_qp(hr_qp);
5360
5361 out:
5362 return ret;
5363 }
5364
to_ib_qp_st(enum hns_roce_v2_qp_state state)5365 static int to_ib_qp_st(enum hns_roce_v2_qp_state state)
5366 {
5367 static const enum ib_qp_state map[] = {
5368 [HNS_ROCE_QP_ST_RST] = IB_QPS_RESET,
5369 [HNS_ROCE_QP_ST_INIT] = IB_QPS_INIT,
5370 [HNS_ROCE_QP_ST_RTR] = IB_QPS_RTR,
5371 [HNS_ROCE_QP_ST_RTS] = IB_QPS_RTS,
5372 [HNS_ROCE_QP_ST_SQD] = IB_QPS_SQD,
5373 [HNS_ROCE_QP_ST_SQER] = IB_QPS_SQE,
5374 [HNS_ROCE_QP_ST_ERR] = IB_QPS_ERR,
5375 [HNS_ROCE_QP_ST_SQ_DRAINING] = IB_QPS_SQD
5376 };
5377
5378 return (state < ARRAY_SIZE(map)) ? map[state] : -1;
5379 }
5380
hns_roce_v2_query_qpc(struct hns_roce_dev * hr_dev,u32 qpn,void * buffer)5381 static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev, u32 qpn,
5382 void *buffer)
5383 {
5384 struct hns_roce_cmd_mailbox *mailbox;
5385 int ret;
5386
5387 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5388 if (IS_ERR(mailbox))
5389 return PTR_ERR(mailbox);
5390
5391 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_QPC,
5392 qpn);
5393 if (ret)
5394 goto out;
5395
5396 memcpy(buffer, mailbox->buf, hr_dev->caps.qpc_sz);
5397
5398 out:
5399 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5400 return ret;
5401 }
5402
hns_roce_v2_query_srqc(struct hns_roce_dev * hr_dev,u32 srqn,void * buffer)5403 static int hns_roce_v2_query_srqc(struct hns_roce_dev *hr_dev, u32 srqn,
5404 void *buffer)
5405 {
5406 struct hns_roce_srq_context *context;
5407 struct hns_roce_cmd_mailbox *mailbox;
5408 int ret;
5409
5410 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5411 if (IS_ERR(mailbox))
5412 return PTR_ERR(mailbox);
5413
5414 context = mailbox->buf;
5415 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_SRQC,
5416 srqn);
5417 if (ret)
5418 goto out;
5419
5420 memcpy(buffer, context, sizeof(*context));
5421
5422 out:
5423 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5424 return ret;
5425 }
5426
hns_roce_v2_query_sccc(struct hns_roce_dev * hr_dev,u32 qpn,void * buffer)5427 static int hns_roce_v2_query_sccc(struct hns_roce_dev *hr_dev, u32 qpn,
5428 void *buffer)
5429 {
5430 struct hns_roce_v2_scc_context *context;
5431 struct hns_roce_cmd_mailbox *mailbox;
5432 int ret;
5433
5434 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5435 if (IS_ERR(mailbox))
5436 return PTR_ERR(mailbox);
5437
5438 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_SCCC,
5439 qpn);
5440 if (ret)
5441 goto out;
5442
5443 context = mailbox->buf;
5444 memcpy(buffer, context, sizeof(*context));
5445
5446 out:
5447 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5448 return ret;
5449 }
5450
get_qp_timeout_attr(struct hns_roce_dev * hr_dev,struct hns_roce_v2_qp_context * context)5451 static u8 get_qp_timeout_attr(struct hns_roce_dev *hr_dev,
5452 struct hns_roce_v2_qp_context *context)
5453 {
5454 u8 timeout;
5455
5456 timeout = (u8)hr_reg_read(context, QPC_AT);
5457 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08)
5458 timeout -= HNS_ROCE_V2_QP_ACK_TIMEOUT_OFS_HIP08;
5459
5460 return timeout;
5461 }
5462
hns_roce_v2_query_qp(struct ib_qp * ibqp,struct ib_qp_attr * qp_attr,int qp_attr_mask,struct ib_qp_init_attr * qp_init_attr)5463 static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
5464 int qp_attr_mask,
5465 struct ib_qp_init_attr *qp_init_attr)
5466 {
5467 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
5468 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
5469 struct hns_roce_v2_qp_context context = {};
5470 struct ib_device *ibdev = &hr_dev->ib_dev;
5471 int tmp_qp_state;
5472 int state;
5473 int ret;
5474
5475 memset(qp_attr, 0, sizeof(*qp_attr));
5476 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
5477
5478 mutex_lock(&hr_qp->mutex);
5479
5480 if (hr_qp->state == IB_QPS_RESET) {
5481 qp_attr->qp_state = IB_QPS_RESET;
5482 ret = 0;
5483 goto done;
5484 }
5485
5486 ret = hns_roce_v2_query_qpc(hr_dev, hr_qp->qpn, &context);
5487 if (ret) {
5488 ibdev_err_ratelimited(ibdev,
5489 "failed to query QPC, ret = %d.\n",
5490 ret);
5491 ret = -EINVAL;
5492 goto out;
5493 }
5494
5495 state = hr_reg_read(&context, QPC_QP_ST);
5496 tmp_qp_state = to_ib_qp_st((enum hns_roce_v2_qp_state)state);
5497 if (tmp_qp_state == -1) {
5498 ibdev_err_ratelimited(ibdev, "Illegal ib_qp_state\n");
5499 ret = -EINVAL;
5500 goto out;
5501 }
5502 hr_qp->state = (u8)tmp_qp_state;
5503 qp_attr->qp_state = (enum ib_qp_state)hr_qp->state;
5504 qp_attr->path_mtu = (enum ib_mtu)hr_reg_read(&context, QPC_MTU);
5505 qp_attr->path_mig_state = IB_MIG_ARMED;
5506 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
5507 if (hr_qp->ibqp.qp_type == IB_QPT_UD)
5508 qp_attr->qkey = le32_to_cpu(context.qkey_xrcd);
5509
5510 qp_attr->rq_psn = hr_reg_read(&context, QPC_RX_REQ_EPSN);
5511 qp_attr->sq_psn = (u32)hr_reg_read(&context, QPC_SQ_CUR_PSN);
5512 qp_attr->dest_qp_num = hr_reg_read(&context, QPC_DQPN);
5513 qp_attr->qp_access_flags =
5514 ((hr_reg_read(&context, QPC_RRE)) << V2_QP_RRE_S) |
5515 ((hr_reg_read(&context, QPC_RWE)) << V2_QP_RWE_S) |
5516 ((hr_reg_read(&context, QPC_ATE)) << V2_QP_ATE_S);
5517
5518 if (hr_qp->ibqp.qp_type == IB_QPT_RC ||
5519 hr_qp->ibqp.qp_type == IB_QPT_XRC_INI ||
5520 hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT) {
5521 struct ib_global_route *grh =
5522 rdma_ah_retrieve_grh(&qp_attr->ah_attr);
5523
5524 rdma_ah_set_sl(&qp_attr->ah_attr,
5525 hr_reg_read(&context, QPC_SL));
5526 rdma_ah_set_port_num(&qp_attr->ah_attr, hr_qp->port + 1);
5527 rdma_ah_set_ah_flags(&qp_attr->ah_attr, IB_AH_GRH);
5528 grh->flow_label = hr_reg_read(&context, QPC_FL);
5529 grh->sgid_index = hr_reg_read(&context, QPC_GMV_IDX);
5530 grh->hop_limit = hr_reg_read(&context, QPC_HOPLIMIT);
5531 grh->traffic_class = hr_reg_read(&context, QPC_TC);
5532
5533 memcpy(grh->dgid.raw, context.dgid, sizeof(grh->dgid.raw));
5534 }
5535
5536 qp_attr->port_num = hr_qp->port + 1;
5537 qp_attr->sq_draining = 0;
5538 qp_attr->max_rd_atomic = 1 << hr_reg_read(&context, QPC_SR_MAX);
5539 qp_attr->max_dest_rd_atomic = 1 << hr_reg_read(&context, QPC_RR_MAX);
5540
5541 qp_attr->min_rnr_timer = (u8)hr_reg_read(&context, QPC_MIN_RNR_TIME);
5542 qp_attr->timeout = get_qp_timeout_attr(hr_dev, &context);
5543 qp_attr->retry_cnt = hr_reg_read(&context, QPC_RETRY_NUM_INIT);
5544 qp_attr->rnr_retry = hr_reg_read(&context, QPC_RNR_NUM_INIT);
5545
5546 done:
5547 qp_attr->cur_qp_state = qp_attr->qp_state;
5548 qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt;
5549 qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge;
5550 qp_attr->cap.max_inline_data = hr_qp->max_inline_data;
5551
5552 qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt;
5553 qp_attr->cap.max_send_sge = hr_qp->sq.max_gs;
5554
5555 qp_init_attr->qp_context = ibqp->qp_context;
5556 qp_init_attr->qp_type = ibqp->qp_type;
5557 qp_init_attr->recv_cq = ibqp->recv_cq;
5558 qp_init_attr->send_cq = ibqp->send_cq;
5559 qp_init_attr->srq = ibqp->srq;
5560 qp_init_attr->cap = qp_attr->cap;
5561 qp_init_attr->sq_sig_type = hr_qp->sq_signal_bits;
5562
5563 out:
5564 mutex_unlock(&hr_qp->mutex);
5565 return ret;
5566 }
5567
modify_qp_is_ok(struct hns_roce_qp * hr_qp)5568 static inline int modify_qp_is_ok(struct hns_roce_qp *hr_qp)
5569 {
5570 return ((hr_qp->ibqp.qp_type == IB_QPT_RC ||
5571 hr_qp->ibqp.qp_type == IB_QPT_UD ||
5572 hr_qp->ibqp.qp_type == IB_QPT_XRC_INI ||
5573 hr_qp->ibqp.qp_type == IB_QPT_XRC_TGT) &&
5574 hr_qp->state != IB_QPS_RESET);
5575 }
5576
hns_roce_v2_destroy_qp_common(struct hns_roce_dev * hr_dev,struct hns_roce_qp * hr_qp,struct ib_udata * udata)5577 static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev,
5578 struct hns_roce_qp *hr_qp,
5579 struct ib_udata *udata)
5580 {
5581 struct ib_device *ibdev = &hr_dev->ib_dev;
5582 struct hns_roce_cq *send_cq, *recv_cq;
5583 unsigned long flags;
5584 int ret = 0;
5585
5586 if (modify_qp_is_ok(hr_qp)) {
5587 /* Modify qp to reset before destroying qp */
5588 ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0,
5589 hr_qp->state, IB_QPS_RESET, udata);
5590 if (ret)
5591 ibdev_err_ratelimited(ibdev,
5592 "failed to modify QP to RST, ret = %d.\n",
5593 ret);
5594 }
5595
5596 send_cq = hr_qp->ibqp.send_cq ? to_hr_cq(hr_qp->ibqp.send_cq) : NULL;
5597 recv_cq = hr_qp->ibqp.recv_cq ? to_hr_cq(hr_qp->ibqp.recv_cq) : NULL;
5598
5599 spin_lock_irqsave(&hr_dev->qp_list_lock, flags);
5600 hns_roce_lock_cqs(send_cq, recv_cq);
5601
5602 if (!udata) {
5603 if (recv_cq)
5604 __hns_roce_v2_cq_clean(recv_cq, hr_qp->qpn,
5605 (hr_qp->ibqp.srq ?
5606 to_hr_srq(hr_qp->ibqp.srq) :
5607 NULL));
5608
5609 if (send_cq && send_cq != recv_cq)
5610 __hns_roce_v2_cq_clean(send_cq, hr_qp->qpn, NULL);
5611 }
5612
5613 hns_roce_qp_remove(hr_dev, hr_qp);
5614
5615 hns_roce_unlock_cqs(send_cq, recv_cq);
5616 spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags);
5617
5618 return ret;
5619 }
5620
put_dip_ctx_idx(struct hns_roce_dev * hr_dev,struct hns_roce_qp * hr_qp)5621 static void put_dip_ctx_idx(struct hns_roce_dev *hr_dev,
5622 struct hns_roce_qp *hr_qp)
5623 {
5624 struct hns_roce_dip *hr_dip = hr_qp->dip;
5625
5626 if (!hr_dip)
5627 return;
5628
5629 xa_lock(&hr_dev->qp_table.dip_xa);
5630
5631 hr_dip->qp_cnt--;
5632 if (!hr_dip->qp_cnt)
5633 memset(hr_dip->dgid, 0, GID_LEN_V2);
5634
5635 xa_unlock(&hr_dev->qp_table.dip_xa);
5636 }
5637
hns_roce_v2_destroy_qp(struct ib_qp * ibqp,struct ib_udata * udata)5638 int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
5639 {
5640 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
5641 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
5642 unsigned long flags;
5643 int ret;
5644
5645 /* Make sure flush_cqe() is completed */
5646 spin_lock_irqsave(&hr_qp->flush_lock, flags);
5647 set_bit(HNS_ROCE_STOP_FLUSH_FLAG, &hr_qp->flush_flag);
5648 spin_unlock_irqrestore(&hr_qp->flush_lock, flags);
5649 flush_work(&hr_qp->flush_work.work);
5650
5651 if (hr_qp->cong_type == CONG_TYPE_DIP)
5652 put_dip_ctx_idx(hr_dev, hr_qp);
5653
5654 ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata);
5655 if (ret)
5656 ibdev_err_ratelimited(&hr_dev->ib_dev,
5657 "failed to destroy QP, QPN = 0x%06lx, ret = %d.\n",
5658 hr_qp->qpn, ret);
5659
5660 hns_roce_qp_destroy(hr_dev, hr_qp, udata);
5661
5662 return 0;
5663 }
5664
hns_roce_v2_qp_flow_control_init(struct hns_roce_dev * hr_dev,struct hns_roce_qp * hr_qp)5665 static int hns_roce_v2_qp_flow_control_init(struct hns_roce_dev *hr_dev,
5666 struct hns_roce_qp *hr_qp)
5667 {
5668 struct ib_device *ibdev = &hr_dev->ib_dev;
5669 struct hns_roce_sccc_clr_done *resp;
5670 struct hns_roce_sccc_clr *clr;
5671 struct hns_roce_cmq_desc desc;
5672 int ret, i;
5673
5674 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
5675 return 0;
5676
5677 mutex_lock(&hr_dev->qp_table.scc_mutex);
5678
5679 /* set scc ctx clear done flag */
5680 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_RESET_SCCC, false);
5681 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
5682 if (ret) {
5683 ibdev_err(ibdev, "failed to reset SCC ctx, ret = %d.\n", ret);
5684 goto out;
5685 }
5686
5687 /* clear scc context */
5688 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CLR_SCCC, false);
5689 clr = (struct hns_roce_sccc_clr *)desc.data;
5690 clr->qpn = cpu_to_le32(hr_qp->qpn);
5691 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
5692 if (ret) {
5693 ibdev_err(ibdev, "failed to clear SCC ctx, ret = %d.\n", ret);
5694 goto out;
5695 }
5696
5697 /* query scc context clear is done or not */
5698 resp = (struct hns_roce_sccc_clr_done *)desc.data;
5699 for (i = 0; i <= HNS_ROCE_CMQ_SCC_CLR_DONE_CNT; i++) {
5700 hns_roce_cmq_setup_basic_desc(&desc,
5701 HNS_ROCE_OPC_QUERY_SCCC, true);
5702 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
5703 if (ret) {
5704 ibdev_err(ibdev, "failed to query clr cmq, ret = %d\n",
5705 ret);
5706 goto out;
5707 }
5708
5709 if (resp->clr_done)
5710 goto out;
5711
5712 msleep(20);
5713 }
5714
5715 ibdev_err(ibdev, "query SCC clr done flag overtime.\n");
5716 ret = -ETIMEDOUT;
5717
5718 out:
5719 mutex_unlock(&hr_dev->qp_table.scc_mutex);
5720 return ret;
5721 }
5722
5723 #define DMA_IDX_SHIFT 3
5724 #define DMA_WQE_SHIFT 3
5725
hns_roce_v2_write_srqc_index_queue(struct hns_roce_srq * srq,struct hns_roce_srq_context * ctx)5726 static int hns_roce_v2_write_srqc_index_queue(struct hns_roce_srq *srq,
5727 struct hns_roce_srq_context *ctx)
5728 {
5729 struct hns_roce_idx_que *idx_que = &srq->idx_que;
5730 struct ib_device *ibdev = srq->ibsrq.device;
5731 struct hns_roce_dev *hr_dev = to_hr_dev(ibdev);
5732 u64 mtts_idx[MTT_MIN_COUNT] = {};
5733 dma_addr_t dma_handle_idx;
5734 int ret;
5735
5736 /* Get physical address of idx que buf */
5737 ret = hns_roce_mtr_find(hr_dev, &idx_que->mtr, 0, mtts_idx,
5738 ARRAY_SIZE(mtts_idx));
5739 if (ret) {
5740 ibdev_err(ibdev, "failed to find mtr for SRQ idx, ret = %d.\n",
5741 ret);
5742 return ret;
5743 }
5744
5745 dma_handle_idx = hns_roce_get_mtr_ba(&idx_que->mtr);
5746
5747 hr_reg_write(ctx, SRQC_IDX_HOP_NUM,
5748 to_hr_hem_hopnum(hr_dev->caps.idx_hop_num, srq->wqe_cnt));
5749
5750 hr_reg_write(ctx, SRQC_IDX_BT_BA_L, dma_handle_idx >> DMA_IDX_SHIFT);
5751 hr_reg_write(ctx, SRQC_IDX_BT_BA_H,
5752 upper_32_bits(dma_handle_idx >> DMA_IDX_SHIFT));
5753
5754 hr_reg_write(ctx, SRQC_IDX_BA_PG_SZ,
5755 to_hr_hw_page_shift(idx_que->mtr.hem_cfg.ba_pg_shift));
5756 hr_reg_write(ctx, SRQC_IDX_BUF_PG_SZ,
5757 to_hr_hw_page_shift(idx_que->mtr.hem_cfg.buf_pg_shift));
5758
5759 hr_reg_write(ctx, SRQC_IDX_CUR_BLK_ADDR_L,
5760 to_hr_hw_page_addr(mtts_idx[0]));
5761 hr_reg_write(ctx, SRQC_IDX_CUR_BLK_ADDR_H,
5762 upper_32_bits(to_hr_hw_page_addr(mtts_idx[0])));
5763
5764 hr_reg_write(ctx, SRQC_IDX_NXT_BLK_ADDR_L,
5765 to_hr_hw_page_addr(mtts_idx[1]));
5766 hr_reg_write(ctx, SRQC_IDX_NXT_BLK_ADDR_H,
5767 upper_32_bits(to_hr_hw_page_addr(mtts_idx[1])));
5768
5769 return 0;
5770 }
5771
hns_roce_v2_write_srqc(struct hns_roce_srq * srq,void * mb_buf)5772 static int hns_roce_v2_write_srqc(struct hns_roce_srq *srq, void *mb_buf)
5773 {
5774 struct ib_device *ibdev = srq->ibsrq.device;
5775 struct hns_roce_dev *hr_dev = to_hr_dev(ibdev);
5776 struct hns_roce_srq_context *ctx = mb_buf;
5777 u64 mtts_wqe[MTT_MIN_COUNT] = {};
5778 dma_addr_t dma_handle_wqe;
5779 int ret;
5780
5781 memset(ctx, 0, sizeof(*ctx));
5782
5783 /* Get the physical address of srq buf */
5784 ret = hns_roce_mtr_find(hr_dev, &srq->buf_mtr, 0, mtts_wqe,
5785 ARRAY_SIZE(mtts_wqe));
5786 if (ret) {
5787 ibdev_err(ibdev, "failed to find mtr for SRQ WQE, ret = %d.\n",
5788 ret);
5789 return ret;
5790 }
5791
5792 dma_handle_wqe = hns_roce_get_mtr_ba(&srq->buf_mtr);
5793
5794 hr_reg_write(ctx, SRQC_SRQ_ST, 1);
5795 hr_reg_write_bool(ctx, SRQC_SRQ_TYPE,
5796 srq->ibsrq.srq_type == IB_SRQT_XRC);
5797 hr_reg_write(ctx, SRQC_PD, to_hr_pd(srq->ibsrq.pd)->pdn);
5798 hr_reg_write(ctx, SRQC_SRQN, srq->srqn);
5799 hr_reg_write(ctx, SRQC_XRCD, srq->xrcdn);
5800 hr_reg_write(ctx, SRQC_XRC_CQN, srq->cqn);
5801 hr_reg_write(ctx, SRQC_SHIFT, ilog2(srq->wqe_cnt));
5802 hr_reg_write(ctx, SRQC_RQWS,
5803 srq->max_gs <= 0 ? 0 : fls(srq->max_gs - 1));
5804
5805 hr_reg_write(ctx, SRQC_WQE_HOP_NUM,
5806 to_hr_hem_hopnum(hr_dev->caps.srqwqe_hop_num,
5807 srq->wqe_cnt));
5808
5809 hr_reg_write(ctx, SRQC_WQE_BT_BA_L, dma_handle_wqe >> DMA_WQE_SHIFT);
5810 hr_reg_write(ctx, SRQC_WQE_BT_BA_H,
5811 upper_32_bits(dma_handle_wqe >> DMA_WQE_SHIFT));
5812
5813 hr_reg_write(ctx, SRQC_WQE_BA_PG_SZ,
5814 to_hr_hw_page_shift(srq->buf_mtr.hem_cfg.ba_pg_shift));
5815 hr_reg_write(ctx, SRQC_WQE_BUF_PG_SZ,
5816 to_hr_hw_page_shift(srq->buf_mtr.hem_cfg.buf_pg_shift));
5817
5818 if (srq->cap_flags & HNS_ROCE_SRQ_CAP_RECORD_DB) {
5819 hr_reg_enable(ctx, SRQC_DB_RECORD_EN);
5820 hr_reg_write(ctx, SRQC_DB_RECORD_ADDR_L,
5821 lower_32_bits(srq->rdb.dma) >> 1);
5822 hr_reg_write(ctx, SRQC_DB_RECORD_ADDR_H,
5823 upper_32_bits(srq->rdb.dma));
5824 }
5825
5826 return hns_roce_v2_write_srqc_index_queue(srq, ctx);
5827 }
5828
hns_roce_v2_modify_srq(struct ib_srq * ibsrq,struct ib_srq_attr * srq_attr,enum ib_srq_attr_mask srq_attr_mask,struct ib_udata * udata)5829 static int hns_roce_v2_modify_srq(struct ib_srq *ibsrq,
5830 struct ib_srq_attr *srq_attr,
5831 enum ib_srq_attr_mask srq_attr_mask,
5832 struct ib_udata *udata)
5833 {
5834 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
5835 struct hns_roce_srq *srq = to_hr_srq(ibsrq);
5836 struct hns_roce_srq_context *srq_context;
5837 struct hns_roce_srq_context *srqc_mask;
5838 struct hns_roce_cmd_mailbox *mailbox;
5839 int ret = 0;
5840
5841 /* Resizing SRQs is not supported yet */
5842 if (srq_attr_mask & IB_SRQ_MAX_WR) {
5843 ret = -EOPNOTSUPP;
5844 goto out;
5845 }
5846
5847 if (srq_attr_mask & IB_SRQ_LIMIT) {
5848 if (srq_attr->srq_limit > srq->wqe_cnt) {
5849 ret = -EINVAL;
5850 goto out;
5851 }
5852
5853 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5854 if (IS_ERR(mailbox)) {
5855 ret = PTR_ERR(mailbox);
5856 goto out;
5857 }
5858
5859 srq_context = mailbox->buf;
5860 srqc_mask = (struct hns_roce_srq_context *)mailbox->buf + 1;
5861
5862 memset(srqc_mask, 0xff, sizeof(*srqc_mask));
5863
5864 hr_reg_write(srq_context, SRQC_LIMIT_WL, srq_attr->srq_limit);
5865 hr_reg_clear(srqc_mask, SRQC_LIMIT_WL);
5866
5867 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0,
5868 HNS_ROCE_CMD_MODIFY_SRQC, srq->srqn);
5869 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5870 if (ret)
5871 ibdev_err(&hr_dev->ib_dev,
5872 "failed to handle cmd of modifying SRQ, ret = %d.\n",
5873 ret);
5874 }
5875
5876 out:
5877 if (ret)
5878 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_SRQ_MODIFY_ERR_CNT]);
5879
5880 return ret;
5881 }
5882
hns_roce_v2_query_srq(struct ib_srq * ibsrq,struct ib_srq_attr * attr)5883 static int hns_roce_v2_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
5884 {
5885 struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
5886 struct hns_roce_srq *srq = to_hr_srq(ibsrq);
5887 struct hns_roce_srq_context *srq_context;
5888 struct hns_roce_cmd_mailbox *mailbox;
5889 int ret;
5890
5891 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5892 if (IS_ERR(mailbox))
5893 return PTR_ERR(mailbox);
5894
5895 srq_context = mailbox->buf;
5896 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma,
5897 HNS_ROCE_CMD_QUERY_SRQC, srq->srqn);
5898 if (ret) {
5899 ibdev_err(&hr_dev->ib_dev,
5900 "failed to process cmd of querying SRQ, ret = %d.\n",
5901 ret);
5902 goto out;
5903 }
5904
5905 attr->srq_limit = hr_reg_read(srq_context, SRQC_LIMIT_WL);
5906 attr->max_wr = srq->wqe_cnt;
5907 attr->max_sge = srq->max_gs - srq->rsv_sge;
5908
5909 out:
5910 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5911 return ret;
5912 }
5913
hns_roce_v2_modify_cq(struct ib_cq * cq,u16 cq_count,u16 cq_period)5914 static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
5915 {
5916 struct hns_roce_dev *hr_dev = to_hr_dev(cq->device);
5917 struct hns_roce_v2_cq_context *cq_context;
5918 struct hns_roce_cq *hr_cq = to_hr_cq(cq);
5919 struct hns_roce_v2_cq_context *cqc_mask;
5920 struct hns_roce_cmd_mailbox *mailbox;
5921 int ret;
5922
5923 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5924 ret = PTR_ERR_OR_ZERO(mailbox);
5925 if (ret)
5926 goto err_out;
5927
5928 cq_context = mailbox->buf;
5929 cqc_mask = (struct hns_roce_v2_cq_context *)mailbox->buf + 1;
5930
5931 memset(cqc_mask, 0xff, sizeof(*cqc_mask));
5932
5933 hr_reg_write(cq_context, CQC_CQ_MAX_CNT, cq_count);
5934 hr_reg_clear(cqc_mask, CQC_CQ_MAX_CNT);
5935
5936 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {
5937 if (cq_period * HNS_ROCE_CLOCK_ADJUST > USHRT_MAX) {
5938 dev_info(hr_dev->dev,
5939 "cq_period(%u) reached the upper limit, adjusted to 65.\n",
5940 cq_period);
5941 cq_period = HNS_ROCE_MAX_CQ_PERIOD_HIP08;
5942 }
5943 cq_period *= HNS_ROCE_CLOCK_ADJUST;
5944 }
5945 hr_reg_write(cq_context, CQC_CQ_PERIOD, cq_period);
5946 hr_reg_clear(cqc_mask, CQC_CQ_PERIOD);
5947
5948 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0,
5949 HNS_ROCE_CMD_MODIFY_CQC, hr_cq->cqn);
5950 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5951 if (ret)
5952 ibdev_err_ratelimited(&hr_dev->ib_dev,
5953 "failed to process cmd when modifying CQ, ret = %d.\n",
5954 ret);
5955
5956 err_out:
5957 if (ret)
5958 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_CQ_MODIFY_ERR_CNT]);
5959
5960 return ret;
5961 }
5962
hns_roce_v2_query_cqc(struct hns_roce_dev * hr_dev,u32 cqn,void * buffer)5963 static int hns_roce_v2_query_cqc(struct hns_roce_dev *hr_dev, u32 cqn,
5964 void *buffer)
5965 {
5966 struct hns_roce_v2_cq_context *context;
5967 struct hns_roce_cmd_mailbox *mailbox;
5968 int ret;
5969
5970 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5971 if (IS_ERR(mailbox))
5972 return PTR_ERR(mailbox);
5973
5974 context = mailbox->buf;
5975 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma,
5976 HNS_ROCE_CMD_QUERY_CQC, cqn);
5977 if (ret) {
5978 ibdev_err_ratelimited(&hr_dev->ib_dev,
5979 "failed to process cmd when querying CQ, ret = %d.\n",
5980 ret);
5981 goto err_mailbox;
5982 }
5983
5984 memcpy(buffer, context, sizeof(*context));
5985
5986 err_mailbox:
5987 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5988
5989 return ret;
5990 }
5991
hns_roce_v2_query_mpt(struct hns_roce_dev * hr_dev,u32 key,void * buffer)5992 static int hns_roce_v2_query_mpt(struct hns_roce_dev *hr_dev, u32 key,
5993 void *buffer)
5994 {
5995 struct hns_roce_v2_mpt_entry *context;
5996 struct hns_roce_cmd_mailbox *mailbox;
5997 int ret;
5998
5999 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
6000 if (IS_ERR(mailbox))
6001 return PTR_ERR(mailbox);
6002
6003 context = mailbox->buf;
6004 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_MPT,
6005 key_to_hw_index(key));
6006 if (ret) {
6007 ibdev_err(&hr_dev->ib_dev,
6008 "failed to process cmd when querying MPT, ret = %d.\n",
6009 ret);
6010 goto err_mailbox;
6011 }
6012
6013 memcpy(buffer, context, sizeof(*context));
6014
6015 err_mailbox:
6016 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
6017
6018 return ret;
6019 }
6020
dump_aeqe_log(struct hns_roce_work * irq_work)6021 static void dump_aeqe_log(struct hns_roce_work *irq_work)
6022 {
6023 struct hns_roce_dev *hr_dev = irq_work->hr_dev;
6024 struct ib_device *ibdev = &hr_dev->ib_dev;
6025
6026 switch (irq_work->event_type) {
6027 case HNS_ROCE_EVENT_TYPE_PATH_MIG:
6028 ibdev_info(ibdev, "path migrated succeeded.\n");
6029 break;
6030 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
6031 ibdev_warn(ibdev, "path migration failed.\n");
6032 break;
6033 case HNS_ROCE_EVENT_TYPE_COMM_EST:
6034 break;
6035 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
6036 ibdev_dbg(ibdev, "send queue drained.\n");
6037 break;
6038 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
6039 ibdev_err(ibdev, "local work queue 0x%x catast error, sub_event type is: %d\n",
6040 irq_work->queue_num, irq_work->sub_type);
6041 break;
6042 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
6043 ibdev_err(ibdev, "invalid request local work queue 0x%x error.\n",
6044 irq_work->queue_num);
6045 break;
6046 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
6047 ibdev_err(ibdev, "local access violation work queue 0x%x error, sub_event type is: %d\n",
6048 irq_work->queue_num, irq_work->sub_type);
6049 break;
6050 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
6051 ibdev_dbg(ibdev, "SRQ limit reach.\n");
6052 break;
6053 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
6054 ibdev_dbg(ibdev, "SRQ last wqe reach.\n");
6055 break;
6056 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
6057 ibdev_err(ibdev, "SRQ catas error.\n");
6058 break;
6059 case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
6060 ibdev_err(ibdev, "CQ 0x%x access err.\n", irq_work->queue_num);
6061 break;
6062 case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
6063 ibdev_warn(ibdev, "CQ 0x%x overflow\n", irq_work->queue_num);
6064 break;
6065 case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
6066 ibdev_warn(ibdev, "DB overflow.\n");
6067 break;
6068 case HNS_ROCE_EVENT_TYPE_MB:
6069 break;
6070 case HNS_ROCE_EVENT_TYPE_FLR:
6071 ibdev_warn(ibdev, "function level reset.\n");
6072 break;
6073 case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION:
6074 ibdev_err(ibdev, "xrc domain violation error.\n");
6075 break;
6076 case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH:
6077 ibdev_err(ibdev, "invalid xrceth error.\n");
6078 break;
6079 default:
6080 ibdev_info(ibdev, "Undefined event %d.\n",
6081 irq_work->event_type);
6082 break;
6083 }
6084 }
6085
hns_roce_irq_work_handle(struct work_struct * work)6086 static void hns_roce_irq_work_handle(struct work_struct *work)
6087 {
6088 struct hns_roce_work *irq_work =
6089 container_of(work, struct hns_roce_work, work);
6090 struct hns_roce_dev *hr_dev = irq_work->hr_dev;
6091 int event_type = irq_work->event_type;
6092 u32 queue_num = irq_work->queue_num;
6093
6094 switch (event_type) {
6095 case HNS_ROCE_EVENT_TYPE_PATH_MIG:
6096 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
6097 case HNS_ROCE_EVENT_TYPE_COMM_EST:
6098 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
6099 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
6100 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
6101 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
6102 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
6103 case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION:
6104 case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH:
6105 hns_roce_qp_event(hr_dev, queue_num, event_type);
6106 break;
6107 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
6108 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
6109 hns_roce_srq_event(hr_dev, queue_num, event_type);
6110 break;
6111 case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
6112 case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
6113 hns_roce_cq_event(hr_dev, queue_num, event_type);
6114 break;
6115 default:
6116 break;
6117 }
6118
6119 dump_aeqe_log(irq_work);
6120
6121 kfree(irq_work);
6122 }
6123
hns_roce_v2_init_irq_work(struct hns_roce_dev * hr_dev,struct hns_roce_eq * eq,u32 queue_num)6124 static void hns_roce_v2_init_irq_work(struct hns_roce_dev *hr_dev,
6125 struct hns_roce_eq *eq, u32 queue_num)
6126 {
6127 struct hns_roce_work *irq_work;
6128
6129 irq_work = kzalloc(sizeof(struct hns_roce_work), GFP_ATOMIC);
6130 if (!irq_work)
6131 return;
6132
6133 INIT_WORK(&irq_work->work, hns_roce_irq_work_handle);
6134 irq_work->hr_dev = hr_dev;
6135 irq_work->event_type = eq->event_type;
6136 irq_work->sub_type = eq->sub_type;
6137 irq_work->queue_num = queue_num;
6138 queue_work(hr_dev->irq_workq, &irq_work->work);
6139 }
6140
update_eq_db(struct hns_roce_eq * eq)6141 static void update_eq_db(struct hns_roce_eq *eq)
6142 {
6143 struct hns_roce_dev *hr_dev = eq->hr_dev;
6144 struct hns_roce_v2_db eq_db = {};
6145
6146 if (eq->type_flag == HNS_ROCE_AEQ) {
6147 hr_reg_write(&eq_db, EQ_DB_CMD,
6148 eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?
6149 HNS_ROCE_EQ_DB_CMD_AEQ :
6150 HNS_ROCE_EQ_DB_CMD_AEQ_ARMED);
6151 } else {
6152 hr_reg_write(&eq_db, EQ_DB_TAG, eq->eqn);
6153
6154 hr_reg_write(&eq_db, EQ_DB_CMD,
6155 eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?
6156 HNS_ROCE_EQ_DB_CMD_CEQ :
6157 HNS_ROCE_EQ_DB_CMD_CEQ_ARMED);
6158 }
6159
6160 hr_reg_write(&eq_db, EQ_DB_CI, eq->cons_index);
6161
6162 hns_roce_write64(hr_dev, (__le32 *)&eq_db, eq->db_reg);
6163 }
6164
next_aeqe_sw_v2(struct hns_roce_eq * eq)6165 static struct hns_roce_aeqe *next_aeqe_sw_v2(struct hns_roce_eq *eq)
6166 {
6167 struct hns_roce_aeqe *aeqe;
6168
6169 aeqe = hns_roce_buf_offset(eq->mtr.kmem,
6170 (eq->cons_index & (eq->entries - 1)) *
6171 eq->eqe_size);
6172
6173 return (hr_reg_read(aeqe, AEQE_OWNER) ^
6174 !!(eq->cons_index & eq->entries)) ? aeqe : NULL;
6175 }
6176
hns_roce_v2_aeq_int(struct hns_roce_dev * hr_dev,struct hns_roce_eq * eq)6177 static irqreturn_t hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev,
6178 struct hns_roce_eq *eq)
6179 {
6180 struct hns_roce_aeqe *aeqe = next_aeqe_sw_v2(eq);
6181 irqreturn_t aeqe_found = IRQ_NONE;
6182 int num_aeqes = 0;
6183 int event_type;
6184 u32 queue_num;
6185 int sub_type;
6186
6187 while (aeqe && num_aeqes < HNS_AEQ_POLLING_BUDGET) {
6188 /* Make sure we read AEQ entry after we have checked the
6189 * ownership bit
6190 */
6191 dma_rmb();
6192
6193 event_type = hr_reg_read(aeqe, AEQE_EVENT_TYPE);
6194 sub_type = hr_reg_read(aeqe, AEQE_SUB_TYPE);
6195 queue_num = hr_reg_read(aeqe, AEQE_EVENT_QUEUE_NUM);
6196
6197 switch (event_type) {
6198 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
6199 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
6200 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
6201 case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION:
6202 case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH:
6203 hns_roce_flush_cqe(hr_dev, queue_num);
6204 break;
6205 case HNS_ROCE_EVENT_TYPE_MB:
6206 hns_roce_cmd_event(hr_dev,
6207 le16_to_cpu(aeqe->event.cmd.token),
6208 aeqe->event.cmd.status,
6209 le64_to_cpu(aeqe->event.cmd.out_param));
6210 break;
6211 default:
6212 break;
6213 }
6214
6215 eq->event_type = event_type;
6216 eq->sub_type = sub_type;
6217 ++eq->cons_index;
6218 aeqe_found = IRQ_HANDLED;
6219
6220 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_AEQE_CNT]);
6221
6222 hns_roce_v2_init_irq_work(hr_dev, eq, queue_num);
6223
6224 aeqe = next_aeqe_sw_v2(eq);
6225 ++num_aeqes;
6226 }
6227
6228 update_eq_db(eq);
6229
6230 return IRQ_RETVAL(aeqe_found);
6231 }
6232
next_ceqe_sw_v2(struct hns_roce_eq * eq)6233 static struct hns_roce_ceqe *next_ceqe_sw_v2(struct hns_roce_eq *eq)
6234 {
6235 struct hns_roce_ceqe *ceqe;
6236
6237 ceqe = hns_roce_buf_offset(eq->mtr.kmem,
6238 (eq->cons_index & (eq->entries - 1)) *
6239 eq->eqe_size);
6240
6241 return (hr_reg_read(ceqe, CEQE_OWNER) ^
6242 !!(eq->cons_index & eq->entries)) ? ceqe : NULL;
6243 }
6244
hns_roce_v2_ceq_int(struct hns_roce_eq * eq)6245 static irqreturn_t hns_roce_v2_ceq_int(struct hns_roce_eq *eq)
6246 {
6247 queue_work(system_bh_wq, &eq->work);
6248
6249 return IRQ_HANDLED;
6250 }
6251
hns_roce_v2_msix_interrupt_eq(int irq,void * eq_ptr)6252 static irqreturn_t hns_roce_v2_msix_interrupt_eq(int irq, void *eq_ptr)
6253 {
6254 struct hns_roce_eq *eq = eq_ptr;
6255 struct hns_roce_dev *hr_dev = eq->hr_dev;
6256 irqreturn_t int_work;
6257
6258 if (eq->type_flag == HNS_ROCE_CEQ)
6259 /* Completion event interrupt */
6260 int_work = hns_roce_v2_ceq_int(eq);
6261 else
6262 /* Asynchronous event interrupt */
6263 int_work = hns_roce_v2_aeq_int(hr_dev, eq);
6264
6265 return IRQ_RETVAL(int_work);
6266 }
6267
abnormal_interrupt_basic(struct hns_roce_dev * hr_dev,u32 int_st)6268 static irqreturn_t abnormal_interrupt_basic(struct hns_roce_dev *hr_dev,
6269 u32 int_st)
6270 {
6271 struct pci_dev *pdev = hr_dev->pci_dev;
6272 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
6273 const struct hnae3_ae_ops *ops = ae_dev->ops;
6274 enum hnae3_reset_type reset_type;
6275 irqreturn_t int_work = IRQ_NONE;
6276 u32 int_en;
6277
6278 int_en = roce_read(hr_dev, ROCEE_VF_ABN_INT_EN_REG);
6279
6280 if (int_st & BIT(HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S)) {
6281 dev_err(hr_dev->dev, "AEQ overflow!\n");
6282
6283 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG,
6284 1 << HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S);
6285
6286 reset_type = hr_dev->is_vf ?
6287 HNAE3_VF_FUNC_RESET : HNAE3_FUNC_RESET;
6288
6289 /* Set reset level for reset_event() */
6290 if (ops->set_default_reset_request)
6291 ops->set_default_reset_request(ae_dev, reset_type);
6292 if (ops->reset_event)
6293 ops->reset_event(pdev, NULL);
6294
6295 int_en |= 1 << HNS_ROCE_V2_VF_ABN_INT_EN_S;
6296 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
6297
6298 int_work = IRQ_HANDLED;
6299 } else {
6300 dev_err(hr_dev->dev, "there is no basic abn irq found.\n");
6301 }
6302
6303 return IRQ_RETVAL(int_work);
6304 }
6305
fmea_ram_ecc_query(struct hns_roce_dev * hr_dev,struct fmea_ram_ecc * ecc_info)6306 static int fmea_ram_ecc_query(struct hns_roce_dev *hr_dev,
6307 struct fmea_ram_ecc *ecc_info)
6308 {
6309 struct hns_roce_cmq_desc desc;
6310 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;
6311 int ret;
6312
6313 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_QUERY_RAM_ECC, true);
6314 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
6315 if (ret)
6316 return ret;
6317
6318 ecc_info->is_ecc_err = hr_reg_read(req, QUERY_RAM_ECC_1BIT_ERR);
6319 ecc_info->res_type = hr_reg_read(req, QUERY_RAM_ECC_RES_TYPE);
6320 ecc_info->index = hr_reg_read(req, QUERY_RAM_ECC_TAG);
6321
6322 return 0;
6323 }
6324
fmea_recover_gmv(struct hns_roce_dev * hr_dev,u32 idx)6325 static int fmea_recover_gmv(struct hns_roce_dev *hr_dev, u32 idx)
6326 {
6327 struct hns_roce_cmq_desc desc;
6328 struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;
6329 u32 addr_upper;
6330 u32 addr_low;
6331 int ret;
6332
6333 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT, true);
6334 hr_reg_write(req, CFG_GMV_BT_IDX, idx);
6335
6336 ret = hns_roce_cmq_send(hr_dev, &desc, 1);
6337 if (ret) {
6338 dev_err(hr_dev->dev,
6339 "failed to execute cmd to read gmv, ret = %d.\n", ret);
6340 return ret;
6341 }
6342
6343 addr_low = hr_reg_read(req, CFG_GMV_BT_BA_L);
6344 addr_upper = hr_reg_read(req, CFG_GMV_BT_BA_H);
6345
6346 hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT, false);
6347 hr_reg_write(req, CFG_GMV_BT_BA_L, addr_low);
6348 hr_reg_write(req, CFG_GMV_BT_BA_H, addr_upper);
6349 hr_reg_write(req, CFG_GMV_BT_IDX, idx);
6350
6351 return hns_roce_cmq_send(hr_dev, &desc, 1);
6352 }
6353
fmea_get_ram_res_addr(u32 res_type,__le64 * data)6354 static u64 fmea_get_ram_res_addr(u32 res_type, __le64 *data)
6355 {
6356 if (res_type == ECC_RESOURCE_QPC_TIMER ||
6357 res_type == ECC_RESOURCE_CQC_TIMER ||
6358 res_type == ECC_RESOURCE_SCCC)
6359 return le64_to_cpu(*data);
6360
6361 return le64_to_cpu(*data) << HNS_HW_PAGE_SHIFT;
6362 }
6363
fmea_recover_others(struct hns_roce_dev * hr_dev,u32 res_type,u32 index)6364 static int fmea_recover_others(struct hns_roce_dev *hr_dev, u32 res_type,
6365 u32 index)
6366 {
6367 u8 write_bt0_op = fmea_ram_res[res_type].write_bt0_op;
6368 u8 read_bt0_op = fmea_ram_res[res_type].read_bt0_op;
6369 struct hns_roce_cmd_mailbox *mailbox;
6370 u64 addr;
6371 int ret;
6372
6373 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
6374 if (IS_ERR(mailbox))
6375 return PTR_ERR(mailbox);
6376
6377 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, read_bt0_op, index);
6378 if (ret) {
6379 dev_err(hr_dev->dev,
6380 "failed to execute cmd to read fmea ram, ret = %d.\n",
6381 ret);
6382 goto out;
6383 }
6384
6385 addr = fmea_get_ram_res_addr(res_type, mailbox->buf);
6386
6387 ret = hns_roce_cmd_mbox(hr_dev, addr, 0, write_bt0_op, index);
6388 if (ret)
6389 dev_err(hr_dev->dev,
6390 "failed to execute cmd to write fmea ram, ret = %d.\n",
6391 ret);
6392
6393 out:
6394 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
6395 return ret;
6396 }
6397
fmea_ram_ecc_recover(struct hns_roce_dev * hr_dev,struct fmea_ram_ecc * ecc_info)6398 static void fmea_ram_ecc_recover(struct hns_roce_dev *hr_dev,
6399 struct fmea_ram_ecc *ecc_info)
6400 {
6401 u32 res_type = ecc_info->res_type;
6402 u32 index = ecc_info->index;
6403 int ret;
6404
6405 BUILD_BUG_ON(ARRAY_SIZE(fmea_ram_res) != ECC_RESOURCE_COUNT);
6406
6407 if (res_type >= ECC_RESOURCE_COUNT) {
6408 dev_err(hr_dev->dev, "unsupported fmea ram ecc type %u.\n",
6409 res_type);
6410 return;
6411 }
6412
6413 if (res_type == ECC_RESOURCE_GMV)
6414 ret = fmea_recover_gmv(hr_dev, index);
6415 else
6416 ret = fmea_recover_others(hr_dev, res_type, index);
6417 if (ret)
6418 dev_err(hr_dev->dev,
6419 "failed to recover %s, index = %u, ret = %d.\n",
6420 fmea_ram_res[res_type].name, index, ret);
6421 }
6422
fmea_ram_ecc_work(struct work_struct * ecc_work)6423 static void fmea_ram_ecc_work(struct work_struct *ecc_work)
6424 {
6425 struct hns_roce_dev *hr_dev =
6426 container_of(ecc_work, struct hns_roce_dev, ecc_work);
6427 struct fmea_ram_ecc ecc_info = {};
6428
6429 if (fmea_ram_ecc_query(hr_dev, &ecc_info)) {
6430 dev_err(hr_dev->dev, "failed to query fmea ram ecc.\n");
6431 return;
6432 }
6433
6434 if (!ecc_info.is_ecc_err) {
6435 dev_err(hr_dev->dev, "there is no fmea ram ecc err found.\n");
6436 return;
6437 }
6438
6439 fmea_ram_ecc_recover(hr_dev, &ecc_info);
6440 }
6441
hns_roce_v2_msix_interrupt_abn(int irq,void * dev_id)6442 static irqreturn_t hns_roce_v2_msix_interrupt_abn(int irq, void *dev_id)
6443 {
6444 struct hns_roce_dev *hr_dev = dev_id;
6445 irqreturn_t int_work = IRQ_NONE;
6446 u32 int_st;
6447
6448 int_st = roce_read(hr_dev, ROCEE_VF_ABN_INT_ST_REG);
6449
6450 if (int_st) {
6451 int_work = abnormal_interrupt_basic(hr_dev, int_st);
6452 } else if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) {
6453 queue_work(hr_dev->irq_workq, &hr_dev->ecc_work);
6454 int_work = IRQ_HANDLED;
6455 } else {
6456 dev_err(hr_dev->dev, "there is no abnormal irq found.\n");
6457 }
6458
6459 return IRQ_RETVAL(int_work);
6460 }
6461
hns_roce_v2_int_mask_enable(struct hns_roce_dev * hr_dev,int eq_num,u32 enable_flag)6462 static void hns_roce_v2_int_mask_enable(struct hns_roce_dev *hr_dev,
6463 int eq_num, u32 enable_flag)
6464 {
6465 int i;
6466
6467 for (i = 0; i < eq_num; i++)
6468 roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG +
6469 i * EQ_REG_OFFSET, enable_flag);
6470
6471 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, enable_flag);
6472 roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG, enable_flag);
6473 }
6474
free_eq_buf(struct hns_roce_dev * hr_dev,struct hns_roce_eq * eq)6475 static void free_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
6476 {
6477 hns_roce_mtr_destroy(hr_dev, &eq->mtr);
6478 }
6479
hns_roce_v2_destroy_eqc(struct hns_roce_dev * hr_dev,struct hns_roce_eq * eq)6480 static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev,
6481 struct hns_roce_eq *eq)
6482 {
6483 struct device *dev = hr_dev->dev;
6484 int eqn = eq->eqn;
6485 int ret;
6486 u8 cmd;
6487
6488 if (eqn < hr_dev->caps.num_comp_vectors)
6489 cmd = HNS_ROCE_CMD_DESTROY_CEQC;
6490 else
6491 cmd = HNS_ROCE_CMD_DESTROY_AEQC;
6492
6493 ret = hns_roce_destroy_hw_ctx(hr_dev, cmd, eqn & HNS_ROCE_V2_EQN_M);
6494 if (ret)
6495 dev_err(dev, "[mailbox cmd] destroy eqc(%d) failed.\n", eqn);
6496
6497 free_eq_buf(hr_dev, eq);
6498 }
6499
init_eq_config(struct hns_roce_dev * hr_dev,struct hns_roce_eq * eq)6500 static void init_eq_config(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
6501 {
6502 eq->db_reg = hr_dev->reg_base + ROCEE_VF_EQ_DB_CFG0_REG;
6503 eq->cons_index = 0;
6504 eq->over_ignore = HNS_ROCE_V2_EQ_OVER_IGNORE_0;
6505 eq->coalesce = HNS_ROCE_V2_EQ_COALESCE_0;
6506 eq->arm_st = HNS_ROCE_V2_EQ_ALWAYS_ARMED;
6507 eq->shift = ilog2((unsigned int)eq->entries);
6508 }
6509
config_eqc(struct hns_roce_dev * hr_dev,struct hns_roce_eq * eq,void * mb_buf)6510 static int config_eqc(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq,
6511 void *mb_buf)
6512 {
6513 u64 eqe_ba[MTT_MIN_COUNT] = { 0 };
6514 struct hns_roce_eq_context *eqc;
6515 u64 bt_ba = 0;
6516 int ret;
6517
6518 eqc = mb_buf;
6519 memset(eqc, 0, sizeof(struct hns_roce_eq_context));
6520
6521 init_eq_config(hr_dev, eq);
6522
6523 /* if not multi-hop, eqe buffer only use one trunk */
6524 ret = hns_roce_mtr_find(hr_dev, &eq->mtr, 0, eqe_ba,
6525 ARRAY_SIZE(eqe_ba));
6526 if (ret) {
6527 dev_err(hr_dev->dev, "failed to find EQE mtr, ret = %d\n", ret);
6528 return ret;
6529 }
6530
6531 bt_ba = hns_roce_get_mtr_ba(&eq->mtr);
6532
6533 hr_reg_write(eqc, EQC_EQ_ST, HNS_ROCE_V2_EQ_STATE_VALID);
6534 hr_reg_write(eqc, EQC_EQE_HOP_NUM, eq->hop_num);
6535 hr_reg_write(eqc, EQC_OVER_IGNORE, eq->over_ignore);
6536 hr_reg_write(eqc, EQC_COALESCE, eq->coalesce);
6537 hr_reg_write(eqc, EQC_ARM_ST, eq->arm_st);
6538 hr_reg_write(eqc, EQC_EQN, eq->eqn);
6539 hr_reg_write(eqc, EQC_EQE_CNT, HNS_ROCE_EQ_INIT_EQE_CNT);
6540 hr_reg_write(eqc, EQC_EQE_BA_PG_SZ,
6541 to_hr_hw_page_shift(eq->mtr.hem_cfg.ba_pg_shift));
6542 hr_reg_write(eqc, EQC_EQE_BUF_PG_SZ,
6543 to_hr_hw_page_shift(eq->mtr.hem_cfg.buf_pg_shift));
6544 hr_reg_write(eqc, EQC_EQ_PROD_INDX, HNS_ROCE_EQ_INIT_PROD_IDX);
6545 hr_reg_write(eqc, EQC_EQ_MAX_CNT, eq->eq_max_cnt);
6546
6547 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {
6548 if (eq->eq_period * HNS_ROCE_CLOCK_ADJUST > USHRT_MAX) {
6549 dev_info(hr_dev->dev, "eq_period(%u) reached the upper limit, adjusted to 65.\n",
6550 eq->eq_period);
6551 eq->eq_period = HNS_ROCE_MAX_EQ_PERIOD;
6552 }
6553 eq->eq_period *= HNS_ROCE_CLOCK_ADJUST;
6554 }
6555
6556 hr_reg_write(eqc, EQC_EQ_PERIOD, eq->eq_period);
6557 hr_reg_write(eqc, EQC_EQE_REPORT_TIMER, HNS_ROCE_EQ_INIT_REPORT_TIMER);
6558 hr_reg_write(eqc, EQC_EQE_BA_L, bt_ba >> 3);
6559 hr_reg_write(eqc, EQC_EQE_BA_H, bt_ba >> 35);
6560 hr_reg_write(eqc, EQC_SHIFT, eq->shift);
6561 hr_reg_write(eqc, EQC_MSI_INDX, HNS_ROCE_EQ_INIT_MSI_IDX);
6562 hr_reg_write(eqc, EQC_CUR_EQE_BA_L, eqe_ba[0] >> 12);
6563 hr_reg_write(eqc, EQC_CUR_EQE_BA_M, eqe_ba[0] >> 28);
6564 hr_reg_write(eqc, EQC_CUR_EQE_BA_H, eqe_ba[0] >> 60);
6565 hr_reg_write(eqc, EQC_EQ_CONS_INDX, HNS_ROCE_EQ_INIT_CONS_IDX);
6566 hr_reg_write(eqc, EQC_NEX_EQE_BA_L, eqe_ba[1] >> 12);
6567 hr_reg_write(eqc, EQC_NEX_EQE_BA_H, eqe_ba[1] >> 44);
6568 hr_reg_write(eqc, EQC_EQE_SIZE, eq->eqe_size == HNS_ROCE_V3_EQE_SIZE);
6569
6570 return 0;
6571 }
6572
alloc_eq_buf(struct hns_roce_dev * hr_dev,struct hns_roce_eq * eq)6573 static int alloc_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
6574 {
6575 struct hns_roce_buf_attr buf_attr = {};
6576 int err;
6577
6578 if (hr_dev->caps.eqe_hop_num == HNS_ROCE_HOP_NUM_0)
6579 eq->hop_num = 0;
6580 else
6581 eq->hop_num = hr_dev->caps.eqe_hop_num;
6582
6583 buf_attr.page_shift = hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT;
6584 buf_attr.region[0].size = eq->entries * eq->eqe_size;
6585 buf_attr.region[0].hopnum = eq->hop_num;
6586 buf_attr.region_count = 1;
6587
6588 err = hns_roce_mtr_create(hr_dev, &eq->mtr, &buf_attr,
6589 hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT, NULL,
6590 0);
6591 if (err)
6592 dev_err(hr_dev->dev, "failed to alloc EQE mtr, err %d\n", err);
6593
6594 return err;
6595 }
6596
hns_roce_v2_create_eq(struct hns_roce_dev * hr_dev,struct hns_roce_eq * eq,u8 eq_cmd)6597 static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev,
6598 struct hns_roce_eq *eq, u8 eq_cmd)
6599 {
6600 struct hns_roce_cmd_mailbox *mailbox;
6601 int ret;
6602
6603 /* Allocate mailbox memory */
6604 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
6605 if (IS_ERR(mailbox))
6606 return PTR_ERR(mailbox);
6607
6608 ret = alloc_eq_buf(hr_dev, eq);
6609 if (ret)
6610 goto free_cmd_mbox;
6611
6612 ret = config_eqc(hr_dev, eq, mailbox->buf);
6613 if (ret)
6614 goto err_cmd_mbox;
6615
6616 ret = hns_roce_create_hw_ctx(hr_dev, mailbox, eq_cmd, eq->eqn);
6617 if (ret) {
6618 dev_err(hr_dev->dev, "[mailbox cmd] create eqc failed.\n");
6619 goto err_cmd_mbox;
6620 }
6621
6622 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
6623
6624 return 0;
6625
6626 err_cmd_mbox:
6627 free_eq_buf(hr_dev, eq);
6628
6629 free_cmd_mbox:
6630 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
6631
6632 return ret;
6633 }
6634
hns_roce_ceq_work(struct work_struct * work)6635 static void hns_roce_ceq_work(struct work_struct *work)
6636 {
6637 struct hns_roce_eq *eq = from_work(eq, work, work);
6638 struct hns_roce_ceqe *ceqe = next_ceqe_sw_v2(eq);
6639 struct hns_roce_dev *hr_dev = eq->hr_dev;
6640 int ceqe_num = 0;
6641 u32 cqn;
6642
6643 while (ceqe && ceqe_num < hr_dev->caps.ceqe_depth) {
6644 /* Make sure we read CEQ entry after we have checked the
6645 * ownership bit
6646 */
6647 dma_rmb();
6648
6649 cqn = hr_reg_read(ceqe, CEQE_CQN);
6650
6651 hns_roce_cq_completion(hr_dev, cqn);
6652
6653 ++eq->cons_index;
6654 ++ceqe_num;
6655 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_CEQE_CNT]);
6656
6657 ceqe = next_ceqe_sw_v2(eq);
6658 }
6659
6660 update_eq_db(eq);
6661 }
6662
__hns_roce_request_irq(struct hns_roce_dev * hr_dev,int irq_num,int comp_num,int aeq_num,int other_num)6663 static int __hns_roce_request_irq(struct hns_roce_dev *hr_dev, int irq_num,
6664 int comp_num, int aeq_num, int other_num)
6665 {
6666 struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
6667 int i, j;
6668 int ret;
6669
6670 for (i = 0; i < irq_num; i++) {
6671 hr_dev->irq_names[i] = kzalloc(HNS_ROCE_INT_NAME_LEN,
6672 GFP_KERNEL);
6673 if (!hr_dev->irq_names[i]) {
6674 ret = -ENOMEM;
6675 goto err_kzalloc_failed;
6676 }
6677 }
6678
6679 /* irq contains: abnormal + AEQ + CEQ */
6680 for (j = 0; j < other_num; j++)
6681 snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN,
6682 "hns-%s-abn-%d", pci_name(hr_dev->pci_dev), j);
6683
6684 for (j = other_num; j < (other_num + aeq_num); j++)
6685 snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN,
6686 "hns-%s-aeq-%d", pci_name(hr_dev->pci_dev), j - other_num);
6687
6688 for (j = (other_num + aeq_num); j < irq_num; j++)
6689 snprintf((char *)hr_dev->irq_names[j], HNS_ROCE_INT_NAME_LEN,
6690 "hns-%s-ceq-%d", pci_name(hr_dev->pci_dev),
6691 j - other_num - aeq_num);
6692
6693 for (j = 0; j < irq_num; j++) {
6694 if (j < other_num) {
6695 ret = request_irq(hr_dev->irq[j],
6696 hns_roce_v2_msix_interrupt_abn,
6697 0, hr_dev->irq_names[j], hr_dev);
6698 } else if (j < (other_num + comp_num)) {
6699 INIT_WORK(&eq_table->eq[j - other_num].work,
6700 hns_roce_ceq_work);
6701 ret = request_irq(eq_table->eq[j - other_num].irq,
6702 hns_roce_v2_msix_interrupt_eq,
6703 0, hr_dev->irq_names[j + aeq_num],
6704 &eq_table->eq[j - other_num]);
6705 } else {
6706 ret = request_irq(eq_table->eq[j - other_num].irq,
6707 hns_roce_v2_msix_interrupt_eq,
6708 0, hr_dev->irq_names[j - comp_num],
6709 &eq_table->eq[j - other_num]);
6710 }
6711
6712 if (ret) {
6713 dev_err(hr_dev->dev, "request irq error!\n");
6714 goto err_request_failed;
6715 }
6716 }
6717
6718 return 0;
6719
6720 err_request_failed:
6721 for (j -= 1; j >= 0; j--) {
6722 if (j < other_num) {
6723 free_irq(hr_dev->irq[j], hr_dev);
6724 continue;
6725 }
6726 free_irq(eq_table->eq[j - other_num].irq,
6727 &eq_table->eq[j - other_num]);
6728 if (j < other_num + comp_num)
6729 cancel_work_sync(&eq_table->eq[j - other_num].work);
6730 }
6731
6732 err_kzalloc_failed:
6733 for (i -= 1; i >= 0; i--)
6734 kfree(hr_dev->irq_names[i]);
6735
6736 return ret;
6737 }
6738
__hns_roce_free_irq(struct hns_roce_dev * hr_dev)6739 static void __hns_roce_free_irq(struct hns_roce_dev *hr_dev)
6740 {
6741 int irq_num;
6742 int eq_num;
6743 int i;
6744
6745 eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
6746 irq_num = eq_num + hr_dev->caps.num_other_vectors;
6747
6748 for (i = 0; i < hr_dev->caps.num_other_vectors; i++)
6749 free_irq(hr_dev->irq[i], hr_dev);
6750
6751 for (i = 0; i < eq_num; i++) {
6752 free_irq(hr_dev->eq_table.eq[i].irq, &hr_dev->eq_table.eq[i]);
6753 if (i < hr_dev->caps.num_comp_vectors)
6754 cancel_work_sync(&hr_dev->eq_table.eq[i].work);
6755 }
6756
6757 for (i = 0; i < irq_num; i++)
6758 kfree(hr_dev->irq_names[i]);
6759 }
6760
hns_roce_v2_init_eq_table(struct hns_roce_dev * hr_dev)6761 static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev)
6762 {
6763 struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
6764 struct device *dev = hr_dev->dev;
6765 struct hns_roce_eq *eq;
6766 int other_num;
6767 int comp_num;
6768 int aeq_num;
6769 int irq_num;
6770 int eq_num;
6771 u8 eq_cmd;
6772 int ret;
6773 int i;
6774
6775 if (hr_dev->caps.aeqe_depth < HNS_AEQ_POLLING_BUDGET)
6776 return -EINVAL;
6777
6778 other_num = hr_dev->caps.num_other_vectors;
6779 comp_num = hr_dev->caps.num_comp_vectors;
6780 aeq_num = hr_dev->caps.num_aeq_vectors;
6781
6782 eq_num = comp_num + aeq_num;
6783 irq_num = eq_num + other_num;
6784
6785 eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL);
6786 if (!eq_table->eq)
6787 return -ENOMEM;
6788
6789 /* create eq */
6790 for (i = 0; i < eq_num; i++) {
6791 eq = &eq_table->eq[i];
6792 eq->hr_dev = hr_dev;
6793 eq->eqn = i;
6794 if (i < comp_num) {
6795 /* CEQ */
6796 eq_cmd = HNS_ROCE_CMD_CREATE_CEQC;
6797 eq->type_flag = HNS_ROCE_CEQ;
6798 eq->entries = hr_dev->caps.ceqe_depth;
6799 eq->eqe_size = hr_dev->caps.ceqe_size;
6800 eq->irq = hr_dev->irq[i + other_num + aeq_num];
6801 eq->eq_max_cnt = HNS_ROCE_CEQ_DEFAULT_BURST_NUM;
6802 eq->eq_period = HNS_ROCE_CEQ_DEFAULT_INTERVAL;
6803 } else {
6804 /* AEQ */
6805 eq_cmd = HNS_ROCE_CMD_CREATE_AEQC;
6806 eq->type_flag = HNS_ROCE_AEQ;
6807 eq->entries = hr_dev->caps.aeqe_depth;
6808 eq->eqe_size = hr_dev->caps.aeqe_size;
6809 eq->irq = hr_dev->irq[i - comp_num + other_num];
6810 eq->eq_max_cnt = HNS_ROCE_AEQ_DEFAULT_BURST_NUM;
6811 eq->eq_period = HNS_ROCE_AEQ_DEFAULT_INTERVAL;
6812 }
6813
6814 ret = hns_roce_v2_create_eq(hr_dev, eq, eq_cmd);
6815 if (ret) {
6816 dev_err(dev, "failed to create eq.\n");
6817 goto err_create_eq_fail;
6818 }
6819 }
6820
6821 INIT_WORK(&hr_dev->ecc_work, fmea_ram_ecc_work);
6822
6823 hr_dev->irq_workq = alloc_ordered_workqueue("hns_roce_irq_workq", 0);
6824 if (!hr_dev->irq_workq) {
6825 dev_err(dev, "failed to create irq workqueue.\n");
6826 ret = -ENOMEM;
6827 goto err_create_eq_fail;
6828 }
6829
6830 ret = __hns_roce_request_irq(hr_dev, irq_num, comp_num, aeq_num,
6831 other_num);
6832 if (ret) {
6833 dev_err(dev, "failed to request irq.\n");
6834 goto err_request_irq_fail;
6835 }
6836
6837 /* enable irq */
6838 hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_ENABLE);
6839
6840 return 0;
6841
6842 err_request_irq_fail:
6843 destroy_workqueue(hr_dev->irq_workq);
6844
6845 err_create_eq_fail:
6846 for (i -= 1; i >= 0; i--)
6847 hns_roce_v2_destroy_eqc(hr_dev, &eq_table->eq[i]);
6848 kfree(eq_table->eq);
6849
6850 return ret;
6851 }
6852
hns_roce_v2_cleanup_eq_table(struct hns_roce_dev * hr_dev)6853 static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev)
6854 {
6855 struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
6856 int eq_num;
6857 int i;
6858
6859 eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
6860
6861 /* Disable irq */
6862 hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE);
6863
6864 __hns_roce_free_irq(hr_dev);
6865 destroy_workqueue(hr_dev->irq_workq);
6866
6867 for (i = 0; i < eq_num; i++)
6868 hns_roce_v2_destroy_eqc(hr_dev, &eq_table->eq[i]);
6869
6870 kfree(eq_table->eq);
6871 }
6872
6873 static const struct ib_device_ops hns_roce_v2_dev_ops = {
6874 .destroy_qp = hns_roce_v2_destroy_qp,
6875 .modify_cq = hns_roce_v2_modify_cq,
6876 .poll_cq = hns_roce_v2_poll_cq,
6877 .post_recv = hns_roce_v2_post_recv,
6878 .post_send = hns_roce_v2_post_send,
6879 .query_qp = hns_roce_v2_query_qp,
6880 .req_notify_cq = hns_roce_v2_req_notify_cq,
6881 };
6882
6883 static const struct ib_device_ops hns_roce_v2_dev_srq_ops = {
6884 .modify_srq = hns_roce_v2_modify_srq,
6885 .post_srq_recv = hns_roce_v2_post_srq_recv,
6886 .query_srq = hns_roce_v2_query_srq,
6887 };
6888
6889 static const struct hns_roce_hw hns_roce_hw_v2 = {
6890 .cmq_init = hns_roce_v2_cmq_init,
6891 .cmq_exit = hns_roce_v2_cmq_exit,
6892 .hw_profile = hns_roce_v2_profile,
6893 .hw_init = hns_roce_v2_init,
6894 .hw_exit = hns_roce_v2_exit,
6895 .post_mbox = v2_post_mbox,
6896 .poll_mbox_done = v2_poll_mbox_done,
6897 .chk_mbox_avail = v2_chk_mbox_is_avail,
6898 .set_gid = hns_roce_v2_set_gid,
6899 .set_mac = hns_roce_v2_set_mac,
6900 .write_mtpt = hns_roce_v2_write_mtpt,
6901 .rereg_write_mtpt = hns_roce_v2_rereg_write_mtpt,
6902 .frmr_write_mtpt = hns_roce_v2_frmr_write_mtpt,
6903 .mw_write_mtpt = hns_roce_v2_mw_write_mtpt,
6904 .write_cqc = hns_roce_v2_write_cqc,
6905 .set_hem = hns_roce_v2_set_hem,
6906 .clear_hem = hns_roce_v2_clear_hem,
6907 .modify_qp = hns_roce_v2_modify_qp,
6908 .dereg_mr = hns_roce_v2_dereg_mr,
6909 .qp_flow_control_init = hns_roce_v2_qp_flow_control_init,
6910 .init_eq = hns_roce_v2_init_eq_table,
6911 .cleanup_eq = hns_roce_v2_cleanup_eq_table,
6912 .write_srqc = hns_roce_v2_write_srqc,
6913 .query_cqc = hns_roce_v2_query_cqc,
6914 .query_qpc = hns_roce_v2_query_qpc,
6915 .query_mpt = hns_roce_v2_query_mpt,
6916 .query_srqc = hns_roce_v2_query_srqc,
6917 .query_sccc = hns_roce_v2_query_sccc,
6918 .query_hw_counter = hns_roce_hw_v2_query_counter,
6919 .get_dscp = hns_roce_hw_v2_get_dscp,
6920 .hns_roce_dev_ops = &hns_roce_v2_dev_ops,
6921 .hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops,
6922 };
6923
6924 static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = {
6925 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 0},
6926 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 0},
6927 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0},
6928 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0},
6929 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0},
6930 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA), 0},
6931 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_RDMA_DCB_PFC_VF),
6932 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
6933 /* required last entry */
6934 {0, }
6935 };
6936
6937 MODULE_DEVICE_TABLE(pci, hns_roce_hw_v2_pci_tbl);
6938
hns_roce_hw_v2_get_cfg(struct hns_roce_dev * hr_dev,struct hnae3_handle * handle)6939 static void hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,
6940 struct hnae3_handle *handle)
6941 {
6942 struct hns_roce_v2_priv *priv = hr_dev->priv;
6943 const struct pci_device_id *id;
6944 int i;
6945
6946 hr_dev->pci_dev = handle->pdev;
6947 id = pci_match_id(hns_roce_hw_v2_pci_tbl, hr_dev->pci_dev);
6948 hr_dev->is_vf = id->driver_data;
6949 hr_dev->dev = &handle->pdev->dev;
6950 hr_dev->hw = &hns_roce_hw_v2;
6951 hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG;
6952 hr_dev->odb_offset = hr_dev->sdb_offset;
6953
6954 /* Get info from NIC driver. */
6955 hr_dev->reg_base = handle->rinfo.roce_io_base;
6956 hr_dev->mem_base = handle->rinfo.roce_mem_base;
6957 hr_dev->caps.num_ports = 1;
6958 hr_dev->iboe.netdevs[0] = handle->rinfo.netdev;
6959 hr_dev->iboe.phy_port[0] = 0;
6960
6961 addrconf_addr_eui48((u8 *)&hr_dev->ib_dev.node_guid,
6962 hr_dev->iboe.netdevs[0]->dev_addr);
6963
6964 for (i = 0; i < handle->rinfo.num_vectors; i++)
6965 hr_dev->irq[i] = pci_irq_vector(handle->pdev,
6966 i + handle->rinfo.base_vector);
6967
6968 /* cmd issue mode: 0 is poll, 1 is event */
6969 hr_dev->cmd_mod = 1;
6970 hr_dev->loop_idc = 0;
6971
6972 hr_dev->reset_cnt = handle->ae_algo->ops->ae_dev_reset_cnt(handle);
6973 priv->handle = handle;
6974 }
6975
__hns_roce_hw_v2_init_instance(struct hnae3_handle * handle)6976 static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
6977 {
6978 struct hns_roce_dev *hr_dev;
6979 int ret;
6980
6981 hr_dev = ib_alloc_device(hns_roce_dev, ib_dev);
6982 if (!hr_dev)
6983 return -ENOMEM;
6984
6985 hr_dev->priv = kzalloc(sizeof(struct hns_roce_v2_priv), GFP_KERNEL);
6986 if (!hr_dev->priv) {
6987 ret = -ENOMEM;
6988 goto error_failed_kzalloc;
6989 }
6990
6991 hns_roce_hw_v2_get_cfg(hr_dev, handle);
6992
6993 ret = hns_roce_init(hr_dev);
6994 if (ret) {
6995 dev_err(hr_dev->dev, "RoCE Engine init failed!\n");
6996 goto error_failed_roce_init;
6997 }
6998
6999 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {
7000 ret = free_mr_init(hr_dev);
7001 if (ret) {
7002 dev_err(hr_dev->dev, "failed to init free mr!\n");
7003 goto error_failed_free_mr_init;
7004 }
7005 }
7006
7007 handle->priv = hr_dev;
7008
7009 return 0;
7010
7011 error_failed_free_mr_init:
7012 hns_roce_exit(hr_dev);
7013
7014 error_failed_roce_init:
7015 kfree(hr_dev->priv);
7016
7017 error_failed_kzalloc:
7018 ib_dealloc_device(&hr_dev->ib_dev);
7019
7020 return ret;
7021 }
7022
__hns_roce_hw_v2_uninit_instance(struct hnae3_handle * handle,bool reset)7023 static void __hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
7024 bool reset)
7025 {
7026 struct hns_roce_dev *hr_dev = handle->priv;
7027
7028 if (!hr_dev)
7029 return;
7030
7031 handle->priv = NULL;
7032
7033 hr_dev->state = HNS_ROCE_DEVICE_STATE_UNINIT;
7034 hns_roce_handle_device_err(hr_dev);
7035
7036 hns_roce_exit(hr_dev);
7037 kfree(hr_dev->priv);
7038 ib_dealloc_device(&hr_dev->ib_dev);
7039 }
7040
hns_roce_hw_v2_init_instance(struct hnae3_handle * handle)7041 static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
7042 {
7043 const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
7044 const struct pci_device_id *id;
7045 struct device *dev = &handle->pdev->dev;
7046 int ret;
7047
7048 handle->rinfo.instance_state = HNS_ROCE_STATE_INIT;
7049
7050 if (ops->ae_dev_resetting(handle) || ops->get_hw_reset_stat(handle)) {
7051 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
7052 goto reset_chk_err;
7053 }
7054
7055 id = pci_match_id(hns_roce_hw_v2_pci_tbl, handle->pdev);
7056 if (!id)
7057 return 0;
7058
7059 if (id->driver_data && handle->pdev->revision == PCI_REVISION_ID_HIP08)
7060 return 0;
7061
7062 ret = __hns_roce_hw_v2_init_instance(handle);
7063 if (ret) {
7064 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
7065 dev_err(dev, "RoCE instance init failed! ret = %d\n", ret);
7066 if (ops->ae_dev_resetting(handle) ||
7067 ops->get_hw_reset_stat(handle))
7068 goto reset_chk_err;
7069 else
7070 return ret;
7071 }
7072
7073 handle->rinfo.instance_state = HNS_ROCE_STATE_INITED;
7074
7075 return 0;
7076
7077 reset_chk_err:
7078 dev_err(dev, "Device is busy in resetting state.\n"
7079 "please retry later.\n");
7080
7081 return -EBUSY;
7082 }
7083
hns_roce_hw_v2_uninit_instance(struct hnae3_handle * handle,bool reset)7084 static void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
7085 bool reset)
7086 {
7087 if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED)
7088 return;
7089
7090 handle->rinfo.instance_state = HNS_ROCE_STATE_UNINIT;
7091
7092 __hns_roce_hw_v2_uninit_instance(handle, reset);
7093
7094 handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
7095 }
7096
hns_roce_hw_v2_reset_notify_down(struct hnae3_handle * handle)7097 static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle)
7098 {
7099 struct hns_roce_dev *hr_dev;
7100
7101 if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED) {
7102 set_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state);
7103 return 0;
7104 }
7105
7106 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_DOWN;
7107 clear_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state);
7108
7109 hr_dev = handle->priv;
7110 if (!hr_dev)
7111 return 0;
7112
7113 hr_dev->active = false;
7114 hr_dev->dis_db = true;
7115
7116 rdma_user_mmap_disassociate(&hr_dev->ib_dev);
7117
7118 hr_dev->state = HNS_ROCE_DEVICE_STATE_RST_DOWN;
7119
7120 return 0;
7121 }
7122
hns_roce_hw_v2_reset_notify_init(struct hnae3_handle * handle)7123 static int hns_roce_hw_v2_reset_notify_init(struct hnae3_handle *handle)
7124 {
7125 struct device *dev = &handle->pdev->dev;
7126 int ret;
7127
7128 if (test_and_clear_bit(HNS_ROCE_RST_DIRECT_RETURN,
7129 &handle->rinfo.state)) {
7130 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED;
7131 return 0;
7132 }
7133
7134 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INIT;
7135
7136 dev_info(&handle->pdev->dev, "In reset process RoCE client reinit.\n");
7137 ret = __hns_roce_hw_v2_init_instance(handle);
7138 if (ret) {
7139 /* when reset notify type is HNAE3_INIT_CLIENT In reset notify
7140 * callback function, RoCE Engine reinitialize. If RoCE reinit
7141 * failed, we should inform NIC driver.
7142 */
7143 handle->priv = NULL;
7144 dev_err(dev, "In reset process RoCE reinit failed %d.\n", ret);
7145 } else {
7146 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED;
7147 dev_info(dev, "reset done, RoCE client reinit finished.\n");
7148 }
7149
7150 return ret;
7151 }
7152
hns_roce_hw_v2_reset_notify_uninit(struct hnae3_handle * handle)7153 static int hns_roce_hw_v2_reset_notify_uninit(struct hnae3_handle *handle)
7154 {
7155 if (test_bit(HNS_ROCE_RST_DIRECT_RETURN, &handle->rinfo.state))
7156 return 0;
7157
7158 handle->rinfo.reset_state = HNS_ROCE_STATE_RST_UNINIT;
7159 dev_info(&handle->pdev->dev, "In reset process RoCE client uninit.\n");
7160 msleep(HNS_ROCE_V2_HW_RST_UNINT_DELAY);
7161 __hns_roce_hw_v2_uninit_instance(handle, false);
7162
7163 return 0;
7164 }
7165
hns_roce_hw_v2_reset_notify(struct hnae3_handle * handle,enum hnae3_reset_notify_type type)7166 static int hns_roce_hw_v2_reset_notify(struct hnae3_handle *handle,
7167 enum hnae3_reset_notify_type type)
7168 {
7169 int ret = 0;
7170
7171 switch (type) {
7172 case HNAE3_DOWN_CLIENT:
7173 ret = hns_roce_hw_v2_reset_notify_down(handle);
7174 break;
7175 case HNAE3_INIT_CLIENT:
7176 ret = hns_roce_hw_v2_reset_notify_init(handle);
7177 break;
7178 case HNAE3_UNINIT_CLIENT:
7179 ret = hns_roce_hw_v2_reset_notify_uninit(handle);
7180 break;
7181 default:
7182 break;
7183 }
7184
7185 return ret;
7186 }
7187
7188 static const struct hnae3_client_ops hns_roce_hw_v2_ops = {
7189 .init_instance = hns_roce_hw_v2_init_instance,
7190 .uninit_instance = hns_roce_hw_v2_uninit_instance,
7191 .reset_notify = hns_roce_hw_v2_reset_notify,
7192 };
7193
7194 static struct hnae3_client hns_roce_hw_v2_client = {
7195 .name = "hns_roce_hw_v2",
7196 .type = HNAE3_CLIENT_ROCE,
7197 .ops = &hns_roce_hw_v2_ops,
7198 };
7199
hns_roce_hw_v2_init(void)7200 static int __init hns_roce_hw_v2_init(void)
7201 {
7202 hns_roce_init_debugfs();
7203 return hnae3_register_client(&hns_roce_hw_v2_client);
7204 }
7205
hns_roce_hw_v2_exit(void)7206 static void __exit hns_roce_hw_v2_exit(void)
7207 {
7208 hnae3_unregister_client(&hns_roce_hw_v2_client);
7209 hns_roce_cleanup_debugfs();
7210 }
7211
7212 module_init(hns_roce_hw_v2_init);
7213 module_exit(hns_roce_hw_v2_exit);
7214
7215 MODULE_LICENSE("Dual BSD/GPL");
7216 MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>");
7217 MODULE_AUTHOR("Lijun Ou <oulijun@huawei.com>");
7218 MODULE_AUTHOR("Shaobo Xu <xushaobo2@huawei.com>");
7219 MODULE_DESCRIPTION("Hisilicon Hip08 Family RoCE Driver");
7220