1 /*-
2 * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
3 *
4 * Copyright (c) 2018 - 2026 Intel Corporation
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenFabrics.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #include "irdma_main.h"
36
37 #define IRDMA_ROCE_UDP_ENCAP_VALID_PORT_MIN (0xC000)
38
kc_rdma_flow_label_to_udp_sport(u32 fl)39 static u16 kc_rdma_flow_label_to_udp_sport(u32 fl) {
40 u32 fl_low = fl & 0x03FFF;
41 u32 fl_high = fl & 0xFC000;
42
43 fl_low ^= fl_high >> 14;
44
45 return (u16)(fl_low | IRDMA_ROCE_UDP_ENCAP_VALID_PORT_MIN);
46 }
47
48 #define IRDMA_GRH_FLOWLABEL_MASK (0x000FFFFF)
49
kc_rdma_calc_flow_label(u32 lqpn,u32 rqpn)50 static u32 kc_rdma_calc_flow_label(u32 lqpn, u32 rqpn) {
51 u64 fl = (u64)lqpn * rqpn;
52
53 fl ^= fl >> 20;
54 fl ^= fl >> 40;
55
56 return (u32)(fl & IRDMA_GRH_FLOWLABEL_MASK);
57 }
58
59 u16
kc_rdma_get_udp_sport(u32 fl,u32 lqpn,u32 rqpn)60 kc_rdma_get_udp_sport(u32 fl, u32 lqpn, u32 rqpn)
61 {
62 if (!fl)
63 fl = kc_rdma_calc_flow_label(lqpn, rqpn);
64 return kc_rdma_flow_label_to_udp_sport(fl);
65 }
66
67 void
irdma_get_dev_fw_str(struct ib_device * dev,char * str,size_t str_len)68 irdma_get_dev_fw_str(struct ib_device *dev,
69 char *str,
70 size_t str_len)
71 {
72 struct irdma_device *iwdev = to_iwdev(dev);
73
74 snprintf(str, str_len, "%u.%u",
75 irdma_fw_major_ver(&iwdev->rf->sc_dev),
76 irdma_fw_minor_ver(&iwdev->rf->sc_dev));
77 }
78
79 int
irdma_add_gid(const struct ib_gid_attr * attr,void ** context)80 irdma_add_gid(const struct ib_gid_attr *attr,
81 void **context)
82 {
83 return 0;
84 }
85
86 int
irdma_del_gid(const struct ib_gid_attr * attr,void ** context)87 irdma_del_gid(const struct ib_gid_attr *attr,
88 void **context)
89 {
90 return 0;
91 }
92
93 /**
94 * irdma_alloc_mr - register stag for fast memory registration
95 * @pd: ibpd pointer
96 * @mr_type: memory for stag registrion
97 * @max_num_sg: man number of pages
98 * @udata: user data
99 */
100 struct ib_mr *
irdma_alloc_mr(struct ib_pd * pd,enum ib_mr_type mr_type,u32 max_num_sg,struct ib_udata * udata)101 irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
102 u32 max_num_sg, struct ib_udata *udata)
103 {
104 struct irdma_device *iwdev = to_iwdev(pd->device);
105 struct irdma_pble_alloc *palloc;
106 struct irdma_pbl *iwpbl;
107 struct irdma_mr *iwmr;
108 int status;
109 u32 stag;
110 int err_code = -ENOMEM;
111
112 iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
113 if (!iwmr)
114 return ERR_PTR(-ENOMEM);
115
116 stag = irdma_create_stag(iwdev);
117 if (!stag) {
118 err_code = -ENOMEM;
119 goto err;
120 }
121
122 iwmr->stag = stag;
123 iwmr->ibmr.rkey = stag;
124 iwmr->ibmr.lkey = stag;
125 iwmr->ibmr.pd = pd;
126 iwmr->ibmr.device = pd->device;
127 iwpbl = &iwmr->iwpbl;
128 iwpbl->iwmr = iwmr;
129 iwmr->type = IRDMA_MEMREG_TYPE_MEM;
130 palloc = &iwpbl->pble_alloc;
131 iwmr->page_cnt = max_num_sg;
132 /* Assume system PAGE_SIZE as the sg page sizes are unknown. */
133 iwmr->len = max_num_sg * PAGE_SIZE;
134 status = irdma_get_pble(iwdev->rf->pble_rsrc, palloc, iwmr->page_cnt,
135 false);
136 if (status)
137 goto err_get_pble;
138
139 err_code = irdma_hw_alloc_stag(iwdev, iwmr);
140 if (err_code)
141 goto err_alloc_stag;
142
143 iwpbl->pbl_allocated = true;
144
145 return &iwmr->ibmr;
146 err_alloc_stag:
147 irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
148 err_get_pble:
149 irdma_free_stag(iwdev, stag);
150 err:
151 kfree(iwmr);
152
153 return ERR_PTR(err_code);
154 }
155
156 #define IRDMA_ALLOC_UCTX_MIN_REQ_LEN offsetofend(struct irdma_alloc_ucontext_req, rsvd8)
157 #define IRDMA_ALLOC_UCTX_MIN_RESP_LEN offsetofend(struct irdma_alloc_ucontext_resp, rsvd)
158 /**
159 * irdma_alloc_ucontext - Allocate the user context data structure
160 * @uctx: context
161 * @udata: user data
162 *
163 * This keeps track of all objects associated with a particular
164 * user-mode client.
165 */
166 int
irdma_alloc_ucontext(struct ib_ucontext * uctx,struct ib_udata * udata)167 irdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
168 {
169 struct ib_device *ibdev = uctx->device;
170 struct irdma_device *iwdev = to_iwdev(ibdev);
171 struct irdma_alloc_ucontext_req req = {0};
172 struct irdma_alloc_ucontext_resp uresp = {0};
173 struct irdma_ucontext *ucontext = to_ucontext(uctx);
174 struct irdma_uk_attrs *uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs;
175
176 if (udata->inlen < IRDMA_ALLOC_UCTX_MIN_REQ_LEN ||
177 udata->outlen < IRDMA_ALLOC_UCTX_MIN_RESP_LEN)
178 return -EINVAL;
179
180 if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen)))
181 return -EINVAL;
182
183 if (req.userspace_ver < 4 || req.userspace_ver > IRDMA_ABI_VER)
184 goto ver_error;
185
186 ucontext->iwdev = iwdev;
187 ucontext->abi_ver = req.userspace_ver;
188
189 if (req.comp_mask & IRDMA_ALLOC_UCTX_USE_RAW_ATTR)
190 ucontext->use_raw_attrs = true;
191
192 /* GEN_1 support for libi40iw */
193 if (udata->outlen == IRDMA_ALLOC_UCTX_MIN_RESP_LEN) {
194 if (uk_attrs->hw_rev != IRDMA_GEN_1)
195 return -EOPNOTSUPP;
196
197 ucontext->legacy_mode = true;
198 uresp.max_qps = iwdev->rf->max_qp;
199 uresp.max_pds = iwdev->rf->sc_dev.hw_attrs.max_hw_pds;
200 uresp.wq_size = iwdev->rf->sc_dev.hw_attrs.max_qp_wr * 2;
201 uresp.kernel_ver = req.userspace_ver;
202 if (ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen)))
203 return -EFAULT;
204 } else {
205 u64 bar_off;
206
207 uresp.kernel_ver = IRDMA_ABI_VER;
208 uresp.feature_flags = uk_attrs->feature_flags;
209 uresp.max_hw_wq_frags = uk_attrs->max_hw_wq_frags;
210 uresp.max_hw_read_sges = uk_attrs->max_hw_read_sges;
211 uresp.max_hw_inline = uk_attrs->max_hw_inline;
212 uresp.max_hw_rq_quanta = uk_attrs->max_hw_rq_quanta;
213 uresp.max_hw_wq_quanta = uk_attrs->max_hw_wq_quanta;
214 uresp.max_hw_sq_chunk = uk_attrs->max_hw_sq_chunk;
215 uresp.max_hw_cq_size = uk_attrs->max_hw_cq_size;
216 uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size;
217 uresp.hw_rev = uk_attrs->hw_rev;
218 uresp.comp_mask |= IRDMA_ALLOC_UCTX_USE_RAW_ATTR;
219 uresp.min_hw_wq_size = uk_attrs->min_hw_wq_size;
220 uresp.comp_mask |= IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE;
221
222 bar_off =
223 (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET];
224 ucontext->db_mmap_entry =
225 irdma_user_mmap_entry_insert(ucontext, bar_off,
226 IRDMA_MMAP_IO_NC,
227 &uresp.db_mmap_key);
228 if (!ucontext->db_mmap_entry) {
229 return -ENOMEM;
230 }
231
232 if (ib_copy_to_udata(udata, &uresp,
233 min(sizeof(uresp), udata->outlen))) {
234 rdma_user_mmap_entry_remove(ucontext->db_mmap_entry);
235 return -EFAULT;
236 }
237 }
238
239 INIT_LIST_HEAD(&ucontext->cq_reg_mem_list);
240 spin_lock_init(&ucontext->cq_reg_mem_list_lock);
241 INIT_LIST_HEAD(&ucontext->qp_reg_mem_list);
242 spin_lock_init(&ucontext->qp_reg_mem_list_lock);
243 INIT_LIST_HEAD(&ucontext->vma_list);
244 mutex_init(&ucontext->vma_list_mutex);
245
246 return 0;
247
248 ver_error:
249 irdma_dev_err(&iwdev->ibdev,
250 "Invalid userspace driver version detected. Detected version %d, should be %d\n",
251 req.userspace_ver, IRDMA_ABI_VER);
252 return -EINVAL;
253 }
254
255
256 /**
257 * irdma_dealloc_ucontext - deallocate the user context data structure
258 * @context: user context created during alloc
259 */
260 void
irdma_dealloc_ucontext(struct ib_ucontext * context)261 irdma_dealloc_ucontext(struct ib_ucontext *context)
262 {
263 struct irdma_ucontext *ucontext = to_ucontext(context);
264
265 rdma_user_mmap_entry_remove(ucontext->db_mmap_entry);
266
267 return;
268 }
269
270
271 #define IRDMA_ALLOC_PD_MIN_RESP_LEN offsetofend(struct irdma_alloc_pd_resp, rsvd)
272 /**
273 * irdma_alloc_pd - allocate protection domain
274 * @pd: protection domain
275 * @udata: user data
276 */
277 int
irdma_alloc_pd(struct ib_pd * pd,struct ib_udata * udata)278 irdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata)
279 {
280 struct irdma_pd *iwpd = to_iwpd(pd);
281 struct irdma_device *iwdev = to_iwdev(pd->device);
282 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;
283 struct irdma_pci_f *rf = iwdev->rf;
284 struct irdma_alloc_pd_resp uresp = {0};
285 struct irdma_sc_pd *sc_pd;
286 u32 pd_id = 0;
287 int err;
288
289 if (udata && udata->outlen < IRDMA_ALLOC_PD_MIN_RESP_LEN)
290 return -EINVAL;
291
292 err = irdma_alloc_rsrc(rf, rf->allocated_pds, rf->max_pd, &pd_id,
293 &rf->next_pd);
294 if (err)
295 return err;
296
297 sc_pd = &iwpd->sc_pd;
298 if (udata) {
299 struct irdma_ucontext *ucontext =
300 rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
301
302 irdma_sc_pd_init(dev, sc_pd, pd_id, ucontext->abi_ver);
303 uresp.pd_id = pd_id;
304 if (ib_copy_to_udata(udata, &uresp,
305 min(sizeof(uresp), udata->outlen))) {
306 err = -EFAULT;
307 goto error;
308 }
309 } else {
310 irdma_sc_pd_init(dev, sc_pd, pd_id, IRDMA_ABI_VER);
311 }
312
313 spin_lock_init(&iwpd->udqp_list_lock);
314 INIT_LIST_HEAD(&iwpd->udqp_list);
315
316 return 0;
317
318 error:
319
320 irdma_free_rsrc(rf, rf->allocated_pds, pd_id);
321
322 return err;
323 }
324
325
326 void
irdma_dealloc_pd(struct ib_pd * ibpd,struct ib_udata * udata)327 irdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
328 {
329 struct irdma_pd *iwpd = to_iwpd(ibpd);
330 struct irdma_device *iwdev = to_iwdev(ibpd->device);
331
332 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_pds, iwpd->sc_pd.pd_id);
333 }
334
335
336
337 /**
338 * irdma_find_qp_update_qs - update QS handle for UD QPs
339 * @rf: RDMA PCI function
340 * @pd: protection domain object
341 * @user_pri: selected user priority
342 */
343 static void
irdma_find_qp_update_qs(struct irdma_pci_f * rf,struct irdma_pd * pd,u8 user_pri)344 irdma_find_qp_update_qs(struct irdma_pci_f *rf,
345 struct irdma_pd *pd, u8 user_pri)
346 {
347 struct irdma_qp *iwqp;
348 struct list_head *tmp_node, *list_node;
349 struct irdma_udqs_work *work;
350 unsigned long flags;
351 bool qs_change;
352
353 spin_lock_irqsave(&pd->udqp_list_lock, flags);
354 list_for_each_safe(list_node, tmp_node, &pd->udqp_list) {
355 qs_change = true;
356 iwqp = list_entry(list_node, struct irdma_qp, ud_list_elem);
357 irdma_qp_add_ref(&iwqp->ibqp);
358 /* check if qs_handle needs to be changed */
359 if (iwqp->sc_qp.qs_handle == iwqp->sc_qp.vsi->qos[user_pri].qs_handle) {
360 if (iwqp->ctx_info.user_pri == user_pri) {
361 /* qs_handle and user_pri don't change */
362 irdma_qp_rem_ref(&iwqp->ibqp);
363 continue;
364 }
365 qs_change = false;
366 }
367 /* perform qp qos change */
368 work = kzalloc(sizeof(*work), GFP_ATOMIC);
369 if (!work) {
370 irdma_qp_rem_ref(&iwqp->ibqp);
371 spin_unlock_irqrestore(&pd->udqp_list_lock, flags);
372 return;
373 }
374 work->iwqp = iwqp;
375 work->user_prio = user_pri;
376 work->qs_change = qs_change;
377 INIT_WORK(&work->work, irdma_udqp_qs_worker);
378 if (qs_change)
379 irdma_cqp_qp_suspend_resume(&iwqp->sc_qp, IRDMA_OP_SUSPEND);
380 queue_work(rf->iwdev->cleanup_wq, &work->work);
381 }
382 spin_unlock_irqrestore(&pd->udqp_list_lock, flags);
383 }
384
385 static void
irdma_fill_ah_info(struct vnet * vnet,struct irdma_ah_info * ah_info,const struct ib_gid_attr * sgid_attr,union irdma_sockaddr * sgid_addr,union irdma_sockaddr * dgid_addr,u8 * dmac,u8 net_type)386 irdma_fill_ah_info(struct vnet *vnet, struct irdma_ah_info *ah_info,
387 const struct ib_gid_attr *sgid_attr,
388 union irdma_sockaddr *sgid_addr,
389 union irdma_sockaddr *dgid_addr,
390 u8 *dmac, u8 net_type)
391 {
392 if (net_type == RDMA_NETWORK_IPV4) {
393 ah_info->ipv4_valid = true;
394 ah_info->dest_ip_addr[0] =
395 ntohl(dgid_addr->saddr_in.sin_addr.s_addr);
396 ah_info->src_ip_addr[0] =
397 ntohl(sgid_addr->saddr_in.sin_addr.s_addr);
398 CURVNET_SET_QUIET(vnet);
399 ah_info->do_lpbk = irdma_ipv4_is_lpb(ah_info->src_ip_addr[0],
400 ah_info->dest_ip_addr[0]);
401 CURVNET_RESTORE();
402 if (ipv4_is_multicast(dgid_addr->saddr_in.sin_addr.s_addr)) {
403 irdma_mcast_mac_v4(ah_info->dest_ip_addr, dmac);
404 }
405 } else {
406 irdma_copy_ip_ntohl(ah_info->dest_ip_addr,
407 dgid_addr->saddr_in6.sin6_addr.__u6_addr.__u6_addr32);
408 irdma_copy_ip_ntohl(ah_info->src_ip_addr,
409 sgid_addr->saddr_in6.sin6_addr.__u6_addr.__u6_addr32);
410 ah_info->do_lpbk = irdma_ipv6_is_lpb(ah_info->src_ip_addr,
411 ah_info->dest_ip_addr);
412 if (rdma_is_multicast_addr(&dgid_addr->saddr_in6.sin6_addr)) {
413 irdma_mcast_mac_v6(ah_info->dest_ip_addr, dmac);
414 }
415 }
416 }
417
irdma_roce_get_vlan_prio(if_t ndev,u8 prio)418 static inline u8 irdma_roce_get_vlan_prio(if_t ndev, u8 prio)
419 {
420 return prio;
421 }
422
423 static int
irdma_create_ah_vlan_tag(struct irdma_device * iwdev,struct irdma_pd * pd,struct irdma_ah_info * ah_info,const struct ib_gid_attr * sgid_attr,u8 * dmac)424 irdma_create_ah_vlan_tag(struct irdma_device *iwdev,
425 struct irdma_pd *pd,
426 struct irdma_ah_info *ah_info,
427 const struct ib_gid_attr *sgid_attr,
428 u8 *dmac)
429 {
430 u16 vlan_prio;
431
432 if (sgid_attr->ndev && is_vlan_dev(sgid_attr->ndev))
433 ah_info->vlan_tag = vlan_dev_vlan_id(sgid_attr->ndev);
434 else
435 ah_info->vlan_tag = VLAN_N_VID;
436
437 ah_info->dst_arpindex = irdma_add_arp(iwdev->rf, ah_info->dest_ip_addr, dmac);
438
439 if (ah_info->dst_arpindex == -1)
440 return -EINVAL;
441
442 if (ah_info->vlan_tag >= VLAN_N_VID && iwdev->dcb_vlan_mode)
443 ah_info->vlan_tag = 0;
444
445 if (ah_info->vlan_tag < VLAN_N_VID) {
446 ah_info->insert_vlan_tag = true;
447 vlan_prio = (u16)irdma_roce_get_vlan_prio(sgid_attr->ndev,
448 rt_tos2priority(ah_info->tc_tos));
449 ah_info->vlan_tag |= vlan_prio << VLAN_PRIO_SHIFT;
450 irdma_find_qp_update_qs(iwdev->rf, pd, vlan_prio);
451 }
452 if (iwdev->roce_dcqcn_en) {
453 ah_info->tc_tos &= ~ECN_CODE_PT_MASK;
454 ah_info->tc_tos |= ECN_CODE_PT_VAL;
455 }
456
457 return 0;
458 }
459
460 static int
irdma_create_ah_wait(struct irdma_pci_f * rf,struct irdma_sc_ah * sc_ah,bool sleep)461 irdma_create_ah_wait(struct irdma_pci_f *rf,
462 struct irdma_sc_ah *sc_ah, bool sleep)
463 {
464 int ret;
465
466 if (!sleep) {
467 bool timeout = false;
468 u64 start = get_jiffies_64();
469 u64 completed_ops = atomic64_read(&rf->sc_dev.cqp->completed_ops);
470 struct irdma_cqp_request *cqp_request =
471 sc_ah->ah_info.cqp_request;
472 const u64 timeout_jiffies =
473 msecs_to_jiffies(rf->sc_dev.hw_attrs.max_cqp_compl_wait_time_ms *
474 CQP_TIMEOUT_THRESHOLD);
475
476 /*
477 * NOTE: irdma_check_cqp_progress is not used here because it relies on a notion of a cycle count, but
478 * we want to avoid unnecessary delays. We are in an atomic context here, so we might as well check in
479 * a tight loop.
480 */
481 while (!READ_ONCE(cqp_request->request_done)) {
482 u64 tmp;
483 u64 curr_jiffies;
484
485 irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq);
486
487 curr_jiffies = get_jiffies_64();
488 tmp = atomic64_read(&rf->sc_dev.cqp->completed_ops);
489 if (tmp != completed_ops) {
490 /* CQP is progressing. Reset timer. */
491 completed_ops = tmp;
492 start = curr_jiffies;
493 }
494
495 if ((curr_jiffies - start) > timeout_jiffies) {
496 timeout = true;
497 break;
498 }
499 }
500
501 if (!timeout && !cqp_request->compl_info.op_ret_val) {
502 irdma_put_cqp_request(&rf->cqp, cqp_request);
503 sc_ah->ah_info.ah_valid = true;
504 } else {
505 ret = timeout ? -ETIMEDOUT : -EINVAL;
506 irdma_dev_err(&rf->iwdev->ibdev, "CQP create AH error ret = %d opt_ret_val = %d",
507 ret, cqp_request->compl_info.op_ret_val);
508 irdma_put_cqp_request(&rf->cqp, cqp_request);
509 if (timeout && !rf->reset) {
510 rf->reset = true;
511 rf->gen_ops.request_reset(rf);
512 }
513 return ret;
514 }
515 }
516
517 return 0;
518 }
519
520 #define IRDMA_CREATE_AH_MIN_RESP_LEN offsetofend(struct irdma_create_ah_resp, rsvd)
521
522 static int
irdma_create_sleepable_ah(struct ib_ah * ib_ah,struct rdma_ah_attr * attr,u32 flags,struct ib_udata * udata)523 irdma_create_sleepable_ah(struct ib_ah *ib_ah,
524 struct rdma_ah_attr *attr, u32 flags,
525 struct ib_udata *udata)
526 {
527 struct irdma_pd *pd = to_iwpd(ib_ah->pd);
528 struct irdma_ah *ah = container_of(ib_ah, struct irdma_ah, ibah);
529 struct irdma_device *iwdev = to_iwdev(ib_ah->pd->device);
530 const union ib_gid *sgid;
531 const struct ib_gid_attr *sgid_attr;
532 struct irdma_pci_f *rf = iwdev->rf;
533 struct irdma_sc_ah *sc_ah;
534 u32 ah_id = 0;
535 struct irdma_ah_info *ah_info;
536 struct irdma_create_ah_resp uresp = {};
537 union irdma_sockaddr sgid_addr, dgid_addr;
538 int err;
539 u8 dmac[ETHER_ADDR_LEN];
540 bool sleep = (flags & RDMA_CREATE_AH_SLEEPABLE) != 0;
541
542 if (udata && udata->outlen < IRDMA_CREATE_AH_MIN_RESP_LEN)
543 return -EINVAL;
544
545 err = irdma_alloc_rsrc(rf, rf->allocated_ahs,
546 rf->max_ah, &ah_id, &rf->next_ah);
547
548 if (err)
549 return err;
550
551 ah->pd = pd;
552 sc_ah = &ah->sc_ah;
553 sc_ah->ah_info.ah_idx = ah_id;
554 sc_ah->ah_info.vsi = &iwdev->vsi;
555 irdma_sc_init_ah(&rf->sc_dev, sc_ah);
556 ah->sgid_index = attr->grh.sgid_index;
557 memcpy(&ah->dgid, &attr->grh.dgid, sizeof(ah->dgid));
558 sgid_attr = attr->grh.sgid_attr;
559 sgid = &sgid_attr->gid;
560 rdma_gid2ip((struct sockaddr *)&sgid_addr, sgid);
561 rdma_gid2ip((struct sockaddr *)&dgid_addr, &attr->grh.dgid);
562 ah->av.attrs = *attr;
563 ah->av.net_type = rdma_gid_attr_network_type(sgid_attr);
564
565 ah_info = &sc_ah->ah_info;
566 ah_info->ah_idx = ah_id;
567 ah_info->pd_idx = pd->sc_pd.pd_id;
568 ether_addr_copy(ah_info->mac_addr, if_getlladdr(iwdev->netdev));
569
570 if (attr->ah_flags & IB_AH_GRH) {
571 ah_info->flow_label = attr->grh.flow_label;
572 ah_info->hop_ttl = attr->grh.hop_limit;
573 ah_info->tc_tos = attr->grh.traffic_class;
574 }
575
576 ether_addr_copy(dmac, attr->roce.dmac);
577
578 irdma_fill_ah_info(if_getvnet(iwdev->netdev), ah_info, sgid_attr, &sgid_addr, &dgid_addr,
579 dmac, ah->av.net_type);
580
581 err = irdma_create_ah_vlan_tag(iwdev, pd, ah_info, sgid_attr, dmac);
582 if (err)
583 goto err_gid_l2;
584
585 err = irdma_ah_cqp_op(iwdev->rf, sc_ah, IRDMA_OP_AH_CREATE,
586 sleep, NULL, sc_ah);
587 if (err) {
588 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DEV, "CQP-OP Create AH fail");
589 goto err_gid_l2;
590 }
591
592 err = irdma_create_ah_wait(rf, sc_ah, sleep);
593 if (err)
594 goto err_gid_l2;
595
596 if (udata) {
597 uresp.ah_id = ah->sc_ah.ah_info.ah_idx;
598 err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen));
599 if (err) {
600 irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah,
601 IRDMA_OP_AH_DESTROY, false, NULL, ah);
602 goto err_gid_l2;
603 }
604 }
605
606 return 0;
607 err_gid_l2:
608 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, ah_id);
609
610 return err;
611 }
612
613 /**
614 * irdma_create_ah - create address handle
615 * @ib_ah: ptr to AH
616 * @attr: address handle attributes
617 * @flags: AH flags to wait
618 * @udata: user data
619 *
620 * returns 0 on success, error otherwise
621 */
622 int
irdma_create_ah(struct ib_ah * ib_ah,struct rdma_ah_attr * attr,u32 flags,struct ib_udata * udata)623 irdma_create_ah(struct ib_ah *ib_ah,
624 struct rdma_ah_attr *attr, u32 flags,
625 struct ib_udata *udata)
626 {
627 return irdma_create_sleepable_ah(ib_ah, attr, flags, udata);
628 }
629
630 void
irdma_ether_copy(u8 * dmac,struct rdma_ah_attr * attr)631 irdma_ether_copy(u8 *dmac, struct rdma_ah_attr *attr)
632 {
633 ether_addr_copy(dmac, attr->roce.dmac);
634 }
635
636 int
irdma_create_ah_stub(struct ib_ah * ib_ah,struct rdma_ah_attr * attr,u32 flags,struct ib_udata * udata)637 irdma_create_ah_stub(struct ib_ah *ib_ah,
638 struct rdma_ah_attr *attr, u32 flags,
639 struct ib_udata *udata)
640 {
641 return -ENOSYS;
642 }
643
644 void
irdma_destroy_ah_stub(struct ib_ah * ibah,u32 flags)645 irdma_destroy_ah_stub(struct ib_ah *ibah, u32 flags)
646 {
647 return;
648 }
649
650
651 /**
652 * irdma_free_qp_rsrc - free up memory resources for qp
653 * @iwqp: qp ptr (user or kernel)
654 */
655 void
irdma_free_qp_rsrc(struct irdma_qp * iwqp)656 irdma_free_qp_rsrc(struct irdma_qp *iwqp)
657 {
658 struct irdma_device *iwdev = iwqp->iwdev;
659 struct irdma_pci_f *rf = iwdev->rf;
660 u32 qp_num = iwqp->ibqp.qp_num;
661
662 irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp);
663 irdma_dealloc_push_page(rf, iwqp);
664 if (iwqp->sc_qp.vsi) {
665 irdma_qp_rem_qos(&iwqp->sc_qp);
666 iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi,
667 iwqp->sc_qp.user_pri);
668 }
669
670 if (qp_num > 2)
671 irdma_free_rsrc(rf, rf->allocated_qps, qp_num);
672 irdma_free_dma_mem(rf->sc_dev.hw, &iwqp->q2_ctx_mem);
673 irdma_free_dma_mem(rf->sc_dev.hw, &iwqp->kqp.dma_mem);
674 kfree(iwqp->kqp.sig_trk_mem);
675 iwqp->kqp.sig_trk_mem = NULL;
676 kfree(iwqp->kqp.sq_wrid_mem);
677 kfree(iwqp->kqp.rq_wrid_mem);
678 kfree(iwqp->sg_list);
679 kfree(iwqp);
680 }
681
682 /**
683 * irdma_create_qp - create qp
684 * @ibpd: ptr of pd
685 * @init_attr: attributes for qp
686 * @udata: user data for create qp
687 */
688 struct ib_qp *
irdma_create_qp(struct ib_pd * ibpd,struct ib_qp_init_attr * init_attr,struct ib_udata * udata)689 irdma_create_qp(struct ib_pd *ibpd,
690 struct ib_qp_init_attr *init_attr,
691 struct ib_udata *udata)
692 {
693 #define IRDMA_CREATE_QP_MIN_REQ_LEN offsetofend(struct irdma_create_qp_req, user_compl_ctx)
694 #define IRDMA_CREATE_QP_MIN_RESP_LEN offsetofend(struct irdma_create_qp_resp, rsvd)
695 struct irdma_pd *iwpd = to_iwpd(ibpd);
696 struct irdma_device *iwdev = to_iwdev(ibpd->device);
697 struct irdma_pci_f *rf = iwdev->rf;
698 struct irdma_qp *iwqp;
699 struct irdma_create_qp_resp uresp = {0};
700 u32 qp_num = 0;
701 int ret;
702 int err_code;
703 struct irdma_sc_qp *qp;
704 struct irdma_sc_dev *dev = &rf->sc_dev;
705 struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs;
706 struct irdma_qp_init_info init_info = {{0}};
707 struct irdma_qp_host_ctx_info *ctx_info;
708 u32 next_qp = 0;
709 unsigned long flags;
710
711 err_code = irdma_validate_qp_attrs(init_attr, iwdev);
712 if (err_code)
713 return ERR_PTR(err_code);
714
715 if (udata && (udata->inlen < IRDMA_CREATE_QP_MIN_REQ_LEN ||
716 udata->outlen < IRDMA_CREATE_QP_MIN_RESP_LEN))
717 return ERR_PTR(-EINVAL);
718
719 init_info.vsi = &iwdev->vsi;
720 init_info.qp_uk_init_info.uk_attrs = uk_attrs;
721 init_info.qp_uk_init_info.sq_size = init_attr->cap.max_send_wr;
722 init_info.qp_uk_init_info.rq_size = init_attr->cap.max_recv_wr;
723 init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr->cap.max_send_sge;
724 init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr->cap.max_recv_sge;
725 init_info.qp_uk_init_info.max_inline_data = init_attr->cap.max_inline_data;
726
727 iwqp = kzalloc(sizeof(*iwqp), GFP_KERNEL);
728 if (!iwqp)
729 return ERR_PTR(-ENOMEM);
730
731 iwqp->sg_list = kcalloc(uk_attrs->max_hw_wq_frags, sizeof(*iwqp->sg_list),
732 GFP_KERNEL);
733 if (!iwqp->sg_list) {
734 kfree(iwqp);
735 return ERR_PTR(-ENOMEM);
736 }
737
738 qp = &iwqp->sc_qp;
739 qp->qp_uk.back_qp = iwqp;
740 qp->qp_uk.lock = &iwqp->lock;
741 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX;
742
743 iwqp->iwdev = iwdev;
744 iwqp->q2_ctx_mem.size = IRDMA_Q2_BUF_SIZE + IRDMA_QP_CTX_SIZE;
745 iwqp->q2_ctx_mem.va = irdma_allocate_dma_mem(dev->hw, &iwqp->q2_ctx_mem,
746 iwqp->q2_ctx_mem.size,
747 256);
748 if (!iwqp->q2_ctx_mem.va) {
749 kfree(iwqp->sg_list);
750 kfree(iwqp);
751 return ERR_PTR(-ENOMEM);
752 }
753
754 init_info.q2 = iwqp->q2_ctx_mem.va;
755 init_info.q2_pa = iwqp->q2_ctx_mem.pa;
756 init_info.host_ctx = (__le64 *) (init_info.q2 + IRDMA_Q2_BUF_SIZE);
757 init_info.host_ctx_pa = init_info.q2_pa + IRDMA_Q2_BUF_SIZE;
758
759 if (init_attr->qp_type == IB_QPT_GSI)
760 qp_num = 1;
761 else if (dev->hw_attrs.uk_attrs.hw_rev <= IRDMA_GEN_2)
762 err_code = irdma_alloc_rsrc(rf, rf->allocated_qps, rf->max_qp,
763 &qp_num, &next_qp);
764 else
765 err_code = irdma_alloc_rsrc(rf, rf->allocated_qps, rf->max_qp,
766 &qp_num, &rf->next_qp);
767 if (err_code)
768 goto error;
769
770 iwqp->iwpd = iwpd;
771 iwqp->ibqp.qp_num = qp_num;
772 qp = &iwqp->sc_qp;
773 iwqp->iwscq = to_iwcq(init_attr->send_cq);
774 iwqp->iwrcq = to_iwcq(init_attr->recv_cq);
775 iwqp->host_ctx.va = init_info.host_ctx;
776 iwqp->host_ctx.pa = init_info.host_ctx_pa;
777 iwqp->host_ctx.size = IRDMA_QP_CTX_SIZE;
778
779 init_info.pd = &iwpd->sc_pd;
780 init_info.qp_uk_init_info.qp_id = qp_num;
781 if (!rdma_protocol_roce(&iwdev->ibdev, 1))
782 init_info.qp_uk_init_info.first_sq_wq = 1;
783 iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp;
784 init_waitqueue_head(&iwqp->waitq);
785 init_waitqueue_head(&iwqp->mod_qp_waitq);
786
787 spin_lock_init(&iwqp->dwork_flush_lock);
788
789 if (udata) {
790 INIT_DELAYED_WORK(&iwqp->dwork_flush, irdma_user_flush_worker);
791 init_info.qp_uk_init_info.abi_ver = iwpd->sc_pd.abi_ver;
792 err_code = irdma_setup_umode_qp(udata, iwdev, iwqp, &init_info, init_attr);
793 } else {
794 INIT_DELAYED_WORK(&iwqp->dwork_flush, irdma_kern_flush_worker);
795 init_info.qp_uk_init_info.abi_ver = IRDMA_ABI_VER;
796 err_code = irdma_setup_kmode_qp(iwdev, iwqp, &init_info, init_attr);
797 }
798
799 if (err_code) {
800 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "setup qp failed\n");
801 goto error;
802 }
803
804 if (rdma_protocol_roce(&iwdev->ibdev, 1)) {
805 if (init_attr->qp_type == IB_QPT_RC) {
806 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_RC;
807 init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM |
808 IRDMA_WRITE_WITH_IMM |
809 IRDMA_ROCE;
810 } else {
811 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_UD;
812 init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM |
813 IRDMA_ROCE;
814 }
815 } else {
816 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_IWARP;
817 init_info.qp_uk_init_info.qp_caps = IRDMA_WRITE_WITH_IMM;
818 }
819
820 ret = irdma_sc_qp_init(qp, &init_info);
821 if (ret) {
822 err_code = -EPROTO;
823 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "qp_init fail\n");
824 goto error;
825 }
826
827 ctx_info = &iwqp->ctx_info;
828 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;
829 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;
830
831 if (rdma_protocol_roce(&iwdev->ibdev, 1))
832 irdma_roce_fill_and_set_qpctx_info(iwqp, ctx_info);
833 else
834 irdma_iw_fill_and_set_qpctx_info(iwqp, ctx_info);
835
836 err_code = irdma_cqp_create_qp_cmd(iwqp);
837 if (err_code)
838 goto error;
839
840 atomic_set(&iwqp->refcnt, 1);
841 spin_lock_init(&iwqp->lock);
842 spin_lock_init(&iwqp->sc_qp.pfpdu.lock);
843 iwqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1 : 0;
844 rf->qp_table[qp_num] = iwqp;
845
846 if (rdma_protocol_roce(&iwdev->ibdev, 1)) {
847 if (dev->ws_add(&iwdev->vsi, 0)) {
848 irdma_cqp_qp_destroy_cmd(&rf->sc_dev, &iwqp->sc_qp);
849 err_code = -EINVAL;
850 goto error;
851 }
852
853 irdma_qp_add_qos(&iwqp->sc_qp);
854 spin_lock_irqsave(&iwpd->udqp_list_lock, flags);
855 if (iwqp->sc_qp.qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD)
856 list_add_tail(&iwqp->ud_list_elem, &iwpd->udqp_list);
857 spin_unlock_irqrestore(&iwpd->udqp_list_lock, flags);
858 }
859
860 if (udata) {
861 /* GEN_1 legacy support with libi40iw does not have expanded uresp struct */
862 if (udata->outlen == IRDMA_CREATE_QP_MIN_RESP_LEN) {
863 uresp.lsmm = 1;
864 uresp.push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1;
865 } else {
866 if (rdma_protocol_iwarp(&iwdev->ibdev, 1)) {
867 uresp.lsmm = 1;
868 if (qp->qp_uk.start_wqe_idx) {
869 uresp.comp_mask |= IRDMA_CREATE_QP_USE_START_WQE_IDX;
870 uresp.start_wqe_idx = qp->qp_uk.start_wqe_idx;
871 }
872 }
873 }
874 uresp.actual_sq_size = init_info.qp_uk_init_info.sq_size;
875 uresp.actual_rq_size = init_info.qp_uk_init_info.rq_size;
876 uresp.qp_id = qp_num;
877 uresp.qp_caps = qp->qp_uk.qp_caps;
878
879 err_code = ib_copy_to_udata(udata, &uresp,
880 min(sizeof(uresp), udata->outlen));
881 if (err_code) {
882 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "copy_to_udata failed\n");
883 irdma_destroy_qp(&iwqp->ibqp, udata);
884 return ERR_PTR(err_code);
885 }
886 }
887
888 init_completion(&iwqp->free_qp);
889 return &iwqp->ibqp;
890
891 error:
892 irdma_free_qp_rsrc(iwqp);
893
894 return ERR_PTR(err_code);
895 }
896
897 /**
898 * irdma_destroy_qp - destroy qp
899 * @ibqp: qp's ib pointer also to get to device's qp address
900 * @udata: user data
901 */
902 int
irdma_destroy_qp(struct ib_qp * ibqp,struct ib_udata * udata)903 irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
904 {
905 struct irdma_qp *iwqp = to_iwqp(ibqp);
906 struct irdma_device *iwdev = iwqp->iwdev;
907 unsigned long flags;
908
909 if (iwqp->sc_qp.qp_uk.destroy_pending)
910 goto free_rsrc;
911 iwqp->sc_qp.qp_uk.destroy_pending = true;
912
913 spin_lock_irqsave(&iwqp->iwpd->udqp_list_lock, flags);
914 if (iwqp->sc_qp.qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD)
915 list_del(&iwqp->ud_list_elem);
916 spin_unlock_irqrestore(&iwqp->iwpd->udqp_list_lock, flags);
917
918 if (iwqp->iwarp_state >= IRDMA_QP_STATE_IDLE)
919 irdma_modify_qp_to_err(&iwqp->sc_qp);
920
921 if (!iwqp->user_mode) {
922 if (iwqp->iwscq) {
923 irdma_clean_cqes(iwqp, iwqp->iwscq);
924 if (iwqp->iwrcq != iwqp->iwscq)
925 irdma_clean_cqes(iwqp, iwqp->iwrcq);
926 }
927 }
928 irdma_qp_rem_ref(&iwqp->ibqp);
929 wait_for_completion(&iwqp->free_qp);
930 irdma_free_lsmm_rsrc(iwqp);
931 if (!iwdev->rf->reset && irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp))
932 return (iwdev->rf->rdma_ver <= IRDMA_GEN_2 && !iwqp->user_mode) ? 0 : -ENOTRECOVERABLE;
933 free_rsrc:
934 irdma_remove_push_mmap_entries(iwqp);
935 irdma_free_qp_rsrc(iwqp);
936
937 return 0;
938 }
939
940 /**
941 * irdma_create_cq - create cq
942 * @ibcq: CQ allocated
943 * @attr: attributes for cq
944 * @udata: user data
945 */
946 int
irdma_create_cq(struct ib_cq * ibcq,const struct ib_cq_init_attr * attr,struct ib_udata * udata)947 irdma_create_cq(struct ib_cq *ibcq,
948 const struct ib_cq_init_attr *attr,
949 struct ib_udata *udata)
950 {
951 #define IRDMA_CREATE_CQ_MIN_REQ_LEN offsetofend(struct irdma_create_cq_req, user_cq_buf)
952 #define IRDMA_CREATE_CQ_MIN_RESP_LEN offsetofend(struct irdma_create_cq_resp, cq_size)
953 struct ib_device *ibdev = ibcq->device;
954 struct irdma_device *iwdev = to_iwdev(ibdev);
955 struct irdma_pci_f *rf = iwdev->rf;
956 struct irdma_cq *iwcq = to_iwcq(ibcq);
957 u32 cq_num = 0;
958 struct irdma_sc_cq *cq;
959 struct irdma_sc_dev *dev = &rf->sc_dev;
960 struct irdma_cq_init_info info = {0};
961 int status;
962 struct irdma_cqp_request *cqp_request;
963 struct cqp_cmds_info *cqp_info;
964 struct irdma_cq_uk_init_info *ukinfo = &info.cq_uk_init_info;
965 unsigned long flags;
966 int err_code;
967 int entries = attr->cqe;
968
969 err_code = cq_validate_flags(attr->flags, dev->hw_attrs.uk_attrs.hw_rev);
970 if (err_code)
971 return err_code;
972
973 if (udata && (udata->inlen < IRDMA_CREATE_CQ_MIN_REQ_LEN ||
974 udata->outlen < IRDMA_CREATE_CQ_MIN_RESP_LEN))
975 return -EINVAL;
976 err_code = irdma_alloc_rsrc(rf, rf->allocated_cqs, rf->max_cq, &cq_num,
977 &rf->next_cq);
978 if (err_code)
979 return err_code;
980 cq = &iwcq->sc_cq;
981 cq->back_cq = iwcq;
982 atomic_set(&iwcq->refcnt, 1);
983 spin_lock_init(&iwcq->lock);
984 INIT_LIST_HEAD(&iwcq->resize_list);
985 INIT_LIST_HEAD(&iwcq->cmpl_generated);
986 info.dev = dev;
987 ukinfo->cq_size = max_t(int, entries, 4);
988 ukinfo->cq_id = cq_num;
989 iwcq->cq_num = cq_num;
990 iwcq->ibcq.cqe = info.cq_uk_init_info.cq_size;
991 atomic_set(&iwcq->armed, 0);
992 if (attr->comp_vector < rf->ceqs_count)
993 info.ceq_id = attr->comp_vector;
994 info.ceq_id_valid = true;
995 info.ceqe_mask = 1;
996 info.type = IRDMA_CQ_TYPE_IWARP;
997 info.vsi = &iwdev->vsi;
998
999 if (udata) {
1000 struct irdma_ucontext *ucontext;
1001 struct irdma_create_cq_req req = {0};
1002 struct irdma_cq_mr *cqmr;
1003 struct irdma_pbl *iwpbl;
1004 struct irdma_pbl *iwpbl_shadow;
1005 struct irdma_cq_mr *cqmr_shadow;
1006
1007 iwcq->user_mode = true;
1008 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
1009
1010 if (ib_copy_from_udata(&req, udata,
1011 min(sizeof(req), udata->inlen))) {
1012 err_code = -EFAULT;
1013 goto cq_free_rsrc;
1014 }
1015
1016 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
1017 iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf,
1018 &ucontext->cq_reg_mem_list);
1019 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
1020 if (!iwpbl) {
1021 err_code = -EPROTO;
1022 goto cq_free_rsrc;
1023 }
1024 cqmr = &iwpbl->cq_mr;
1025
1026 if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags &
1027 IRDMA_FEATURE_CQ_RESIZE && !ucontext->legacy_mode) {
1028 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
1029 iwpbl_shadow = irdma_get_pbl((unsigned long)req.user_shadow_area,
1030 &ucontext->cq_reg_mem_list);
1031 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
1032
1033 if (!iwpbl_shadow) {
1034 err_code = -EPROTO;
1035 goto cq_free_rsrc;
1036 }
1037 cqmr_shadow = &iwpbl_shadow->cq_mr;
1038 info.shadow_area_pa = cqmr_shadow->cq_pbl.addr;
1039 cqmr->split = true;
1040 } else {
1041 info.shadow_area_pa = cqmr->shadow;
1042 }
1043 if (iwpbl->pbl_allocated) {
1044 info.virtual_map = true;
1045 info.pbl_chunk_size = 1;
1046 info.first_pm_pbl_idx = cqmr->cq_pbl.idx;
1047 } else {
1048 info.cq_base_pa = cqmr->cq_pbl.addr;
1049 }
1050 } else {
1051 /* Kmode allocations */
1052 int rsize;
1053
1054 if (entries < 1 || entries > rf->max_cqe) {
1055 err_code = -EINVAL;
1056 goto cq_free_rsrc;
1057 }
1058
1059 entries++;
1060 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_2)
1061 entries *= 2;
1062 ukinfo->cq_size = entries;
1063
1064 rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_cqe);
1065 iwcq->kmem.size = round_up(rsize, IRDMA_HW_PAGE_SIZE);
1066 iwcq->kmem.va = irdma_allocate_dma_mem(dev->hw, &iwcq->kmem,
1067 iwcq->kmem.size, IRDMA_HW_PAGE_SIZE);
1068 if (!iwcq->kmem.va) {
1069 err_code = -ENOMEM;
1070 goto cq_free_rsrc;
1071 }
1072
1073 iwcq->kmem_shadow.size = IRDMA_SHADOW_AREA_SIZE << 3;
1074 iwcq->kmem_shadow.va = irdma_allocate_dma_mem(dev->hw,
1075 &iwcq->kmem_shadow,
1076 iwcq->kmem_shadow.size,
1077 64);
1078
1079 if (!iwcq->kmem_shadow.va) {
1080 err_code = -ENOMEM;
1081 goto cq_kmem_free;
1082 }
1083 info.shadow_area_pa = iwcq->kmem_shadow.pa;
1084 ukinfo->shadow_area = iwcq->kmem_shadow.va;
1085 ukinfo->cq_base = iwcq->kmem.va;
1086 info.cq_base_pa = iwcq->kmem.pa;
1087 }
1088
1089 info.shadow_read_threshold = min(info.cq_uk_init_info.cq_size / 2,
1090 (u32)IRDMA_MAX_CQ_READ_THRESH);
1091 if (irdma_sc_cq_init(cq, &info)) {
1092 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "init cq fail\n");
1093 err_code = -EPROTO;
1094 goto cq_kmem_free;
1095 }
1096
1097 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1098 if (!cqp_request) {
1099 err_code = -ENOMEM;
1100 goto cq_kmem_free;
1101 }
1102 cqp_info = &cqp_request->info;
1103 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE;
1104 cqp_info->post_sq = 1;
1105 cqp_info->in.u.cq_create.cq = cq;
1106 cqp_info->in.u.cq_create.check_overflow = true;
1107 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request;
1108 cqp_info->create = true;
1109 status = irdma_handle_cqp_op(rf, cqp_request);
1110 irdma_put_cqp_request(&rf->cqp, cqp_request);
1111 if (status) {
1112 err_code = -ENOMEM;
1113 goto cq_kmem_free;
1114 }
1115
1116 if (udata) {
1117 struct irdma_create_cq_resp resp = {0};
1118
1119 resp.cq_id = info.cq_uk_init_info.cq_id;
1120 resp.cq_size = info.cq_uk_init_info.cq_size;
1121 if (ib_copy_to_udata(udata, &resp,
1122 min(sizeof(resp), udata->outlen))) {
1123 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "copy to user data\n");
1124 err_code = -EPROTO;
1125 goto cq_destroy;
1126 }
1127 }
1128
1129 WRITE_ONCE(rf->cq_table[cq_num], iwcq);
1130 init_completion(&iwcq->free_cq);
1131
1132 return 0;
1133 cq_destroy:
1134 irdma_cq_wq_destroy(rf, cq);
1135 cq_kmem_free:
1136 if (!iwcq->user_mode) {
1137 irdma_free_dma_mem(dev->hw, &iwcq->kmem);
1138 irdma_free_dma_mem(dev->hw, &iwcq->kmem_shadow);
1139 }
1140 cq_free_rsrc:
1141 irdma_free_rsrc(rf, rf->allocated_cqs, cq_num);
1142 return err_code;
1143 }
1144
1145 /**
1146 * irdma_copy_user_pgaddrs - copy user page address to pble's os locally
1147 * @iwmr: iwmr for IB's user page addresses
1148 * @pbl: ple pointer to save 1 level or 0 level pble
1149 * @level: indicated level 0, 1 or 2
1150 */
1151
1152 void
irdma_copy_user_pgaddrs(struct irdma_mr * iwmr,u64 * pbl,enum irdma_pble_level level)1153 irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl,
1154 enum irdma_pble_level level)
1155 {
1156 struct ib_umem *region = iwmr->region;
1157 struct irdma_pbl *iwpbl = &iwmr->iwpbl;
1158 int chunk_pages, entry, i;
1159 struct scatterlist *sg;
1160 u64 pg_addr = 0;
1161 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;
1162 struct irdma_pble_info *pinfo;
1163 u32 idx = 0;
1164 u32 pbl_cnt = 0;
1165
1166 pinfo = (level == PBLE_LEVEL_1) ? NULL : palloc->level2.leaf;
1167 for_each_sg(region->sg_head.sgl, sg, region->nmap, entry) {
1168 chunk_pages = DIV_ROUND_UP(sg_dma_len(sg), iwmr->page_size);
1169 if (iwmr->type == IRDMA_MEMREG_TYPE_QP && !iwpbl->qp_mr.sq_page)
1170 iwpbl->qp_mr.sq_page = sg_page(sg);
1171 for (i = 0; i < chunk_pages; i++) {
1172 pg_addr = sg_dma_address(sg) + (i * iwmr->page_size);
1173 if ((entry + i) == 0)
1174 *pbl = pg_addr & iwmr->page_msk;
1175 else if (!(pg_addr & ~iwmr->page_msk))
1176 *pbl = pg_addr;
1177 else
1178 continue;
1179 if (++pbl_cnt == palloc->total_cnt)
1180 break;
1181 pbl = irdma_next_pbl_addr(pbl, &pinfo, &idx);
1182 }
1183 }
1184 }
1185
1186 /**
1187 * irdma_destroy_ah - Destroy address handle
1188 * @ibah: pointer to address handle
1189 * @ah_flags: destroy flags
1190 */
1191
1192 void
irdma_destroy_ah(struct ib_ah * ibah,u32 ah_flags)1193 irdma_destroy_ah(struct ib_ah *ibah, u32 ah_flags)
1194 {
1195 struct irdma_device *iwdev = to_iwdev(ibah->device);
1196 struct irdma_ah *ah = to_iwah(ibah);
1197
1198 irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, IRDMA_OP_AH_DESTROY,
1199 false, NULL, ah);
1200
1201 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs,
1202 ah->sc_ah.ah_info.ah_idx);
1203 }
1204
1205
1206 int
irdma_dereg_mr(struct ib_mr * ib_mr,struct ib_udata * udata)1207 irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
1208 {
1209 struct irdma_mr *iwmr = to_iwmr(ib_mr);
1210 struct irdma_device *iwdev = to_iwdev(ib_mr->device);
1211 struct irdma_pbl *iwpbl = &iwmr->iwpbl;
1212 int ret;
1213
1214 if (iwmr->type != IRDMA_MEMREG_TYPE_MEM) {
1215 if (iwmr->region) {
1216 struct irdma_ucontext *ucontext;
1217
1218 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
1219
1220 irdma_del_memlist(iwmr, ucontext);
1221 }
1222 goto done;
1223 }
1224
1225 ret = irdma_hwdereg_mr(ib_mr);
1226 if (ret)
1227 return ret;
1228
1229 irdma_free_stag(iwdev, iwmr->stag);
1230 done:
1231 if (iwpbl->pbl_allocated)
1232 irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc);
1233
1234 if (iwmr->region)
1235 ib_umem_release(iwmr->region);
1236
1237 kfree(iwmr);
1238
1239 return 0;
1240 }
1241
1242 /**
1243 * irdma_reg_user_mr - Register a user memory region
1244 * @pd: ptr of pd
1245 * @start: virtual start address
1246 * @len: length of mr
1247 * @virt: virtual address
1248 * @access: access of mr
1249 * @udata: user data
1250 */
1251 struct ib_mr *
irdma_reg_user_mr(struct ib_pd * pd,u64 start,u64 len,u64 virt,int access,struct ib_udata * udata)1252 irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len,
1253 u64 virt, int access,
1254 struct ib_udata *udata)
1255 {
1256 #define IRDMA_MEM_REG_MIN_REQ_LEN offsetofend(struct irdma_mem_reg_req, sq_pages)
1257 struct irdma_device *iwdev = to_iwdev(pd->device);
1258 struct irdma_mem_reg_req req = {};
1259 struct ib_umem *region;
1260 struct irdma_mr *iwmr;
1261 int err;
1262
1263 if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)
1264 return ERR_PTR(-EINVAL);
1265
1266 if (udata->inlen < IRDMA_MEM_REG_MIN_REQ_LEN)
1267 return ERR_PTR(-EINVAL);
1268
1269 region = ib_umem_get(pd->uobject->context, start, len, access, 0);
1270
1271 if (IS_ERR(region)) {
1272 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
1273 "Failed to create ib_umem region err=%ld\n",
1274 PTR_ERR(region));
1275 return (struct ib_mr *)region;
1276 }
1277
1278 if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) {
1279 ib_umem_release(region);
1280 return ERR_PTR(-EFAULT);
1281 }
1282
1283 iwmr = irdma_alloc_iwmr(region, pd, virt, req.reg_type);
1284 if (IS_ERR(iwmr)) {
1285 ib_umem_release(region);
1286 return (struct ib_mr *)iwmr;
1287 }
1288
1289 switch (req.reg_type) {
1290 case IRDMA_MEMREG_TYPE_QP:
1291 err = irdma_reg_user_mr_type_qp(req, udata, iwmr);
1292 if (err)
1293 goto error;
1294
1295 break;
1296 case IRDMA_MEMREG_TYPE_CQ:
1297 err = irdma_reg_user_mr_type_cq(req, udata, iwmr);
1298 if (err)
1299 goto error;
1300
1301 break;
1302 case IRDMA_MEMREG_TYPE_MEM:
1303 err = irdma_reg_user_mr_type_mem(iwmr, access, true);
1304 if (err)
1305 goto error;
1306
1307 break;
1308 default:
1309 err = -EINVAL;
1310 goto error;
1311 }
1312
1313 return &iwmr->ibmr;
1314
1315 error:
1316 ib_umem_release(region);
1317 irdma_free_iwmr(iwmr);
1318
1319 return ERR_PTR(err);
1320 }
1321
1322 /*
1323 * irdma_rereg_user_mr - Re-Register a user memory region @ibmr: ib mem to access iwarp mr pointer @flags: bit mask to
1324 * indicate which of the attr's of MR modified @start: virtual start address @len: length of mr @virt: virtual address
1325 * @new access flags: bit mask of access flags @new_pd: ptr of pd @udata: user data
1326 */
1327 int
irdma_rereg_user_mr(struct ib_mr * ib_mr,int flags,u64 start,u64 len,u64 virt,int new_access,struct ib_pd * new_pd,struct ib_udata * udata)1328 irdma_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, u64 len,
1329 u64 virt, int new_access, struct ib_pd *new_pd,
1330 struct ib_udata *udata)
1331 {
1332 struct irdma_device *iwdev = to_iwdev(ib_mr->device);
1333 struct irdma_mr *iwmr = to_iwmr(ib_mr);
1334 struct irdma_pbl *iwpbl = &iwmr->iwpbl;
1335 int ret;
1336
1337 if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)
1338 return -EINVAL;
1339
1340 if (flags & ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS))
1341 return -EOPNOTSUPP;
1342
1343 ret = irdma_hwdereg_mr(ib_mr);
1344 if (ret)
1345 return ret;
1346
1347 if (flags & IB_MR_REREG_ACCESS)
1348 iwmr->access = new_access;
1349
1350 if (flags & IB_MR_REREG_PD) {
1351 iwmr->ibmr.pd = new_pd;
1352 iwmr->ibmr.device = new_pd->device;
1353 }
1354
1355 if (flags & IB_MR_REREG_TRANS) {
1356 if (iwpbl->pbl_allocated) {
1357 irdma_free_pble(iwdev->rf->pble_rsrc,
1358 &iwpbl->pble_alloc);
1359 iwpbl->pbl_allocated = false;
1360 }
1361 if (iwmr->region) {
1362 ib_umem_release(iwmr->region);
1363 iwmr->region = NULL;
1364 }
1365
1366 ib_mr = irdma_rereg_mr_trans(iwmr, start, len, virt, udata);
1367 if (IS_ERR(ib_mr))
1368 return PTR_ERR(ib_mr);
1369
1370 } else {
1371 ret = irdma_hwreg_mr(iwdev, iwmr, iwmr->access);
1372 if (ret)
1373 return ret;
1374 }
1375
1376 return 0;
1377 }
1378
1379 int
kc_irdma_set_roce_cm_info(struct irdma_qp * iwqp,struct ib_qp_attr * attr,u16 * vlan_id)1380 kc_irdma_set_roce_cm_info(struct irdma_qp *iwqp, struct ib_qp_attr *attr,
1381 u16 *vlan_id)
1382 {
1383 const struct ib_gid_attr *sgid_attr;
1384 struct irdma_av *av = &iwqp->roce_ah.av;
1385 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1386
1387 sgid_attr = grh->sgid_attr;
1388
1389 if (sgid_attr->ndev) {
1390 *vlan_id = rdma_vlan_dev_vlan_id(sgid_attr->ndev);
1391 ether_addr_copy(iwqp->ctx_info.roce_info->mac_addr, if_getlladdr(sgid_attr->ndev));
1392 }
1393
1394 av->net_type = rdma_gid_attr_network_type(sgid_attr);
1395 rdma_gid2ip((struct sockaddr *)&av->sgid_addr, &sgid_attr->gid);
1396 iwqp->sc_qp.user_pri = iwqp->ctx_info.user_pri;
1397
1398 return 0;
1399 }
1400
1401 /**
1402 * irdma_destroy_cq - destroy cq
1403 * @ib_cq: cq pointer
1404 * @udata: user data
1405 */
1406 void
irdma_destroy_cq(struct ib_cq * ib_cq,struct ib_udata * udata)1407 irdma_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
1408 {
1409 struct irdma_device *iwdev = to_iwdev(ib_cq->device);
1410 struct irdma_cq *iwcq = to_iwcq(ib_cq);
1411 struct irdma_sc_cq *cq = &iwcq->sc_cq;
1412 struct irdma_sc_dev *dev = cq->dev;
1413 struct irdma_sc_ceq *ceq = dev->ceq[cq->ceq_id];
1414 struct irdma_ceq *iwceq = container_of(ceq, struct irdma_ceq, sc_ceq);
1415 unsigned long flags;
1416
1417 spin_lock_irqsave(&iwcq->lock, flags);
1418 if (!list_empty(&iwcq->cmpl_generated))
1419 irdma_remove_cmpls_list(iwcq);
1420 if (!list_empty(&iwcq->resize_list))
1421 irdma_process_resize_list(iwcq, iwdev, NULL);
1422 spin_unlock_irqrestore(&iwcq->lock, flags);
1423
1424 irdma_cq_rem_ref(ib_cq);
1425 wait_for_completion(&iwcq->free_cq);
1426
1427 irdma_cq_wq_destroy(iwdev->rf, cq);
1428
1429 spin_lock_irqsave(&iwceq->ce_lock, flags);
1430 irdma_sc_cleanup_ceqes(cq, ceq);
1431 spin_unlock_irqrestore(&iwceq->ce_lock, flags);
1432 irdma_cq_free_rsrc(iwdev->rf, iwcq);
1433 }
1434
1435 /**
1436 * kc_set_loc_seq_num_mss - Set local seq number and mss
1437 * @cm_node: cm node info
1438 */
1439 void
kc_set_loc_seq_num_mss(struct irdma_cm_node * cm_node)1440 kc_set_loc_seq_num_mss(struct irdma_cm_node *cm_node)
1441 {
1442 struct timespec ts;
1443
1444 getnanotime(&ts);
1445 cm_node->tcp_cntxt.loc_seq_num = ts.tv_nsec;
1446 if (cm_node->iwdev->vsi.mtu > 1500 &&
1447 2 * cm_node->iwdev->vsi.mtu > cm_node->iwdev->rcv_wnd)
1448 cm_node->tcp_cntxt.mss = (cm_node->ipv4) ?
1449 (1500 - IRDMA_MTU_TO_MSS_IPV4) :
1450 (1500 - IRDMA_MTU_TO_MSS_IPV6);
1451 else
1452 cm_node->tcp_cntxt.mss = (cm_node->ipv4) ?
1453 (cm_node->iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV4) :
1454 (cm_node->iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV6);
1455 }
1456
1457 /**
1458 * irdma_disassociate_ucontext - Disassociate user context
1459 * @context: ib user context
1460 */
1461 void
irdma_disassociate_ucontext(struct ib_ucontext * context)1462 irdma_disassociate_ucontext(struct ib_ucontext *context)
1463 {
1464 }
1465
1466 struct ib_device *
ib_device_get_by_netdev(if_t netdev,int driver_id)1467 ib_device_get_by_netdev(if_t netdev, int driver_id)
1468 {
1469 struct irdma_device *iwdev;
1470 struct irdma_handler *hdl;
1471 unsigned long flags;
1472
1473 spin_lock_irqsave(&irdma_handler_lock, flags);
1474 list_for_each_entry(hdl, &irdma_handlers, list) {
1475 iwdev = hdl->iwdev;
1476 if (netdev == iwdev->netdev) {
1477 spin_unlock_irqrestore(&irdma_handler_lock,
1478 flags);
1479 return &iwdev->ibdev;
1480 }
1481 }
1482 spin_unlock_irqrestore(&irdma_handler_lock, flags);
1483
1484 return NULL;
1485 }
1486
1487 void
ib_unregister_device_put(struct ib_device * device)1488 ib_unregister_device_put(struct ib_device *device)
1489 {
1490 ib_unregister_device(device);
1491 }
1492
1493 /**
1494 * irdma_modify_port - modify port attributes
1495 * @ibdev: device pointer from stack
1496 * @port: port number for query
1497 * @mask: Property mask
1498 * @props: returning device attributes
1499 */
1500 int
irdma_modify_port(struct ib_device * ibdev,u8 port,int mask,struct ib_port_modify * props)1501 irdma_modify_port(struct ib_device *ibdev, u8 port, int mask,
1502 struct ib_port_modify *props)
1503 {
1504 if (port > 1)
1505 return -EINVAL;
1506
1507 return 0;
1508 }
1509
1510 /**
1511 * irdma_query_pkey - Query partition key
1512 * @ibdev: device pointer from stack
1513 * @port: port number
1514 * @index: index of pkey
1515 * @pkey: pointer to store the pkey
1516 */
1517 int
irdma_query_pkey(struct ib_device * ibdev,u8 port,u16 index,u16 * pkey)1518 irdma_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
1519 u16 *pkey)
1520 {
1521 if (index >= IRDMA_PKEY_TBL_SZ)
1522 return -EINVAL;
1523
1524 *pkey = IRDMA_DEFAULT_PKEY;
1525 return 0;
1526 }
1527
1528 int
irdma_roce_port_immutable(struct ib_device * ibdev,u8 port_num,struct ib_port_immutable * immutable)1529 irdma_roce_port_immutable(struct ib_device *ibdev, u8 port_num,
1530 struct ib_port_immutable *immutable)
1531 {
1532 struct ib_port_attr attr;
1533 int err;
1534
1535 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
1536 err = ib_query_port(ibdev, port_num, &attr);
1537 if (err)
1538 return err;
1539
1540 immutable->max_mad_size = IB_MGMT_MAD_SIZE;
1541 immutable->pkey_tbl_len = attr.pkey_tbl_len;
1542 immutable->gid_tbl_len = attr.gid_tbl_len;
1543
1544 return 0;
1545 }
1546
1547 int
irdma_iw_port_immutable(struct ib_device * ibdev,u8 port_num,struct ib_port_immutable * immutable)1548 irdma_iw_port_immutable(struct ib_device *ibdev, u8 port_num,
1549 struct ib_port_immutable *immutable)
1550 {
1551 struct ib_port_attr attr;
1552 int err;
1553
1554 immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
1555 err = ib_query_port(ibdev, port_num, &attr);
1556 if (err)
1557 return err;
1558 immutable->gid_tbl_len = 1;
1559
1560 return 0;
1561 }
1562
1563 /**
1564 * irdma_query_port - get port attributes
1565 * @ibdev: device pointer from stack
1566 * @port: port number for query
1567 * @props: returning device attributes
1568 */
1569 int
irdma_query_port(struct ib_device * ibdev,u8 port,struct ib_port_attr * props)1570 irdma_query_port(struct ib_device *ibdev, u8 port,
1571 struct ib_port_attr *props)
1572 {
1573 struct irdma_device *iwdev = to_iwdev(ibdev);
1574 if_t netdev = iwdev->netdev;
1575
1576 /* no need to zero out pros here. done by caller */
1577
1578 props->max_mtu = IB_MTU_4096;
1579 props->active_mtu = min(props->max_mtu, iboe_get_mtu(if_getmtu(netdev)));
1580 props->lid = 1;
1581 props->lmc = 0;
1582 props->sm_lid = 0;
1583 props->sm_sl = 0;
1584 if ((if_getlinkstate(netdev) == LINK_STATE_UP) && (if_getdrvflags(netdev) & IFF_DRV_RUNNING)) {
1585 props->state = IB_PORT_ACTIVE;
1586 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
1587 } else {
1588 props->state = IB_PORT_DOWN;
1589 props->phys_state = IB_PORT_PHYS_STATE_DISABLED;
1590 }
1591 ib_get_eth_speed(ibdev, port, &props->active_speed, &props->active_width);
1592
1593 if (rdma_protocol_roce(ibdev, 1)) {
1594 props->gid_tbl_len = 32;
1595 props->port_cap_flags |= IB_PORT_IP_BASED_GIDS;
1596 props->pkey_tbl_len = IRDMA_PKEY_TBL_SZ;
1597 } else {
1598 props->gid_tbl_len = 1;
1599 }
1600 props->qkey_viol_cntr = 0;
1601 props->port_cap_flags |= IB_PORT_CM_SUP | IB_PORT_REINIT_SUP;
1602 props->max_msg_sz = iwdev->rf->sc_dev.hw_attrs.max_hw_outbound_msg_size;
1603
1604 return 0;
1605 }
1606
1607 static const char *const irdma_hw_stat_names[] = {
1608 /* gen1 - 32-bit */
1609 [IRDMA_HW_STAT_INDEX_IP4RXDISCARD] = "ip4InDiscards",
1610 [IRDMA_HW_STAT_INDEX_IP4RXTRUNC] = "ip4InTruncatedPkts",
1611 [IRDMA_HW_STAT_INDEX_IP4TXNOROUTE] = "ip4OutNoRoutes",
1612 [IRDMA_HW_STAT_INDEX_IP6RXDISCARD] = "ip6InDiscards",
1613 [IRDMA_HW_STAT_INDEX_IP6RXTRUNC] = "ip6InTruncatedPkts",
1614 [IRDMA_HW_STAT_INDEX_IP6TXNOROUTE] = "ip6OutNoRoutes",
1615 [IRDMA_HW_STAT_INDEX_RXVLANERR] = "rxVlanErrors",
1616 /* gen1 - 64-bit */
1617 [IRDMA_HW_STAT_INDEX_IP4RXOCTS] = "ip4InOctets",
1618 [IRDMA_HW_STAT_INDEX_IP4RXPKTS] = "ip4InPkts",
1619 [IRDMA_HW_STAT_INDEX_IP4RXFRAGS] = "ip4InReasmRqd",
1620 [IRDMA_HW_STAT_INDEX_IP4RXMCPKTS] = "ip4InMcastPkts",
1621 [IRDMA_HW_STAT_INDEX_IP4TXOCTS] = "ip4OutOctets",
1622 [IRDMA_HW_STAT_INDEX_IP4TXPKTS] = "ip4OutPkts",
1623 [IRDMA_HW_STAT_INDEX_IP4TXFRAGS] = "ip4OutSegRqd",
1624 [IRDMA_HW_STAT_INDEX_IP4TXMCPKTS] = "ip4OutMcastPkts",
1625 [IRDMA_HW_STAT_INDEX_IP6RXOCTS] = "ip6InOctets",
1626 [IRDMA_HW_STAT_INDEX_IP6RXPKTS] = "ip6InPkts",
1627 [IRDMA_HW_STAT_INDEX_IP6RXFRAGS] = "ip6InReasmRqd",
1628 [IRDMA_HW_STAT_INDEX_IP6RXMCPKTS] = "ip6InMcastPkts",
1629 [IRDMA_HW_STAT_INDEX_IP6TXOCTS] = "ip6OutOctets",
1630 [IRDMA_HW_STAT_INDEX_IP6TXPKTS] = "ip6OutPkts",
1631 [IRDMA_HW_STAT_INDEX_IP6TXFRAGS] = "ip6OutSegRqd",
1632 [IRDMA_HW_STAT_INDEX_IP6TXMCPKTS] = "ip6OutMcastPkts",
1633 [IRDMA_HW_STAT_INDEX_RDMARXRDS] = "InRdmaReads",
1634 [IRDMA_HW_STAT_INDEX_RDMARXSNDS] = "InRdmaSends",
1635 [IRDMA_HW_STAT_INDEX_RDMARXWRS] = "InRdmaWrites",
1636 [IRDMA_HW_STAT_INDEX_RDMATXRDS] = "OutRdmaReads",
1637 [IRDMA_HW_STAT_INDEX_RDMATXSNDS] = "OutRdmaSends",
1638 [IRDMA_HW_STAT_INDEX_RDMATXWRS] = "OutRdmaWrites",
1639 [IRDMA_HW_STAT_INDEX_RDMAVBND] = "RdmaBnd",
1640 [IRDMA_HW_STAT_INDEX_RDMAVINV] = "RdmaInv",
1641
1642 /* gen2 - 32-bit */
1643 [IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED] = "cnpHandled",
1644 [IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED] = "cnpIgnored",
1645 [IRDMA_HW_STAT_INDEX_TXNPCNPSENT] = "cnpSent",
1646 /* gen2 - 64-bit */
1647 [IRDMA_HW_STAT_INDEX_IP4RXMCOCTS] = "ip4InMcastOctets",
1648 [IRDMA_HW_STAT_INDEX_IP4TXMCOCTS] = "ip4OutMcastOctets",
1649 [IRDMA_HW_STAT_INDEX_IP6RXMCOCTS] = "ip6InMcastOctets",
1650 [IRDMA_HW_STAT_INDEX_IP6TXMCOCTS] = "ip6OutMcastOctets",
1651 [IRDMA_HW_STAT_INDEX_UDPRXPKTS] = "RxUDP",
1652 [IRDMA_HW_STAT_INDEX_UDPTXPKTS] = "TxUDP",
1653 [IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS] = "RxECNMrkd",
1654 [IRDMA_HW_STAT_INDEX_TCPRTXSEG] = "RetransSegs",
1655 [IRDMA_HW_STAT_INDEX_TCPRXOPTERR] = "InOptErrors",
1656 [IRDMA_HW_STAT_INDEX_TCPRXPROTOERR] = "InProtoErrors",
1657 [IRDMA_HW_STAT_INDEX_TCPRXSEGS] = "InSegs",
1658 [IRDMA_HW_STAT_INDEX_TCPTXSEG] = "OutSegs",
1659 };
1660
1661 /**
1662 * irdma_alloc_hw_stats - Allocate a hw stats structure
1663 * @ibdev: device pointer from stack
1664 * @port_num: port number
1665 */
1666 struct rdma_hw_stats *
irdma_alloc_hw_stats(struct ib_device * ibdev,u8 port_num)1667 irdma_alloc_hw_stats(struct ib_device *ibdev,
1668 u8 port_num)
1669 {
1670 struct irdma_device *iwdev = to_iwdev(ibdev);
1671 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;
1672
1673 int num_counters = dev->hw_attrs.max_stat_idx;
1674 unsigned long lifespan = RDMA_HW_STATS_DEFAULT_LIFESPAN;
1675
1676 return rdma_alloc_hw_stats_struct(irdma_hw_stat_names, num_counters,
1677 lifespan);
1678 }
1679
1680 /**
1681 * irdma_get_hw_stats - Populates the rdma_hw_stats structure
1682 * @ibdev: device pointer from stack
1683 * @stats: stats pointer from stack
1684 * @port_num: port number
1685 * @index: which hw counter the stack is requesting we update
1686 */
1687 int
irdma_get_hw_stats(struct ib_device * ibdev,struct rdma_hw_stats * stats,u8 port_num,int index)1688 irdma_get_hw_stats(struct ib_device *ibdev,
1689 struct rdma_hw_stats *stats, u8 port_num,
1690 int index)
1691 {
1692 struct irdma_device *iwdev = to_iwdev(ibdev);
1693 struct irdma_dev_hw_stats *hw_stats = &iwdev->vsi.pestat->hw_stats;
1694
1695 if (iwdev->rf->rdma_ver >= IRDMA_GEN_2)
1696 irdma_cqp_gather_stats_cmd(&iwdev->rf->sc_dev, iwdev->vsi.pestat, true);
1697
1698 memcpy(&stats->value[0], hw_stats, sizeof(u64)* stats->num_counters);
1699
1700 return stats->num_counters;
1701 }
1702
1703 /**
1704 * irdma_query_gid - Query port GID
1705 * @ibdev: device pointer from stack
1706 * @port: port number
1707 * @index: Entry index
1708 * @gid: Global ID
1709 */
1710 int
irdma_query_gid(struct ib_device * ibdev,u8 port,int index,union ib_gid * gid)1711 irdma_query_gid(struct ib_device *ibdev, u8 port, int index,
1712 union ib_gid *gid)
1713 {
1714 struct irdma_device *iwdev = to_iwdev(ibdev);
1715
1716 memset(gid->raw, 0, sizeof(gid->raw));
1717 ether_addr_copy(gid->raw, if_getlladdr(iwdev->netdev));
1718
1719 return 0;
1720 }
1721
1722 enum rdma_link_layer
irdma_get_link_layer(struct ib_device * ibdev,u8 port_num)1723 irdma_get_link_layer(struct ib_device *ibdev,
1724 u8 port_num)
1725 {
1726 return IB_LINK_LAYER_ETHERNET;
1727 }
1728
1729 inline void
kc_set_roce_uverbs_cmd_mask(struct irdma_device * iwdev)1730 kc_set_roce_uverbs_cmd_mask(struct irdma_device *iwdev)
1731 {
1732 iwdev->ibdev.uverbs_cmd_mask |=
1733 BIT_ULL(IB_USER_VERBS_CMD_ATTACH_MCAST) |
1734 BIT_ULL(IB_USER_VERBS_CMD_CREATE_AH) |
1735 BIT_ULL(IB_USER_VERBS_CMD_DESTROY_AH) |
1736 BIT_ULL(IB_USER_VERBS_CMD_DETACH_MCAST);
1737 }
1738
1739 inline void
kc_set_rdma_uverbs_cmd_mask(struct irdma_device * iwdev)1740 kc_set_rdma_uverbs_cmd_mask(struct irdma_device *iwdev)
1741 {
1742 iwdev->ibdev.uverbs_cmd_mask =
1743 BIT_ULL(IB_USER_VERBS_CMD_GET_CONTEXT) |
1744 BIT_ULL(IB_USER_VERBS_CMD_QUERY_DEVICE) |
1745 BIT_ULL(IB_USER_VERBS_CMD_QUERY_PORT) |
1746 BIT_ULL(IB_USER_VERBS_CMD_ALLOC_PD) |
1747 BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_PD) |
1748 BIT_ULL(IB_USER_VERBS_CMD_REG_MR) |
1749 BIT_ULL(IB_USER_VERBS_CMD_REREG_MR) |
1750 BIT_ULL(IB_USER_VERBS_CMD_DEREG_MR) |
1751 BIT_ULL(IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
1752 BIT_ULL(IB_USER_VERBS_CMD_CREATE_CQ) |
1753 BIT_ULL(IB_USER_VERBS_CMD_RESIZE_CQ) |
1754 BIT_ULL(IB_USER_VERBS_CMD_DESTROY_CQ) |
1755 BIT_ULL(IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |
1756 BIT_ULL(IB_USER_VERBS_CMD_CREATE_QP) |
1757 BIT_ULL(IB_USER_VERBS_CMD_MODIFY_QP) |
1758 BIT_ULL(IB_USER_VERBS_CMD_QUERY_QP) |
1759 BIT_ULL(IB_USER_VERBS_CMD_POLL_CQ) |
1760 BIT_ULL(IB_USER_VERBS_CMD_DESTROY_QP) |
1761 BIT_ULL(IB_USER_VERBS_CMD_POST_RECV) |
1762 BIT_ULL(IB_USER_VERBS_CMD_POST_SEND);
1763 iwdev->ibdev.uverbs_ex_cmd_mask =
1764 BIT_ULL(IB_USER_VERBS_EX_CMD_MODIFY_QP) |
1765 BIT_ULL(IB_USER_VERBS_EX_CMD_QUERY_DEVICE);
1766
1767 if (iwdev->rf->rdma_ver >= IRDMA_GEN_2)
1768 iwdev->ibdev.uverbs_ex_cmd_mask |= BIT_ULL(IB_USER_VERBS_EX_CMD_CREATE_CQ);
1769 }
1770
1771 static void
ib_get_width_and_speed(u32 netdev_speed,u32 lanes,u16 * speed,u8 * width)1772 ib_get_width_and_speed(u32 netdev_speed, u32 lanes,
1773 u16 *speed, u8 *width)
1774 {
1775 if (!lanes) {
1776 if (netdev_speed <= SPEED_1000) {
1777 *width = IB_WIDTH_1X;
1778 *speed = IB_SPEED_SDR;
1779 } else if (netdev_speed <= SPEED_10000) {
1780 *width = IB_WIDTH_1X;
1781 *speed = IB_SPEED_FDR10;
1782 } else if (netdev_speed <= SPEED_20000) {
1783 *width = IB_WIDTH_4X;
1784 *speed = IB_SPEED_DDR;
1785 } else if (netdev_speed <= SPEED_25000) {
1786 *width = IB_WIDTH_1X;
1787 *speed = IB_SPEED_EDR;
1788 } else if (netdev_speed <= SPEED_40000) {
1789 *width = IB_WIDTH_4X;
1790 *speed = IB_SPEED_FDR10;
1791 } else if (netdev_speed <= SPEED_50000) {
1792 *width = IB_WIDTH_2X;
1793 *speed = IB_SPEED_EDR;
1794 } else if (netdev_speed <= SPEED_100000) {
1795 *width = IB_WIDTH_4X;
1796 *speed = IB_SPEED_EDR;
1797 } else if (netdev_speed <= SPEED_200000) {
1798 *width = IB_WIDTH_4X;
1799 *speed = IB_SPEED_HDR;
1800 } else {
1801 *width = IB_WIDTH_4X;
1802 *speed = IB_SPEED_NDR;
1803 }
1804
1805 return;
1806 }
1807
1808 switch (lanes) {
1809 case 1:
1810 *width = IB_WIDTH_1X;
1811 break;
1812 case 2:
1813 *width = IB_WIDTH_2X;
1814 break;
1815 case 4:
1816 *width = IB_WIDTH_4X;
1817 break;
1818 case 8:
1819 *width = IB_WIDTH_8X;
1820 break;
1821 case 12:
1822 *width = IB_WIDTH_12X;
1823 break;
1824 default:
1825 *width = IB_WIDTH_1X;
1826 }
1827
1828 switch (netdev_speed / lanes) {
1829 case SPEED_2500:
1830 *speed = IB_SPEED_SDR;
1831 break;
1832 case SPEED_5000:
1833 *speed = IB_SPEED_DDR;
1834 break;
1835 case SPEED_10000:
1836 *speed = IB_SPEED_FDR10;
1837 break;
1838 case SPEED_14000:
1839 *speed = IB_SPEED_FDR;
1840 break;
1841 case SPEED_25000:
1842 *speed = IB_SPEED_EDR;
1843 break;
1844 case SPEED_50000:
1845 *speed = IB_SPEED_HDR;
1846 break;
1847 case SPEED_100000:
1848 *speed = IB_SPEED_NDR;
1849 break;
1850 default:
1851 *speed = IB_SPEED_SDR;
1852 }
1853 }
1854
1855 int
ib_get_eth_speed(struct ib_device * ibdev,u32 port_num,u16 * speed,u8 * width)1856 ib_get_eth_speed(struct ib_device *ibdev, u32 port_num, u16 *speed, u8 *width)
1857 {
1858 if_t netdev = ibdev->get_netdev(ibdev, port_num);
1859 u32 netdev_speed, lanes;
1860
1861 if (!netdev)
1862 return -ENODEV;
1863
1864 netdev_speed = (u32)if_getbaudrate(netdev);
1865 dev_put(netdev);
1866 lanes = 0;
1867
1868 ib_get_width_and_speed(netdev_speed, lanes, speed, width);
1869
1870 return 0;
1871 }
1872
1873 u64
irdma_mac_to_u64(const u8 * eth_add)1874 irdma_mac_to_u64(const u8 *eth_add)
1875 {
1876 int idx;
1877 u64 u64_eth_add;
1878
1879 for (idx = 0, u64_eth_add = 0; idx < ETHER_ADDR_LEN; idx++)
1880 u64_eth_add = u64_eth_add << 8 | eth_add[idx];
1881
1882 return u64_eth_add;
1883 }
1884