1 // SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
2 /* Copyright (c) 2015 - 2024 Intel Corporation */
3
4 #include "main.h"
5 #include <linux/net/intel/iidc_rdma_ice.h>
6
icrdma_prep_tc_change(struct irdma_device * iwdev)7 static void icrdma_prep_tc_change(struct irdma_device *iwdev)
8 {
9 iwdev->vsi.tc_change_pending = true;
10 irdma_sc_suspend_resume_qps(&iwdev->vsi, IRDMA_OP_SUSPEND);
11
12 /* Wait for all qp's to suspend */
13 wait_event_timeout(iwdev->suspend_wq,
14 !atomic_read(&iwdev->vsi.qp_suspend_reqs),
15 msecs_to_jiffies(IRDMA_EVENT_TIMEOUT_MS));
16 irdma_ws_reset(&iwdev->vsi);
17 }
18
icrdma_fill_qos_info(struct irdma_l2params * l2params,struct iidc_rdma_qos_params * qos_info)19 static void icrdma_fill_qos_info(struct irdma_l2params *l2params,
20 struct iidc_rdma_qos_params *qos_info)
21 {
22 int i;
23
24 l2params->num_tc = qos_info->num_tc;
25 l2params->vsi_prio_type = qos_info->vport_priority_type;
26 l2params->vsi_rel_bw = qos_info->vport_relative_bw;
27 for (i = 0; i < l2params->num_tc; i++) {
28 l2params->tc_info[i].egress_virt_up =
29 qos_info->tc_info[i].egress_virt_up;
30 l2params->tc_info[i].ingress_virt_up =
31 qos_info->tc_info[i].ingress_virt_up;
32 l2params->tc_info[i].prio_type = qos_info->tc_info[i].prio_type;
33 l2params->tc_info[i].rel_bw = qos_info->tc_info[i].rel_bw;
34 l2params->tc_info[i].tc_ctx = qos_info->tc_info[i].tc_ctx;
35 }
36 for (i = 0; i < IIDC_MAX_USER_PRIORITY; i++)
37 l2params->up2tc[i] = qos_info->up2tc[i];
38 if (qos_info->pfc_mode == IIDC_DSCP_PFC_MODE) {
39 l2params->dscp_mode = true;
40 memcpy(l2params->dscp_map, qos_info->dscp_map, sizeof(l2params->dscp_map));
41 }
42 }
43
icrdma_iidc_event_handler(struct iidc_rdma_core_dev_info * cdev_info,struct iidc_rdma_event * event)44 static void icrdma_iidc_event_handler(struct iidc_rdma_core_dev_info *cdev_info,
45 struct iidc_rdma_event *event)
46 {
47 struct irdma_device *iwdev = dev_get_drvdata(&cdev_info->adev->dev);
48 struct irdma_l2params l2params = {};
49
50 if (*event->type & BIT(IIDC_RDMA_EVENT_AFTER_MTU_CHANGE)) {
51 ibdev_dbg(&iwdev->ibdev, "CLNT: new MTU = %d\n", iwdev->netdev->mtu);
52 if (iwdev->vsi.mtu != iwdev->netdev->mtu) {
53 l2params.mtu = iwdev->netdev->mtu;
54 l2params.mtu_changed = true;
55 irdma_log_invalid_mtu(l2params.mtu, &iwdev->rf->sc_dev);
56 irdma_change_l2params(&iwdev->vsi, &l2params);
57 }
58 } else if (*event->type & BIT(IIDC_RDMA_EVENT_BEFORE_TC_CHANGE)) {
59 if (iwdev->vsi.tc_change_pending)
60 return;
61
62 icrdma_prep_tc_change(iwdev);
63 } else if (*event->type & BIT(IIDC_RDMA_EVENT_AFTER_TC_CHANGE)) {
64 struct iidc_rdma_priv_dev_info *idc_priv = cdev_info->iidc_priv;
65
66 if (!iwdev->vsi.tc_change_pending)
67 return;
68
69 l2params.tc_changed = true;
70 ibdev_dbg(&iwdev->ibdev, "CLNT: TC Change\n");
71
72 icrdma_fill_qos_info(&l2params, &idc_priv->qos_info);
73 if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY)
74 iwdev->dcb_vlan_mode =
75 l2params.num_tc > 1 && !l2params.dscp_mode;
76 irdma_change_l2params(&iwdev->vsi, &l2params);
77 } else if (*event->type & BIT(IIDC_RDMA_EVENT_CRIT_ERR)) {
78 ibdev_warn(&iwdev->ibdev, "ICE OICR event notification: oicr = 0x%08x\n",
79 event->reg);
80 if (event->reg & IRDMAPFINT_OICR_PE_CRITERR_M) {
81 u32 pe_criterr;
82
83 pe_criterr = readl(iwdev->rf->sc_dev.hw_regs[IRDMA_GLPE_CRITERR]);
84 #define IRDMA_Q1_RESOURCE_ERR 0x0001024d
85 if (pe_criterr != IRDMA_Q1_RESOURCE_ERR) {
86 ibdev_err(&iwdev->ibdev, "critical PE Error, GLPE_CRITERR=0x%08x\n",
87 pe_criterr);
88 iwdev->rf->reset = true;
89 } else {
90 ibdev_warn(&iwdev->ibdev, "Q1 Resource Check\n");
91 }
92 }
93 if (event->reg & IRDMAPFINT_OICR_HMC_ERR_M) {
94 ibdev_err(&iwdev->ibdev, "HMC Error\n");
95 iwdev->rf->reset = true;
96 }
97 if (event->reg & IRDMAPFINT_OICR_PE_PUSH_M) {
98 ibdev_err(&iwdev->ibdev, "PE Push Error\n");
99 iwdev->rf->reset = true;
100 }
101 if (iwdev->rf->reset)
102 iwdev->rf->gen_ops.request_reset(iwdev->rf);
103 }
104 }
105
106 /**
107 * icrdma_lan_register_qset - Register qset with LAN driver
108 * @vsi: vsi structure
109 * @tc_node: Traffic class node
110 */
icrdma_lan_register_qset(struct irdma_sc_vsi * vsi,struct irdma_ws_node * tc_node)111 static int icrdma_lan_register_qset(struct irdma_sc_vsi *vsi,
112 struct irdma_ws_node *tc_node)
113 {
114 struct irdma_device *iwdev = vsi->back_vsi;
115 struct iidc_rdma_core_dev_info *cdev_info = iwdev->rf->cdev;
116 struct iidc_rdma_qset_params qset = {};
117 int ret;
118
119 qset.qs_handle = tc_node->qs_handle;
120 qset.tc = tc_node->traffic_class;
121 qset.vport_id = vsi->vsi_idx;
122 ret = ice_add_rdma_qset(cdev_info, &qset);
123 if (ret) {
124 ibdev_dbg(&iwdev->ibdev, "WS: LAN alloc_res for rdma qset failed.\n");
125 return ret;
126 }
127
128 tc_node->l2_sched_node_id = qset.teid;
129 vsi->qos[tc_node->user_pri].l2_sched_node_id = qset.teid;
130
131 return 0;
132 }
133
134 /**
135 * icrdma_lan_unregister_qset - Unregister qset with LAN driver
136 * @vsi: vsi structure
137 * @tc_node: Traffic class node
138 */
icrdma_lan_unregister_qset(struct irdma_sc_vsi * vsi,struct irdma_ws_node * tc_node)139 static void icrdma_lan_unregister_qset(struct irdma_sc_vsi *vsi,
140 struct irdma_ws_node *tc_node)
141 {
142 struct irdma_device *iwdev = vsi->back_vsi;
143 struct iidc_rdma_core_dev_info *cdev_info = iwdev->rf->cdev;
144 struct iidc_rdma_qset_params qset = {};
145
146 qset.qs_handle = tc_node->qs_handle;
147 qset.tc = tc_node->traffic_class;
148 qset.vport_id = vsi->vsi_idx;
149 qset.teid = tc_node->l2_sched_node_id;
150
151 if (ice_del_rdma_qset(cdev_info, &qset))
152 ibdev_dbg(&iwdev->ibdev, "WS: LAN free_res for rdma qset failed.\n");
153 }
154
155 /**
156 * icrdma_request_reset - Request a reset
157 * @rf: RDMA PCI function
158 */
icrdma_request_reset(struct irdma_pci_f * rf)159 static void icrdma_request_reset(struct irdma_pci_f *rf)
160 {
161 ibdev_warn(&rf->iwdev->ibdev, "Requesting a reset\n");
162 ice_rdma_request_reset(rf->cdev, IIDC_FUNC_RESET);
163 }
164
icrdma_init_interrupts(struct irdma_pci_f * rf,struct iidc_rdma_core_dev_info * cdev)165 static int icrdma_init_interrupts(struct irdma_pci_f *rf, struct iidc_rdma_core_dev_info *cdev)
166 {
167 int i;
168
169 rf->msix_count = num_online_cpus() + IRDMA_NUM_AEQ_MSIX;
170 rf->msix_entries = kzalloc_objs(*rf->msix_entries, rf->msix_count);
171 if (!rf->msix_entries)
172 return -ENOMEM;
173
174 for (i = 0; i < rf->msix_count; i++)
175 if (ice_alloc_rdma_qvector(cdev, &rf->msix_entries[i]))
176 break;
177
178 if (i < IRDMA_MIN_MSIX) {
179 while (--i >= 0)
180 ice_free_rdma_qvector(cdev, &rf->msix_entries[i]);
181
182 kfree(rf->msix_entries);
183 return -ENOMEM;
184 }
185
186 rf->msix_count = i;
187
188 return 0;
189 }
190
icrdma_deinit_interrupts(struct irdma_pci_f * rf,struct iidc_rdma_core_dev_info * cdev)191 static void icrdma_deinit_interrupts(struct irdma_pci_f *rf, struct iidc_rdma_core_dev_info *cdev)
192 {
193 int i;
194
195 for (i = 0; i < rf->msix_count; i++)
196 ice_free_rdma_qvector(cdev, &rf->msix_entries[i]);
197
198 kfree(rf->msix_entries);
199 }
200
icrdma_fill_device_info(struct irdma_device * iwdev,struct iidc_rdma_core_dev_info * cdev_info)201 static void icrdma_fill_device_info(struct irdma_device *iwdev,
202 struct iidc_rdma_core_dev_info *cdev_info)
203 {
204 struct iidc_rdma_priv_dev_info *idc_priv = cdev_info->iidc_priv;
205 struct irdma_pci_f *rf = iwdev->rf;
206
207 rf->sc_dev.hw = &rf->hw;
208 rf->iwdev = iwdev;
209 rf->cdev = cdev_info;
210 rf->hw.hw_addr = idc_priv->hw_addr;
211 rf->pcidev = cdev_info->pdev;
212 rf->hw.device = &rf->pcidev->dev;
213 rf->pf_id = idc_priv->pf_id;
214 rf->rdma_ver = IRDMA_GEN_2;
215 rf->sc_dev.hw_attrs.uk_attrs.hw_rev = IRDMA_GEN_2;
216 rf->sc_dev.is_pf = true;
217 rf->sc_dev.privileged = true;
218
219 rf->gen_ops.register_qset = icrdma_lan_register_qset;
220 rf->gen_ops.unregister_qset = icrdma_lan_unregister_qset;
221
222 rf->default_vsi.vsi_idx = idc_priv->vport_id;
223 rf->protocol_used =
224 cdev_info->rdma_protocol == IIDC_RDMA_PROTOCOL_ROCEV2 ?
225 IRDMA_ROCE_PROTOCOL_ONLY : IRDMA_IWARP_PROTOCOL_ONLY;
226 rf->rsrc_profile = IRDMA_HMC_PROFILE_DEFAULT;
227 rf->rst_to = IRDMA_RST_TIMEOUT_HZ;
228 rf->gen_ops.request_reset = icrdma_request_reset;
229 rf->limits_sel = 7;
230 mutex_init(&rf->ah_tbl_lock);
231
232 iwdev->netdev = idc_priv->netdev;
233 iwdev->vsi_num = idc_priv->vport_id;
234 iwdev->init_state = INITIAL_STATE;
235 iwdev->roce_cwnd = IRDMA_ROCE_CWND_DEFAULT;
236 iwdev->roce_ackcreds = IRDMA_ROCE_ACKCREDS_DEFAULT;
237 iwdev->rcv_wnd = IRDMA_CM_DEFAULT_RCV_WND_SCALED;
238 iwdev->rcv_wscale = IRDMA_CM_DEFAULT_RCV_WND_SCALE;
239 if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY)
240 iwdev->roce_mode = true;
241 }
242
icrdma_probe(struct auxiliary_device * aux_dev,const struct auxiliary_device_id * id)243 static int icrdma_probe(struct auxiliary_device *aux_dev, const struct auxiliary_device_id *id)
244 {
245 struct iidc_rdma_core_auxiliary_dev *iidc_adev;
246 struct iidc_rdma_core_dev_info *cdev_info;
247 struct iidc_rdma_priv_dev_info *idc_priv;
248 struct irdma_l2params l2params = {};
249 struct irdma_device *iwdev;
250 struct irdma_pci_f *rf;
251 int err;
252
253 iidc_adev = container_of(aux_dev, struct iidc_rdma_core_auxiliary_dev, adev);
254 cdev_info = iidc_adev->cdev_info;
255 idc_priv = cdev_info->iidc_priv;
256
257 iwdev = ib_alloc_device(irdma_device, ibdev);
258 if (!iwdev)
259 return -ENOMEM;
260 iwdev->rf = kzalloc_obj(*rf);
261 if (!iwdev->rf) {
262 ib_dealloc_device(&iwdev->ibdev);
263 return -ENOMEM;
264 }
265
266 icrdma_fill_device_info(iwdev, cdev_info);
267 rf = iwdev->rf;
268
269 err = icrdma_init_interrupts(rf, cdev_info);
270 if (err)
271 goto err_init_interrupts;
272
273 err = irdma_ctrl_init_hw(rf);
274 if (err)
275 goto err_ctrl_init;
276
277 l2params.mtu = iwdev->netdev->mtu;
278 icrdma_fill_qos_info(&l2params, &idc_priv->qos_info);
279 if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY)
280 iwdev->dcb_vlan_mode = l2params.num_tc > 1 && !l2params.dscp_mode;
281
282 err = irdma_rt_init_hw(iwdev, &l2params);
283 if (err)
284 goto err_rt_init;
285
286 err = irdma_ib_register_device(iwdev);
287 if (err)
288 goto err_ibreg;
289
290 ice_rdma_update_vsi_filter(cdev_info, iwdev->vsi_num, true);
291
292 ibdev_dbg(&iwdev->ibdev, "INIT: Gen2 PF[%d] device probe success\n", PCI_FUNC(rf->pcidev->devfn));
293 auxiliary_set_drvdata(aux_dev, iwdev);
294
295 return 0;
296
297 err_ibreg:
298 irdma_rt_deinit_hw(iwdev);
299 err_rt_init:
300 irdma_ctrl_deinit_hw(rf);
301 err_ctrl_init:
302 icrdma_deinit_interrupts(rf, cdev_info);
303 err_init_interrupts:
304 mutex_destroy(&rf->ah_tbl_lock);
305 kfree(rf);
306 ib_dealloc_device(&iwdev->ibdev);
307
308 return err;
309 }
310
icrdma_remove(struct auxiliary_device * aux_dev)311 static void icrdma_remove(struct auxiliary_device *aux_dev)
312 {
313 struct iidc_rdma_core_auxiliary_dev *idc_adev =
314 container_of(aux_dev, struct iidc_rdma_core_auxiliary_dev, adev);
315 struct iidc_rdma_core_dev_info *cdev_info = idc_adev->cdev_info;
316 struct irdma_device *iwdev = auxiliary_get_drvdata(aux_dev);
317 u8 rdma_ver = iwdev->rf->rdma_ver;
318
319 ice_rdma_update_vsi_filter(cdev_info, iwdev->vsi_num, false);
320 irdma_ib_unregister_device(iwdev);
321 icrdma_deinit_interrupts(iwdev->rf, cdev_info);
322 mutex_destroy(&iwdev->rf->ah_tbl_lock);
323
324 kfree(iwdev->rf);
325
326 pr_debug("INIT: Gen[%d] func[%d] device remove success\n",
327 rdma_ver, PCI_FUNC(cdev_info->pdev->devfn));
328 }
329
330 static const struct auxiliary_device_id icrdma_auxiliary_id_table[] = {
331 {.name = "ice.iwarp", },
332 {.name = "ice.roce", },
333 {},
334 };
335
336 MODULE_DEVICE_TABLE(auxiliary, icrdma_auxiliary_id_table);
337
338 struct iidc_rdma_core_auxiliary_drv icrdma_core_auxiliary_drv = {
339 .adrv = {
340 .name = "gen_2",
341 .id_table = icrdma_auxiliary_id_table,
342 .probe = icrdma_probe,
343 .remove = icrdma_remove,
344 },
345 .event_handler = icrdma_iidc_event_handler,
346 };
347