1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2023 Intel Corporation */
3
4 #include "idpf.h"
5 #include "idpf_devids.h"
6 #include "idpf_lan_vf_regs.h"
7 #include "idpf_virtchnl.h"
8
9 #define DRV_SUMMARY "Intel(R) Infrastructure Data Path Function Linux Driver"
10
11 #define IDPF_NETWORK_ETHERNET_PROGIF 0x01
12 #define IDPF_CLASS_NETWORK_ETHERNET_PROGIF \
13 (PCI_CLASS_NETWORK_ETHERNET << 8 | IDPF_NETWORK_ETHERNET_PROGIF)
14 #define IDPF_VF_TEST_VAL 0xfeed0000u
15
16 MODULE_DESCRIPTION(DRV_SUMMARY);
17 MODULE_IMPORT_NS("LIBETH");
18 MODULE_IMPORT_NS("LIBIE_CP");
19 MODULE_IMPORT_NS("LIBIE_PCI");
20 MODULE_IMPORT_NS("LIBETH_XDP");
21 MODULE_LICENSE("GPL");
22
23 /**
24 * idpf_get_device_type - Helper to find if it is a VF or PF device
25 * @pdev: PCI device information struct
26 *
27 * Return: PF/VF device ID or -%errno on failure.
28 */
idpf_get_device_type(struct pci_dev * pdev)29 static int idpf_get_device_type(struct pci_dev *pdev)
30 {
31 void __iomem *addr;
32 int ret;
33
34 addr = ioremap(pci_resource_start(pdev, 0) + VF_ARQBAL, 4);
35 if (!addr) {
36 pci_err(pdev, "Failed to allocate BAR0 mbx region\n");
37 return -EIO;
38 }
39
40 writel(IDPF_VF_TEST_VAL, addr);
41 if (readl(addr) == IDPF_VF_TEST_VAL)
42 ret = IDPF_DEV_ID_VF;
43 else
44 ret = IDPF_DEV_ID_PF;
45
46 iounmap(addr);
47
48 return ret;
49 }
50
51 /**
52 * idpf_dev_init - Initialize device specific parameters
53 * @adapter: adapter to initialize
54 * @ent: entry in idpf_pci_tbl
55 *
56 * Return: %0 on success, -%errno on failure.
57 */
idpf_dev_init(struct idpf_adapter * adapter,const struct pci_device_id * ent)58 static int idpf_dev_init(struct idpf_adapter *adapter,
59 const struct pci_device_id *ent)
60 {
61 struct libie_mmio_info *mmio_info = &adapter->ctlq_ctx.mmio_info;
62 int ret;
63
64 ret = libie_pci_init_dev(adapter->pdev);
65 if (ret)
66 return ret;
67
68 mmio_info->pdev = adapter->pdev;
69 INIT_LIST_HEAD(&mmio_info->mmio_list);
70
71 if (ent->class == IDPF_CLASS_NETWORK_ETHERNET_PROGIF) {
72 ret = idpf_get_device_type(adapter->pdev);
73 switch (ret) {
74 case IDPF_DEV_ID_VF:
75 idpf_vf_dev_ops_init(adapter);
76 adapter->crc_enable = true;
77 break;
78 case IDPF_DEV_ID_PF:
79 idpf_dev_ops_init(adapter);
80 break;
81 default:
82 return ret;
83 }
84
85 return 0;
86 }
87
88 switch (ent->device) {
89 case IDPF_DEV_ID_PF:
90 idpf_dev_ops_init(adapter);
91 break;
92 case IDPF_DEV_ID_VF:
93 idpf_vf_dev_ops_init(adapter);
94 adapter->crc_enable = true;
95 break;
96 default:
97 return -ENODEV;
98 }
99
100 return 0;
101 }
102
103 /**
104 * idpf_decfg_device - deconfigure device and device specific resources
105 * @adapter: driver specific private structure
106 */
idpf_decfg_device(struct idpf_adapter * adapter)107 static void idpf_decfg_device(struct idpf_adapter *adapter)
108 {
109 libie_pci_unmap_all_mmio_regions(&adapter->ctlq_ctx.mmio_info);
110 }
111
112 /**
113 * idpf_remove - Device removal routine
114 * @pdev: PCI device information struct
115 */
idpf_remove(struct pci_dev * pdev)116 static void idpf_remove(struct pci_dev *pdev)
117 {
118 struct idpf_adapter *adapter = pci_get_drvdata(pdev);
119 int i;
120
121 set_bit(IDPF_REMOVE_IN_PROG, adapter->flags);
122
123 /* Wait until vc_event_task is done to consider if any hard reset is
124 * in progress else we may go ahead and release the resources but the
125 * thread doing the hard reset might continue the init path and
126 * end up in bad state.
127 */
128 cancel_delayed_work_sync(&adapter->vc_event_task);
129 if (adapter->num_vfs)
130 idpf_sriov_configure(pdev, 0);
131
132 idpf_vc_core_deinit(adapter);
133
134 /* Be a good citizen and leave the device clean on exit */
135 adapter->dev_ops.reg_ops.trigger_reset(adapter, IDPF_HR_FUNC_RESET);
136 idpf_deinit_dflt_mbx(adapter);
137
138 if (!adapter->netdevs)
139 goto destroy_wqs;
140
141 /* There are some cases where it's possible to still have netdevs
142 * registered with the stack at this point, e.g. if the driver detected
143 * a HW reset and rmmod is called before it fully recovers. Unregister
144 * any stale netdevs here.
145 */
146 for (i = 0; i < adapter->max_vports; i++) {
147 if (!adapter->netdevs[i])
148 continue;
149 if (adapter->netdevs[i]->reg_state != NETREG_UNINITIALIZED)
150 unregister_netdev(adapter->netdevs[i]);
151 free_netdev(adapter->netdevs[i]);
152 adapter->netdevs[i] = NULL;
153 }
154
155 destroy_wqs:
156 destroy_workqueue(adapter->init_wq);
157 destroy_workqueue(adapter->serv_wq);
158 destroy_workqueue(adapter->mbx_wq);
159 destroy_workqueue(adapter->stats_wq);
160 destroy_workqueue(adapter->vc_event_wq);
161
162 for (i = 0; i < adapter->max_vports; i++) {
163 if (!adapter->vport_config[i])
164 continue;
165 kfree(adapter->vport_config[i]->user_config.q_coalesce);
166 kfree(adapter->vport_config[i]);
167 adapter->vport_config[i] = NULL;
168 }
169 kfree(adapter->vport_config);
170 adapter->vport_config = NULL;
171 kfree(adapter->netdevs);
172 adapter->netdevs = NULL;
173
174 mutex_destroy(&adapter->vport_ctrl_lock);
175 mutex_destroy(&adapter->vector_lock);
176 mutex_destroy(&adapter->queue_lock);
177 mutex_destroy(&adapter->vc_buf_lock);
178
179 idpf_decfg_device(adapter);
180 pci_set_drvdata(pdev, NULL);
181 kfree(adapter);
182 }
183
184 /**
185 * idpf_shutdown - PCI callback for shutting down device
186 * @pdev: PCI device information struct
187 */
idpf_shutdown(struct pci_dev * pdev)188 static void idpf_shutdown(struct pci_dev *pdev)
189 {
190 struct idpf_adapter *adapter = pci_get_drvdata(pdev);
191
192 cancel_delayed_work_sync(&adapter->serv_task);
193 cancel_delayed_work_sync(&adapter->vc_event_task);
194 idpf_vc_core_deinit(adapter);
195 idpf_deinit_dflt_mbx(adapter);
196
197 if (system_state == SYSTEM_POWER_OFF)
198 pci_set_power_state(pdev, PCI_D3hot);
199 }
200
201 /**
202 * idpf_cfg_device - configure device and device specific resources
203 * @adapter: driver specific private structure
204 *
205 * Return: %0 on success, -%errno on failure.
206 */
idpf_cfg_device(struct idpf_adapter * adapter)207 static int idpf_cfg_device(struct idpf_adapter *adapter)
208 {
209 struct libie_mmio_info *mmio_info = &adapter->ctlq_ctx.mmio_info;
210 struct pci_dev *pdev = adapter->pdev;
211 struct resource *region;
212 bool mapped;
213 int err;
214
215 /* Map mailbox space for virtchnl communication */
216 region = &adapter->dev_ops.static_reg_info[0];
217 mapped = libie_pci_map_mmio_region(mmio_info, region->start,
218 resource_size(region));
219 if (!mapped) {
220 pci_err(pdev, "failed to map BAR0 mbx region\n");
221 return -ENOMEM;
222 }
223
224 /* Map rstat space for resets */
225 region = &adapter->dev_ops.static_reg_info[1];
226
227 mapped = libie_pci_map_mmio_region(mmio_info, region->start,
228 resource_size(region));
229 if (!mapped) {
230 pci_err(pdev, "failed to map BAR0 rstat region\n");
231 libie_pci_unmap_all_mmio_regions(mmio_info);
232 return -ENOMEM;
233 }
234
235 err = pci_enable_ptm(pdev);
236 if (err)
237 pci_dbg(pdev, "PCIe PTM is not supported by PCIe bus/controller\n");
238
239 pci_set_drvdata(pdev, adapter);
240
241 return 0;
242 }
243
244 /**
245 * idpf_probe - Device initialization routine
246 * @pdev: PCI device information struct
247 * @ent: entry in idpf_pci_tbl
248 *
249 * Returns 0 on success, negative on failure
250 */
idpf_probe(struct pci_dev * pdev,const struct pci_device_id * ent)251 static int idpf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
252 {
253 struct device *dev = &pdev->dev;
254 struct idpf_adapter *adapter;
255 int err;
256
257 adapter = kzalloc_obj(*adapter);
258 if (!adapter)
259 return -ENOMEM;
260
261 adapter->req_tx_splitq = true;
262 adapter->req_rx_splitq = true;
263
264 adapter->pdev = pdev;
265
266 err = idpf_dev_init(adapter, ent);
267 if (err) {
268 dev_err(&pdev->dev, "Failed to initialize device (ID 0x%x): %d\n",
269 ent->device, err);
270 goto err_free;
271 }
272
273 err = idpf_cfg_device(adapter);
274 if (err) {
275 pci_err(pdev, "Failed to configure device specific resources: %pe\n",
276 ERR_PTR(err));
277 goto err_free;
278 }
279
280 adapter->init_wq = alloc_workqueue("%s-%s-init",
281 WQ_UNBOUND | WQ_MEM_RECLAIM, 0,
282 dev_driver_string(dev),
283 dev_name(dev));
284 if (!adapter->init_wq) {
285 dev_err(dev, "Failed to allocate init workqueue\n");
286 err = -ENOMEM;
287 goto err_init_wq;
288 }
289
290 adapter->serv_wq = alloc_workqueue("%s-%s-service",
291 WQ_UNBOUND | WQ_MEM_RECLAIM, 0,
292 dev_driver_string(dev),
293 dev_name(dev));
294 if (!adapter->serv_wq) {
295 dev_err(dev, "Failed to allocate service workqueue\n");
296 err = -ENOMEM;
297 goto err_serv_wq_alloc;
298 }
299
300 adapter->mbx_wq = alloc_workqueue("%s-%s-mbx", WQ_UNBOUND | WQ_HIGHPRI,
301 0, dev_driver_string(dev),
302 dev_name(dev));
303 if (!adapter->mbx_wq) {
304 dev_err(dev, "Failed to allocate mailbox workqueue\n");
305 err = -ENOMEM;
306 goto err_mbx_wq_alloc;
307 }
308
309 adapter->stats_wq = alloc_workqueue("%s-%s-stats",
310 WQ_UNBOUND | WQ_MEM_RECLAIM, 0,
311 dev_driver_string(dev),
312 dev_name(dev));
313 if (!adapter->stats_wq) {
314 dev_err(dev, "Failed to allocate workqueue\n");
315 err = -ENOMEM;
316 goto err_stats_wq_alloc;
317 }
318
319 adapter->vc_event_wq = alloc_workqueue("%s-%s-vc_event",
320 WQ_UNBOUND | WQ_MEM_RECLAIM, 0,
321 dev_driver_string(dev),
322 dev_name(dev));
323 if (!adapter->vc_event_wq) {
324 dev_err(dev, "Failed to allocate virtchnl event workqueue\n");
325 err = -ENOMEM;
326 goto err_vc_event_wq_alloc;
327 }
328
329 /* setup msglvl */
330 adapter->msg_enable = netif_msg_init(-1, IDPF_AVAIL_NETIF_M);
331
332 mutex_init(&adapter->vport_ctrl_lock);
333 mutex_init(&adapter->vector_lock);
334 mutex_init(&adapter->queue_lock);
335 mutex_init(&adapter->vc_buf_lock);
336
337 INIT_DELAYED_WORK(&adapter->init_task, idpf_init_task);
338 INIT_DELAYED_WORK(&adapter->serv_task, idpf_service_task);
339 INIT_DELAYED_WORK(&adapter->mbx_task, idpf_mbx_task);
340 INIT_DELAYED_WORK(&adapter->stats_task, idpf_statistics_task);
341 INIT_DELAYED_WORK(&adapter->vc_event_task, idpf_vc_event_task);
342
343 adapter->dev_ops.reg_ops.reset_reg_init(adapter);
344 set_bit(IDPF_HR_DRV_LOAD, adapter->flags);
345 queue_delayed_work(adapter->vc_event_wq, &adapter->vc_event_task,
346 msecs_to_jiffies(10 * (pdev->devfn & 0x07)));
347
348 return 0;
349
350 err_vc_event_wq_alloc:
351 destroy_workqueue(adapter->stats_wq);
352 err_stats_wq_alloc:
353 destroy_workqueue(adapter->mbx_wq);
354 err_mbx_wq_alloc:
355 destroy_workqueue(adapter->serv_wq);
356 err_serv_wq_alloc:
357 destroy_workqueue(adapter->init_wq);
358 err_init_wq:
359 idpf_decfg_device(adapter);
360 err_free:
361 kfree(adapter);
362 return err;
363 }
364
365 /* idpf_pci_tbl - PCI Dev idpf ID Table
366 */
367 static const struct pci_device_id idpf_pci_tbl[] = {
368 { PCI_VDEVICE(INTEL, IDPF_DEV_ID_PF)},
369 { PCI_VDEVICE(INTEL, IDPF_DEV_ID_VF)},
370 { PCI_DEVICE_CLASS(IDPF_CLASS_NETWORK_ETHERNET_PROGIF, ~0)},
371 { /* Sentinel */ }
372 };
373 MODULE_DEVICE_TABLE(pci, idpf_pci_tbl);
374
375 static struct pci_driver idpf_driver = {
376 .name = KBUILD_MODNAME,
377 .id_table = idpf_pci_tbl,
378 .probe = idpf_probe,
379 .sriov_configure = idpf_sriov_configure,
380 .remove = idpf_remove,
381 .shutdown = idpf_shutdown,
382 };
383 module_pci_driver(idpf_driver);
384