1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2023 Intel Corporation */
3
4 #include "idpf.h"
5 #include "idpf_devids.h"
6 #include "idpf_lan_vf_regs.h"
7 #include "idpf_virtchnl.h"
8
9 #define DRV_SUMMARY "Intel(R) Infrastructure Data Path Function Linux Driver"
10
11 #define IDPF_NETWORK_ETHERNET_PROGIF 0x01
12 #define IDPF_CLASS_NETWORK_ETHERNET_PROGIF \
13 (PCI_CLASS_NETWORK_ETHERNET << 8 | IDPF_NETWORK_ETHERNET_PROGIF)
14 #define IDPF_VF_TEST_VAL 0xfeed0000u
15
16 MODULE_DESCRIPTION(DRV_SUMMARY);
17 MODULE_IMPORT_NS("LIBETH");
18 MODULE_IMPORT_NS("LIBIE_CP");
19 MODULE_IMPORT_NS("LIBIE_PCI");
20 MODULE_IMPORT_NS("LIBETH_XDP");
21 MODULE_LICENSE("GPL");
22
23 /**
24 * idpf_get_device_type - Helper to find if it is a VF or PF device
25 * @pdev: PCI device information struct
26 *
27 * Return: PF/VF device ID or -%errno on failure.
28 */
idpf_get_device_type(struct pci_dev * pdev)29 static int idpf_get_device_type(struct pci_dev *pdev)
30 {
31 void __iomem *addr;
32 int ret;
33
34 addr = ioremap(pci_resource_start(pdev, 0) + VF_ARQBAL, 4);
35 if (!addr) {
36 pci_err(pdev, "Failed to allocate BAR0 mbx region\n");
37 return -EIO;
38 }
39
40 writel(IDPF_VF_TEST_VAL, addr);
41 if (readl(addr) == IDPF_VF_TEST_VAL)
42 ret = IDPF_DEV_ID_VF;
43 else
44 ret = IDPF_DEV_ID_PF;
45
46 iounmap(addr);
47
48 return ret;
49 }
50
51 /**
52 * idpf_dev_init - Initialize device specific parameters
53 * @adapter: adapter to initialize
54 * @ent: entry in idpf_pci_tbl
55 *
56 * Return: %0 on success, -%errno on failure.
57 */
idpf_dev_init(struct idpf_adapter * adapter,const struct pci_device_id * ent)58 static int idpf_dev_init(struct idpf_adapter *adapter,
59 const struct pci_device_id *ent)
60 {
61 struct libie_mmio_info *mmio_info = &adapter->ctlq_ctx.mmio_info;
62 int ret;
63
64 ret = libie_pci_init_dev(adapter->pdev);
65 if (ret)
66 return ret;
67
68 mmio_info->pdev = adapter->pdev;
69 INIT_LIST_HEAD(&mmio_info->mmio_list);
70
71 if (ent->class == IDPF_CLASS_NETWORK_ETHERNET_PROGIF) {
72 ret = idpf_get_device_type(adapter->pdev);
73 switch (ret) {
74 case IDPF_DEV_ID_VF:
75 idpf_vf_dev_ops_init(adapter);
76 adapter->crc_enable = true;
77 break;
78 case IDPF_DEV_ID_PF:
79 idpf_dev_ops_init(adapter);
80 break;
81 default:
82 return ret;
83 }
84
85 return 0;
86 }
87
88 switch (ent->device) {
89 case IDPF_DEV_ID_PF:
90 idpf_dev_ops_init(adapter);
91 break;
92 case IDPF_DEV_ID_VF:
93 idpf_vf_dev_ops_init(adapter);
94 adapter->crc_enable = true;
95 break;
96 default:
97 return -ENODEV;
98 }
99
100 return 0;
101 }
102
103 /**
104 * idpf_decfg_device - deconfigure device and device specific resources
105 * @adapter: driver specific private structure
106 */
idpf_decfg_device(struct idpf_adapter * adapter)107 static void idpf_decfg_device(struct idpf_adapter *adapter)
108 {
109 struct pci_dev *pdev = adapter->pdev;
110
111 if (pcie_ptm_enabled(pdev))
112 pci_disable_ptm(pdev);
113
114 libie_pci_unmap_all_mmio_regions(&adapter->ctlq_ctx.mmio_info);
115 }
116
117 /**
118 * idpf_remove - Device removal routine
119 * @pdev: PCI device information struct
120 */
idpf_remove(struct pci_dev * pdev)121 static void idpf_remove(struct pci_dev *pdev)
122 {
123 struct idpf_adapter *adapter = pci_get_drvdata(pdev);
124 int i;
125
126 set_bit(IDPF_REMOVE_IN_PROG, adapter->flags);
127
128 /* Wait until vc_event_task is done to consider if any hard reset is
129 * in progress else we may go ahead and release the resources but the
130 * thread doing the hard reset might continue the init path and
131 * end up in bad state.
132 */
133 cancel_delayed_work_sync(&adapter->vc_event_task);
134 if (adapter->num_vfs)
135 idpf_sriov_configure(pdev, 0);
136
137 idpf_vc_core_deinit(adapter);
138
139 /* Be a good citizen and leave the device clean on exit */
140 adapter->dev_ops.reg_ops.trigger_reset(adapter, IDPF_HR_FUNC_RESET);
141 idpf_deinit_dflt_mbx(adapter);
142
143 if (!adapter->netdevs)
144 goto destroy_wqs;
145
146 /* There are some cases where it's possible to still have netdevs
147 * registered with the stack at this point, e.g. if the driver detected
148 * a HW reset and rmmod is called before it fully recovers. Unregister
149 * any stale netdevs here.
150 */
151 for (i = 0; i < adapter->max_vports; i++) {
152 if (!adapter->netdevs[i])
153 continue;
154 if (adapter->netdevs[i]->reg_state != NETREG_UNINITIALIZED)
155 unregister_netdev(adapter->netdevs[i]);
156 free_netdev(adapter->netdevs[i]);
157 adapter->netdevs[i] = NULL;
158 }
159
160 destroy_wqs:
161 destroy_workqueue(adapter->init_wq);
162 destroy_workqueue(adapter->serv_wq);
163 destroy_workqueue(adapter->mbx_wq);
164 destroy_workqueue(adapter->stats_wq);
165 destroy_workqueue(adapter->vc_event_wq);
166
167 for (i = 0; i < adapter->max_vports; i++) {
168 if (!adapter->vport_config[i])
169 continue;
170 kfree(adapter->vport_config[i]->user_config.q_coalesce);
171 kfree(adapter->vport_config[i]);
172 adapter->vport_config[i] = NULL;
173 }
174 kfree(adapter->vport_config);
175 adapter->vport_config = NULL;
176 kfree(adapter->netdevs);
177 adapter->netdevs = NULL;
178
179 mutex_destroy(&adapter->vport_ctrl_lock);
180 mutex_destroy(&adapter->vector_lock);
181 mutex_destroy(&adapter->queue_lock);
182 mutex_destroy(&adapter->vc_buf_lock);
183
184 idpf_decfg_device(adapter);
185 pci_set_drvdata(pdev, NULL);
186 kfree(adapter);
187 }
188
189 /**
190 * idpf_shutdown - PCI callback for shutting down device
191 * @pdev: PCI device information struct
192 */
idpf_shutdown(struct pci_dev * pdev)193 static void idpf_shutdown(struct pci_dev *pdev)
194 {
195 struct idpf_adapter *adapter = pci_get_drvdata(pdev);
196
197 cancel_delayed_work_sync(&adapter->serv_task);
198 cancel_delayed_work_sync(&adapter->vc_event_task);
199 idpf_vc_core_deinit(adapter);
200 idpf_deinit_dflt_mbx(adapter);
201
202 if (system_state == SYSTEM_POWER_OFF)
203 pci_set_power_state(pdev, PCI_D3hot);
204 }
205
206 /**
207 * idpf_cfg_device - configure device and device specific resources
208 * @adapter: driver specific private structure
209 *
210 * Return: %0 on success, -%errno on failure.
211 */
idpf_cfg_device(struct idpf_adapter * adapter)212 static int idpf_cfg_device(struct idpf_adapter *adapter)
213 {
214 struct libie_mmio_info *mmio_info = &adapter->ctlq_ctx.mmio_info;
215 struct pci_dev *pdev = adapter->pdev;
216 struct resource *region;
217 bool mapped;
218 int err;
219
220 /* Map mailbox space for virtchnl communication */
221 region = &adapter->dev_ops.static_reg_info[0];
222 mapped = libie_pci_map_mmio_region(mmio_info, region->start,
223 resource_size(region));
224 if (!mapped) {
225 pci_err(pdev, "failed to map BAR0 mbx region\n");
226 return -ENOMEM;
227 }
228
229 /* Map rstat space for resets */
230 region = &adapter->dev_ops.static_reg_info[1];
231
232 mapped = libie_pci_map_mmio_region(mmio_info, region->start,
233 resource_size(region));
234 if (!mapped) {
235 pci_err(pdev, "failed to map BAR0 rstat region\n");
236 libie_pci_unmap_all_mmio_regions(mmio_info);
237 return -ENOMEM;
238 }
239
240 err = pci_enable_ptm(pdev);
241 if (err)
242 pci_dbg(pdev, "PCIe PTM is not supported by PCIe bus/controller\n");
243
244 pci_set_drvdata(pdev, adapter);
245
246 return 0;
247 }
248
249 /**
250 * idpf_probe - Device initialization routine
251 * @pdev: PCI device information struct
252 * @ent: entry in idpf_pci_tbl
253 *
254 * Returns 0 on success, negative on failure
255 */
idpf_probe(struct pci_dev * pdev,const struct pci_device_id * ent)256 static int idpf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
257 {
258 struct device *dev = &pdev->dev;
259 struct idpf_adapter *adapter;
260 int err;
261
262 adapter = kzalloc_obj(*adapter);
263 if (!adapter)
264 return -ENOMEM;
265
266 adapter->req_tx_splitq = true;
267 adapter->req_rx_splitq = true;
268
269 adapter->pdev = pdev;
270
271 err = idpf_dev_init(adapter, ent);
272 if (err) {
273 dev_err(&pdev->dev, "Failed to initialize device (ID 0x%x): %d\n",
274 ent->device, err);
275 goto err_free;
276 }
277
278 err = idpf_cfg_device(adapter);
279 if (err) {
280 pci_err(pdev, "Failed to configure device specific resources: %pe\n",
281 ERR_PTR(err));
282 goto err_free;
283 }
284
285 adapter->init_wq = alloc_workqueue("%s-%s-init",
286 WQ_UNBOUND | WQ_MEM_RECLAIM, 0,
287 dev_driver_string(dev),
288 dev_name(dev));
289 if (!adapter->init_wq) {
290 dev_err(dev, "Failed to allocate init workqueue\n");
291 err = -ENOMEM;
292 goto err_init_wq;
293 }
294
295 adapter->serv_wq = alloc_workqueue("%s-%s-service",
296 WQ_UNBOUND | WQ_MEM_RECLAIM, 0,
297 dev_driver_string(dev),
298 dev_name(dev));
299 if (!adapter->serv_wq) {
300 dev_err(dev, "Failed to allocate service workqueue\n");
301 err = -ENOMEM;
302 goto err_serv_wq_alloc;
303 }
304
305 adapter->mbx_wq = alloc_workqueue("%s-%s-mbx", WQ_UNBOUND | WQ_HIGHPRI,
306 0, dev_driver_string(dev),
307 dev_name(dev));
308 if (!adapter->mbx_wq) {
309 dev_err(dev, "Failed to allocate mailbox workqueue\n");
310 err = -ENOMEM;
311 goto err_mbx_wq_alloc;
312 }
313
314 adapter->stats_wq = alloc_workqueue("%s-%s-stats",
315 WQ_UNBOUND | WQ_MEM_RECLAIM, 0,
316 dev_driver_string(dev),
317 dev_name(dev));
318 if (!adapter->stats_wq) {
319 dev_err(dev, "Failed to allocate workqueue\n");
320 err = -ENOMEM;
321 goto err_stats_wq_alloc;
322 }
323
324 adapter->vc_event_wq = alloc_workqueue("%s-%s-vc_event",
325 WQ_UNBOUND | WQ_MEM_RECLAIM, 0,
326 dev_driver_string(dev),
327 dev_name(dev));
328 if (!adapter->vc_event_wq) {
329 dev_err(dev, "Failed to allocate virtchnl event workqueue\n");
330 err = -ENOMEM;
331 goto err_vc_event_wq_alloc;
332 }
333
334 /* setup msglvl */
335 adapter->msg_enable = netif_msg_init(-1, IDPF_AVAIL_NETIF_M);
336
337 mutex_init(&adapter->vport_ctrl_lock);
338 mutex_init(&adapter->vector_lock);
339 mutex_init(&adapter->queue_lock);
340 mutex_init(&adapter->vc_buf_lock);
341
342 INIT_DELAYED_WORK(&adapter->init_task, idpf_init_task);
343 INIT_DELAYED_WORK(&adapter->serv_task, idpf_service_task);
344 INIT_DELAYED_WORK(&adapter->mbx_task, idpf_mbx_task);
345 INIT_DELAYED_WORK(&adapter->stats_task, idpf_statistics_task);
346 INIT_DELAYED_WORK(&adapter->vc_event_task, idpf_vc_event_task);
347
348 adapter->dev_ops.reg_ops.reset_reg_init(adapter);
349 set_bit(IDPF_HR_DRV_LOAD, adapter->flags);
350 queue_delayed_work(adapter->vc_event_wq, &adapter->vc_event_task,
351 msecs_to_jiffies(10 * (pdev->devfn & 0x07)));
352
353 return 0;
354
355 err_vc_event_wq_alloc:
356 destroy_workqueue(adapter->stats_wq);
357 err_stats_wq_alloc:
358 destroy_workqueue(adapter->mbx_wq);
359 err_mbx_wq_alloc:
360 destroy_workqueue(adapter->serv_wq);
361 err_serv_wq_alloc:
362 destroy_workqueue(adapter->init_wq);
363 err_init_wq:
364 idpf_decfg_device(adapter);
365 err_free:
366 kfree(adapter);
367 return err;
368 }
369
370 /* idpf_pci_tbl - PCI Dev idpf ID Table
371 */
372 static const struct pci_device_id idpf_pci_tbl[] = {
373 { PCI_VDEVICE(INTEL, IDPF_DEV_ID_PF)},
374 { PCI_VDEVICE(INTEL, IDPF_DEV_ID_VF)},
375 { PCI_DEVICE_CLASS(IDPF_CLASS_NETWORK_ETHERNET_PROGIF, ~0)},
376 { /* Sentinel */ }
377 };
378 MODULE_DEVICE_TABLE(pci, idpf_pci_tbl);
379
380 static struct pci_driver idpf_driver = {
381 .name = KBUILD_MODNAME,
382 .id_table = idpf_pci_tbl,
383 .probe = idpf_probe,
384 .sriov_configure = idpf_sriov_configure,
385 .remove = idpf_remove,
386 .shutdown = idpf_shutdown,
387 };
388 module_pci_driver(idpf_driver);
389