1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <unistd.h>
3 #include <errno.h>
4 #include <stdint.h>
5 #include <linux/io.h>
6 #include <linux/pci_regs.h>
7 #include <linux/pci_ids.h>
8 #include <linux/kernel.h>
9 #include <linux/compiler.h>
10 #include <asm/barrier.h>
11 #include <linux/mii.h>
12 #include <libvfio/vfio_pci_device.h>
13
14 #include "e1000_regs.h"
15 #include "e1000_defines.h"
16 #include "e1000_82575.h"
17
18 #define PCI_DEVICE_ID_INTEL_82576 0x10C9
19 #define IGB_MAX_CHUNK_SIZE 1024
20 #define MSIX_VECTOR 0
21 #define MSIX_VECTOR_MASK (1 << MSIX_VECTOR)
22 #define RING_SIZE 4096 /* Number of descriptors in ring */
23
24 struct igb_tx_desc {
25 union {
26 struct {
27 u64 buffer_addr; /* Address of descriptor's data buffer */
28 u32 cmd_type_len; /* Command/Type/Length */
29 u32 olinfo_status; /* Context/Buffer info */
30 } read;
31
32 struct {
33 u64 rsvd; /* Reserved */
34 u32 nxtseq_seed; /* Next sequence seed */
35 u32 status; /* Descriptor status */
36 } wb;
37 };
38 };
39
40 struct igb_rx_desc {
41 union {
42 struct {
43 u64 pkt_addr; /* Packet buffer address */
44 u64 hdr_addr; /* Header buffer address */
45 } read;
46 struct {
47 u16 pkt_info; /* RSS type, Packet type */
48 u16 hdr_info; /* Split Head, buf len */
49 u32 rss; /* RSS Hash */
50 u32 status_error; /* ext status/error */
51 u16 length; /* Packet length */
52 u16 vlan; /* VLAN tag */
53 } wb; /* writeback */
54 };
55 };
56
57 struct igb {
58 void *bar0;
59 u32 tx_tail;
60 u32 rx_tail;
61 struct igb_tx_desc tx_ring[RING_SIZE] __attribute__((aligned(128)));
62 struct igb_rx_desc rx_ring[RING_SIZE] __attribute__((aligned(128)));
63 };
64
to_igb_state(struct vfio_pci_device * device)65 static inline struct igb *to_igb_state(struct vfio_pci_device *device)
66 {
67 return (struct igb *)device->driver.region.vaddr;
68 }
69
igb_write32(struct igb * igb,u32 reg,u32 val)70 static inline void igb_write32(struct igb *igb, u32 reg, u32 val)
71 {
72 writel(val, igb->bar0 + reg);
73 }
74
igb_read32(struct igb * igb,u32 reg)75 static inline u32 igb_read32(struct igb *igb, u32 reg)
76 {
77 return readl(igb->bar0 + reg);
78 }
79
igb_write_phy(struct igb * igb,u32 offset,u16 data)80 static int igb_write_phy(struct igb *igb, u32 offset, u16 data)
81 {
82 u32 mdic;
83 int i;
84
85 /*
86 * Write a PHY register over MDIO.
87 *
88 * A production driver would hold the SW/FW semaphore (SWSM.SWESMBI + the
89 * SW_FW_SYNC PHY bit) across the MDIO transaction to serialize against the
90 * device's management firmware. The selftest owns the assigned function
91 * exclusively on a dedicated test device with no active manageability
92 * contending for the PHY, so the sync is omitted; it should be added here
93 * if this ever needs to run on a manageability-enabled NIC.
94 */
95 mdic = (((u32)data) |
96 (offset << E1000_MDIC_REG_SHIFT) |
97 (1 << E1000_MDIC_PHY_SHIFT) |
98 E1000_MDIC_OP_WRITE);
99
100 igb_write32(igb, E1000_MDIC, mdic);
101
102 for (i = 0; i < 1000; i++) {
103 usleep(50);
104 mdic = igb_read32(igb, E1000_MDIC);
105 if (mdic & E1000_MDIC_READY)
106 break;
107 }
108
109 if (!(mdic & E1000_MDIC_READY))
110 return -1;
111
112 if (mdic & E1000_MDIC_ERROR)
113 return -1;
114
115 return 0;
116 }
117
118 /*
119 * Configure the device for PHY internal loopback per 82576 datasheet
120 * section 3.5.6.3.1. Force the PHY to 1Gb/s full duplex with loopback
121 * enabled, then force the MAC link state to match. Internal loopback
122 * wraps data at the end of the PHY datapath (section 3.5.6.3), so the
123 * physical link state is irrelevant.
124 *
125 * Section 3.5.6.1 directs to "Use PHY Loopback instead of MAC Loopback
126 * on the 82576", and section 3.5.6.2 states "MAC Loopback is not used
127 * on this device." RCTL.LBM_MAC is still set elsewhere as a QEMU-only
128 * accommodation; see the RCTL programming in the caller for the
129 * rationale.
130 */
igb_setup_loopback(struct igb * igb)131 static void igb_setup_loopback(struct igb *igb)
132 {
133 u32 ctrl;
134 int ret;
135
136 /*
137 * Kick the autoneg machinery solely to bring STATUS.LU up under
138 * QEMU's igb emulation: QEMU only updates STATUS.LU via its
139 * autoneg-done timer, and without LU set its receive path
140 * (e1000x_hw_rx_enabled) drops every loopback frame. On real
141 * hardware autoneg cannot complete before the next PHY write
142 * below clears the autoneg-enable bit, so this is effectively a
143 * no-op there.
144 */
145 (void)igb_write_phy(igb, MII_BMCR,
146 BMCR_ANENABLE | BMCR_ANRESTART);
147
148 /* PHY control: loopback + 1Gb/s full duplex, autoneg disabled. */
149 ret = igb_write_phy(igb, MII_BMCR,
150 BMCR_LOOPBACK |
151 BMCR_SPEED1000 |
152 BMCR_FULLDPLX);
153 VFIO_ASSERT_EQ(ret, 0, "Failed to write PHY control register");
154
155 /*
156 * Brief delay before forcing the MAC, mirroring the kernel ethtool
157 * selftest in igb_integrated_phy_loopback(). Not specified by the
158 * datasheet, but empirically required by the kernel driver.
159 */
160 usleep(50000);
161
162 /*
163 * Force the MAC to 1Gb/s full duplex with link up. Without forcing
164 * the link state the descriptor engine does not run, since the chip
165 * normally waits for a real negotiated link.
166 */
167 ctrl = igb_read32(igb, E1000_CTRL);
168 ctrl &= ~E1000_CTRL_SPD_SEL;
169 ctrl |= E1000_CTRL_FRCSPD |
170 E1000_CTRL_FRCDPX |
171 E1000_CTRL_SPD_1000 |
172 E1000_CTRL_FD |
173 E1000_CTRL_SLU;
174 igb_write32(igb, E1000_CTRL, ctrl);
175
176 /*
177 * Settling delay matching the kernel ethtool selftest's msleep(500)
178 * at the tail of igb_integrated_phy_loopback(). Not specified by
179 * the datasheet; empirical, and inherited from the kernel driver.
180 */
181 usleep(500000);
182 }
183
igb_probe(struct vfio_pci_device * device)184 static int igb_probe(struct vfio_pci_device *device)
185 {
186 if (!vfio_pci_device_match(device, PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82576))
187 return -EINVAL;
188
189 return 0;
190 }
191
igb_reset(struct igb * igb)192 static void igb_reset(struct igb *igb)
193 {
194 int retries = 20;
195
196 igb_write32(igb, E1000_CTRL, igb_read32(igb, E1000_CTRL) | E1000_CTRL_RST);
197 /*
198 * Must wait at least 1 millisecond after setting the reset bit before
199 * checking if this device is ready to be used (82576 datasheet section
200 * 4.2.1.6.1). The delay also ensures the reset has taken effect and
201 * cleared EECD.AUTO_RD before it is polled below.
202 */
203 usleep(1000);
204
205 /*
206 * Poll NVM Auto Read Done rather than CTRL.RST, matching
207 * igb_get_auto_rd_done() in the igb driver: AUTO_RD implies both that
208 * the reset completed and that the device finished re-reading its
209 * configuration from NVM, which is what actually makes it usable.
210 */
211 while (retries-- > 0 && !(igb_read32(igb, E1000_EECD) & E1000_EECD_AUTO_RD))
212 usleep(1000);
213
214 /*
215 * QEMU's igb emulation does not set E1000_EECD_AUTO_RD. If we timed out,
216 * check if CTRL.RST is cleared, which is what QEMU uses to signal reset
217 * completion.
218 */
219 if (retries < 0) {
220 VFIO_ASSERT_EQ(igb_read32(igb, E1000_CTRL) & E1000_CTRL_RST, 0,
221 "Device reset did not complete (CTRL.RST not cleared)");
222 }
223
224 igb_write32(igb, E1000_IMC, 0xFFFFFFFF);
225 }
226
227 /*
228 * Program the device into a usable state. Split out of igb_init() so it
229 * can be reused after a device reset to re-program the registers that
230 * CTRL.RST clears. Expects bar0 to be mapped and MSI-X already enabled
231 * via VFIO.
232 */
igb_hw_init(struct vfio_pci_device * device)233 static void igb_hw_init(struct vfio_pci_device *device)
234 {
235 struct igb *igb = to_igb_state(device);
236 u64 iova_tx, iova_rx;
237 u32 ctrl, rctl;
238 u16 cmd_reg;
239 int retries;
240
241 iova_tx = to_iova(device, igb->tx_ring);
242 iova_rx = to_iova(device, igb->rx_ring);
243
244
245
246 /* Signal that the driver is loaded */
247 ctrl = igb_read32(igb, E1000_CTRL_EXT);
248 ctrl |= E1000_CTRL_EXT_DRV_LOAD;
249 ctrl &= ~E1000_CTRL_EXT_LINK_MODE_MASK;
250 igb_write32(igb, E1000_CTRL_EXT, ctrl);
251
252 /* Enable PCI Bus Master. */
253 cmd_reg = vfio_pci_config_readw(device, PCI_COMMAND);
254 if ((cmd_reg & (PCI_COMMAND_MASTER | PCI_COMMAND_MEMORY)) !=
255 (PCI_COMMAND_MASTER | PCI_COMMAND_MEMORY)) {
256 cmd_reg |= (PCI_COMMAND_MASTER | PCI_COMMAND_MEMORY);
257 vfio_pci_config_writew(device, PCI_COMMAND, cmd_reg);
258 }
259
260 /* Configure PHY internal loopback for testing. */
261 igb_setup_loopback(igb);
262
263 /*
264 * Disable DMA re-send on PCIe completion timeout (82576 datasheet
265 * section 8.6.1, GCR.Completion_Timeout_Resend, bit 16). The
266 * mix_and_match test intentionally submits descriptors targeting
267 * unmapped IOVAs; with the default (set) value, the device keeps
268 * retrying the failed read indefinitely, which keeps PCIe AER and
269 * IOMMU error handling busy and interferes with reset recovery.
270 */
271 ctrl = igb_read32(igb, E1000_GCR);
272 ctrl &= ~E1000_GCR_CMPL_TMOUT_RESEND;
273 igb_write32(igb, E1000_GCR, ctrl);
274
275 /* Configure TX and RX descriptor rings */
276 igb_write32(igb, E1000_TDBAL(0), (u32)iova_tx);
277 igb_write32(igb, E1000_TDBAH(0), (u32)(iova_tx >> 32));
278 igb_write32(igb, E1000_TDLEN(0), RING_SIZE * sizeof(struct igb_tx_desc));
279 igb_write32(igb, E1000_TDH(0), 0);
280 igb_write32(igb, E1000_TDT(0), 0);
281 igb_write32(igb, E1000_TXDCTL(0), E1000_TXDCTL_QUEUE_ENABLE);
282
283 igb_write32(igb, E1000_RDBAL(0), (u32)iova_rx);
284 igb_write32(igb, E1000_RDBAH(0), (u32)(iova_rx >> 32));
285 igb_write32(igb, E1000_RDLEN(0), RING_SIZE * sizeof(struct igb_rx_desc));
286 igb_write32(igb, E1000_RDH(0), 0);
287 igb_write32(igb, E1000_RDT(0), 0);
288
289 /*
290 * Select the advanced one-buffer descriptor format. Per 82576
291 * datasheet section 7.1.5.2: "SRRCTL[n].DESCTYPE must be set to a
292 * value other than 000b for the 82576 to write back the special
293 * descriptors." struct igb_rx_desc matches the advanced one-buffer
294 * writeback layout (section 7.1.5.2), so polling rx.wb.status_error
295 * requires this format. Section 8.10.2 specifies DESCTYPE[27:25].
296 *
297 * The direct write also zeroes SRRCTL.BSIZEPACKET, which is
298 * intentional: per section 7.1.3.1 a zero BSIZEPACKET falls back to
299 * the RCTL.BSIZE buffer size, whose reset default (00b) is 2048
300 * bytes -- ample for the loopback frames here.
301 */
302 igb_write32(igb, E1000_SRRCTL(0), E1000_SRRCTL_DESCTYPE_ADV_ONEBUF);
303
304 igb_write32(igb, E1000_RXDCTL(0), E1000_RXDCTL_QUEUE_ENABLE);
305
306 /*
307 * Enable Receiver and Transmitter. RCTL.LBM_MAC is set in addition
308 * to PHY loopback as a QEMU-only accommodation: QEMU's emulated igb
309 * does not honor PHY register 0 bit 14 (PHY internal loopback) and
310 * relies on RCTL.LBM_MAC to wrap TX descriptors back to the RX
311 * queue. Datasheet 8.10.1 (RCTL register) advises "When using the
312 * internal PHY, LBM should remain set to 00b", so setting LBM_MAC
313 * here deviates from datasheet guidance; empirically the bit has
314 * no observable effect on real 82576 hardware because MAC loopback
315 * is not implemented (datasheet 3.5.6.2). Setting both lets the
316 * selftest work on both real hardware and QEMU without conditional
317 * code paths.
318 */
319 rctl = E1000_RCTL_EN | /* Receiver Enable */
320 E1000_RCTL_UPE | /* Unicast Promiscuous (for dummy MAC) */
321 E1000_RCTL_MPE | /* Multicast Promiscuous */
322 E1000_RCTL_BAM | /* Broadcast Accept Mode */
323 E1000_RCTL_LBM_MAC | /* MAC Loopback - for QEMU emulation only */
324 E1000_RCTL_SECRC; /* Strip CRC (needed for memcmp) */
325 igb_write32(igb, E1000_RCTL, rctl);
326 igb_write32(igb, E1000_TCTL, E1000_TCTL_EN | E1000_TCTL_PSP);
327
328 /*
329 * Wait for TX and RX queues to be enabled. Per the RXDCTL/TXDCTL
330 * register definitions (8.10.10/8.12.13), the per-queue enable bit
331 * "remains zero" until the global RCTL.RXEN/TCTL.TXEN are set, so
332 * E1000_RCTL_EN and E1000_TCTL_EN must already be written above.
333 */
334 retries = 2000;
335 while (retries-- > 0) {
336 if ((igb_read32(igb, E1000_TXDCTL(0)) & E1000_TXDCTL_QUEUE_ENABLE) &&
337 (igb_read32(igb, E1000_RXDCTL(0)) & E1000_RXDCTL_QUEUE_ENABLE))
338 break;
339 usleep(10);
340 }
341 VFIO_ASSERT_GE(retries, 0);
342
343 /*
344 * Program MSI-X interrupt routing per 82576 datasheet:
345 *
346 * GPIE (section 7.3.2.11, Table 7-47): set Multiple_MSIX (bit 4) to
347 * route interrupt causes through IVAR mapping, and EIAME (bit 30)
348 * to apply EIAM on MSI-X assertion (without EIAME, EIAM only
349 * applies on EICR read/write).
350 *
351 * EIAC (section 8.8.5): enable auto-clear of EICR for vector 0.
352 * Without auto-clear the cause stays set after delivery and the
353 * test can see spurious interrupts on the next memcpy batch.
354 *
355 * EIAM (section 8.8.6): enable auto-mask of EIMS for vector 0 on
356 * MSI-X assertion (effective because EIAME is set).
357 *
358 * IVAR (section 7.3.1.2, register definition in 8.8.13): map RX
359 * cause 0 to MSI-X vector 0 and mark the entry valid.
360 */
361 igb_write32(igb, E1000_GPIE, E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME);
362 igb_write32(igb, E1000_EIAC, MSIX_VECTOR_MASK);
363 igb_write32(igb, E1000_EIAM, MSIX_VECTOR_MASK);
364
365 /* Map vector 0 to interrupt cause 0 and mark it valid */
366 igb_write32(igb, E1000_IVAR0, E1000_IVAR_VALID);
367
368 /* Enable interrupts on vector 0 */
369 igb_write32(igb, E1000_EIMS, MSIX_VECTOR_MASK);
370
371 /* Initialize driver state and capability limits */
372 igb->tx_tail = 0;
373 igb->rx_tail = 0;
374
375 device->driver.max_memcpy_size = IGB_MAX_CHUNK_SIZE;
376 device->driver.max_memcpy_count = RING_SIZE - 1;
377 device->driver.msi = MSIX_VECTOR;
378 }
379
igb_init(struct vfio_pci_device * device)380 static void igb_init(struct vfio_pci_device *device)
381 {
382 struct igb *igb = to_igb_state(device);
383
384 VFIO_ASSERT_GE(device->driver.region.size, sizeof(struct igb));
385
386 igb->bar0 = device->bars[0].vaddr;
387
388 igb_reset(igb);
389
390 /*
391 * Enable MSI-X via VFIO before device-side register programming.
392 * vfio_pci_msix_enable() only touches the VFIO IRQ machinery and the
393 * PCI MSI-X capability via config space; it has no ordering
394 * dependency on the device-side writes performed by igb_hw_init().
395 * Placing it here keeps igb_hw_init() reusable from the reset
396 * recovery path (which calls vfio_pci_irq_reenable() instead).
397 */
398 vfio_pci_msix_enable(device, MSIX_VECTOR, 1);
399
400 igb_hw_init(device);
401 }
402
igb_remove(struct vfio_pci_device * device)403 static void igb_remove(struct vfio_pci_device *device)
404 {
405 struct igb *igb = to_igb_state(device);
406
407 igb_write32(igb, E1000_RCTL, 0);
408 igb_write32(igb, E1000_TCTL, 0);
409 igb_reset(igb);
410
411 vfio_pci_msix_disable(device);
412 }
413
igb_irq_disable(struct igb * igb)414 static void igb_irq_disable(struct igb *igb)
415 {
416 igb_write32(igb, E1000_EIMC, MSIX_VECTOR_MASK);
417 }
418
igb_irq_enable(struct igb * igb)419 static void igb_irq_enable(struct igb *igb)
420 {
421 igb_write32(igb, E1000_EIMS, MSIX_VECTOR_MASK);
422 }
423
igb_irq_clear(struct igb * igb)424 static void igb_irq_clear(struct igb *igb)
425 {
426 /*
427 * Use write-to-clear (datasheet 7.3.4.2). In MSI-X mode with EIAC
428 * programmed, section 8.8.5 explicitly states "If any bits are set
429 * in EIAC, the EICR register should not be read", which rules out
430 * the read-to-clear path in 7.3.4.3. Bits not in EIAC are still
431 * cleared by writing 1.
432 */
433 igb_write32(igb, E1000_EICR, 0xFFFFFFFF);
434 }
435
igb_memcpy_start(struct vfio_pci_device * device,iova_t src,iova_t dst,u64 size,u64 count)436 static void igb_memcpy_start(struct vfio_pci_device *device, iova_t src,
437 iova_t dst, u64 size, u64 count)
438 {
439 struct igb *igb = to_igb_state(device);
440 struct igb_rx_desc *rx;
441 struct igb_tx_desc *tx;
442 u32 i;
443
444 VFIO_ASSERT_GE(size, 60,
445 "IGB driver requires memcpy size to be at least 60 bytes (Ethernet minimum payload size)");
446
447 igb_irq_disable(igb);
448
449 for (i = 0; i < count; i++) {
450 tx = &igb->tx_ring[igb->tx_tail];
451 rx = &igb->rx_ring[igb->rx_tail];
452
453 memset(tx, 0, sizeof(struct igb_tx_desc));
454 memset(rx, 0, sizeof(struct igb_rx_desc));
455
456 rx->read.pkt_addr = cpu_to_le64(dst);
457 rx->read.hdr_addr = cpu_to_le64(0);
458
459 tx->read.buffer_addr = cpu_to_le64(src);
460 /*
461 * Build an advanced data descriptor per 82576 datasheet
462 * section 7.2.2.3. DEXT marks the descriptor as advanced
463 * (required by hardware); DTYP=data selects the data
464 * descriptor; IFCS asks the MAC to append the Ethernet
465 * FCS (without it the frame is dropped as malformed);
466 * EOP marks end of packet. DTALEN is the buffer length
467 * in bits 15:0 of cmd_type_len.
468 */
469 tx->read.cmd_type_len = cpu_to_le32((uint32_t)size |
470 E1000_ADVTXD_DTYP_DATA |
471 E1000_ADVTXD_DCMD_DEXT |
472 E1000_ADVTXD_DCMD_IFCS |
473 E1000_ADVTXD_DCMD_EOP);
474 /*
475 * PAYLEN (section 7.2.2.3.11) is the total payload size
476 * in olinfo_status[31:14].
477 */
478 tx->read.olinfo_status =
479 cpu_to_le32((uint32_t)size << E1000_ADVTXD_PAYLEN_SHIFT);
480
481 igb->tx_tail = (igb->tx_tail + 1) % RING_SIZE;
482 igb->rx_tail = (igb->rx_tail + 1) % RING_SIZE;
483 }
484
485 igb_write32(igb, E1000_RDT(0), igb->rx_tail);
486 igb_write32(igb, E1000_TDT(0), igb->tx_tail);
487 }
488
489 /*
490 * Reset the device via VFIO_DEVICE_RESET (PCIe FLR on the 82576) and
491 * re-program it. VFIO_DEVICE_RESET tears down the kernel-side MSI-X
492 * trigger but leaves user-side eventfds intact, so re-arm the trigger
493 * via vfio_pci_irq_reenable() before reprogramming so any caller-cached
494 * eventfd remains valid.
495 *
496 * FLR clears device-side state to power-on reset values (datasheet
497 * 4.2.1.5.1: a PF FLR is "equivalent to a D0->D3->D0 transition"), so
498 * EIMS and EICR come back as 0 from their register-defined initial
499 * values, and igb_hw_init() resets tx_tail/rx_tail to 0. The next
500 * igb_memcpy_start() will memset each descriptor it touches before
501 * submission, so no explicit IMC/EICR writes or ring memsets are
502 * needed here.
503 */
igb_error_reset_and_reinit(struct vfio_pci_device * device)504 static void igb_error_reset_and_reinit(struct vfio_pci_device *device)
505 {
506 vfio_pci_device_reset(device);
507 vfio_pci_msix_reenable(device, MSIX_VECTOR, 1);
508 igb_hw_init(device);
509 }
510
igb_memcpy_wait(struct vfio_pci_device * device)511 static int igb_memcpy_wait(struct vfio_pci_device *device)
512 {
513 struct igb *igb = to_igb_state(device);
514 struct igb_rx_desc *rx;
515 u32 status = 0;
516 u32 prev_tail;
517 int retries;
518
519 prev_tail = (igb->rx_tail + RING_SIZE - 1) % RING_SIZE;
520 rx = &igb->rx_ring[prev_tail];
521
522 /*
523 * Real 82576 hardware processes the descriptor ring at line rate.
524 * max_memcpy_size = (RING_SIZE - 1) * IGB_MAX_CHUNK_SIZE ~= 4 MB,
525 * split into 4095 1 KB frames. At 1 Gb/s (~125 MB/s) the worst
526 * valid memcpy takes ~32 ms on the wire, plus per-frame preamble,
527 * SFD, IFG and FCS overhead (~3%) and descriptor fetch/writeback
528 * latency. Wait up to ~200 ms before declaring the device hung;
529 * ~6x the line-rate floor leaves comfortable headroom for host
530 * scheduling jitter while keeping the intentional invalid-DMA
531 * tests bounded.
532 */
533 retries = 200;
534 while (retries-- > 0) {
535 status = le32_to_cpu(READ_ONCE(rx->wb.status_error));
536 if (status & 1)
537 break;
538 usleep(1000);
539 }
540
541 if (status & 1)
542 /*
543 * Ensure the test code doesn't speculatively read the DMA
544 * destination buffer before we have verified that the
545 * descriptor writeback is complete.
546 */
547 rmb();
548
549 igb_irq_clear(igb);
550
551 igb_irq_enable(igb);
552
553 if (status & 1)
554 return 0;
555
556 /*
557 * The descriptor never completed. On real 82576 hardware this
558 * typically follows a DMA-read fault from one of the intentional
559 * unmapped-IOVA tests; the fault leaves the descriptor engine
560 * unable to service subsequent valid descriptors. CTRL.RST alone
561 * reinitializes the queue registers but leaves the engine wedged
562 * for the current process, so a broader VFIO_DEVICE_RESET (FLR)
563 * is required.
564 */
565 igb_error_reset_and_reinit(device);
566
567 return -ETIMEDOUT;
568 }
569
igb_send_msi(struct vfio_pci_device * device)570 static void igb_send_msi(struct vfio_pci_device *device)
571 {
572 struct igb *igb = to_igb_state(device);
573
574 igb_write32(igb, E1000_EICS, MSIX_VECTOR_MASK);
575 }
576
577 const struct vfio_pci_driver_ops igb_ops = {
578 .name = "igb",
579 .probe = igb_probe,
580 .init = igb_init,
581 .remove = igb_remove,
582 .memcpy_start = igb_memcpy_start,
583 .memcpy_wait = igb_memcpy_wait,
584 .send_msi = igb_send_msi,
585 };
586