1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <unistd.h> 3 #include <errno.h> 4 #include <stdint.h> 5 #include <linux/io.h> 6 #include <linux/pci_regs.h> 7 #include <linux/pci_ids.h> 8 #include <linux/kernel.h> 9 #include <linux/compiler.h> 10 #include <asm/barrier.h> 11 #include <linux/mii.h> 12 #include <libvfio/vfio_pci_device.h> 13 14 #include "e1000_regs.h" 15 #include "e1000_defines.h" 16 #include "e1000_82575.h" 17 18 #define PCI_DEVICE_ID_INTEL_82576 0x10C9 19 #define IGB_MAX_CHUNK_SIZE 1024 20 #define MSIX_VECTOR 0 21 #define MSIX_VECTOR_MASK (1 << MSIX_VECTOR) 22 #define RING_SIZE 4096 /* Number of descriptors in ring */ 23 24 struct igb_tx_desc { 25 union { 26 struct { 27 u64 buffer_addr; /* Address of descriptor's data buffer */ 28 u32 cmd_type_len; /* Command/Type/Length */ 29 u32 olinfo_status; /* Context/Buffer info */ 30 } read; 31 32 struct { 33 u64 rsvd; /* Reserved */ 34 u32 nxtseq_seed; /* Next sequence seed */ 35 u32 status; /* Descriptor status */ 36 } wb; 37 }; 38 }; 39 40 struct igb_rx_desc { 41 union { 42 struct { 43 u64 pkt_addr; /* Packet buffer address */ 44 u64 hdr_addr; /* Header buffer address */ 45 } read; 46 struct { 47 u16 pkt_info; /* RSS type, Packet type */ 48 u16 hdr_info; /* Split Head, buf len */ 49 u32 rss; /* RSS Hash */ 50 u32 status_error; /* ext status/error */ 51 u16 length; /* Packet length */ 52 u16 vlan; /* VLAN tag */ 53 } wb; /* writeback */ 54 }; 55 }; 56 57 struct igb { 58 void *bar0; 59 u32 tx_tail; 60 u32 rx_tail; 61 struct igb_tx_desc tx_ring[RING_SIZE] __attribute__((aligned(128))); 62 struct igb_rx_desc rx_ring[RING_SIZE] __attribute__((aligned(128))); 63 }; 64 65 static inline struct igb *to_igb_state(struct vfio_pci_device *device) 66 { 67 return (struct igb *)device->driver.region.vaddr; 68 } 69 70 static inline void igb_write32(struct igb *igb, u32 reg, u32 val) 71 { 72 writel(val, igb->bar0 + reg); 73 } 74 75 static inline u32 igb_read32(struct igb *igb, u32 reg) 76 { 77 return readl(igb->bar0 + reg); 78 } 79 80 static int igb_write_phy(struct igb *igb, u32 offset, u16 data) 81 { 82 u32 mdic; 83 int i; 84 85 /* 86 * Write a PHY register over MDIO. 87 * 88 * A production driver would hold the SW/FW semaphore (SWSM.SWESMBI + the 89 * SW_FW_SYNC PHY bit) across the MDIO transaction to serialize against the 90 * device's management firmware. The selftest owns the assigned function 91 * exclusively on a dedicated test device with no active manageability 92 * contending for the PHY, so the sync is omitted; it should be added here 93 * if this ever needs to run on a manageability-enabled NIC. 94 */ 95 mdic = (((u32)data) | 96 (offset << E1000_MDIC_REG_SHIFT) | 97 (1 << E1000_MDIC_PHY_SHIFT) | 98 E1000_MDIC_OP_WRITE); 99 100 igb_write32(igb, E1000_MDIC, mdic); 101 102 for (i = 0; i < 1000; i++) { 103 usleep(50); 104 mdic = igb_read32(igb, E1000_MDIC); 105 if (mdic & E1000_MDIC_READY) 106 break; 107 } 108 109 if (!(mdic & E1000_MDIC_READY)) 110 return -1; 111 112 if (mdic & E1000_MDIC_ERROR) 113 return -1; 114 115 return 0; 116 } 117 118 /* 119 * Configure the device for PHY internal loopback per 82576 datasheet 120 * section 3.5.6.3.1. Force the PHY to 1Gb/s full duplex with loopback 121 * enabled, then force the MAC link state to match. Internal loopback 122 * wraps data at the end of the PHY datapath (section 3.5.6.3), so the 123 * physical link state is irrelevant. 124 * 125 * Section 3.5.6.1 directs to "Use PHY Loopback instead of MAC Loopback 126 * on the 82576", and section 3.5.6.2 states "MAC Loopback is not used 127 * on this device." RCTL.LBM_MAC is still set elsewhere as a QEMU-only 128 * accommodation; see the RCTL programming in the caller for the 129 * rationale. 130 */ 131 static void igb_setup_loopback(struct igb *igb) 132 { 133 u32 ctrl; 134 int ret; 135 136 /* 137 * Kick the autoneg machinery solely to bring STATUS.LU up under 138 * QEMU's igb emulation: QEMU only updates STATUS.LU via its 139 * autoneg-done timer, and without LU set its receive path 140 * (e1000x_hw_rx_enabled) drops every loopback frame. On real 141 * hardware autoneg cannot complete before the next PHY write 142 * below clears the autoneg-enable bit, so this is effectively a 143 * no-op there. 144 */ 145 (void)igb_write_phy(igb, MII_BMCR, 146 BMCR_ANENABLE | BMCR_ANRESTART); 147 148 /* PHY control: loopback + 1Gb/s full duplex, autoneg disabled. */ 149 ret = igb_write_phy(igb, MII_BMCR, 150 BMCR_LOOPBACK | 151 BMCR_SPEED1000 | 152 BMCR_FULLDPLX); 153 VFIO_ASSERT_EQ(ret, 0, "Failed to write PHY control register"); 154 155 /* 156 * Brief delay before forcing the MAC, mirroring the kernel ethtool 157 * selftest in igb_integrated_phy_loopback(). Not specified by the 158 * datasheet, but empirically required by the kernel driver. 159 */ 160 usleep(50000); 161 162 /* 163 * Force the MAC to 1Gb/s full duplex with link up. Without forcing 164 * the link state the descriptor engine does not run, since the chip 165 * normally waits for a real negotiated link. 166 */ 167 ctrl = igb_read32(igb, E1000_CTRL); 168 ctrl &= ~E1000_CTRL_SPD_SEL; 169 ctrl |= E1000_CTRL_FRCSPD | 170 E1000_CTRL_FRCDPX | 171 E1000_CTRL_SPD_1000 | 172 E1000_CTRL_FD | 173 E1000_CTRL_SLU; 174 igb_write32(igb, E1000_CTRL, ctrl); 175 176 /* 177 * Settling delay matching the kernel ethtool selftest's msleep(500) 178 * at the tail of igb_integrated_phy_loopback(). Not specified by 179 * the datasheet; empirical, and inherited from the kernel driver. 180 */ 181 usleep(500000); 182 } 183 184 static int igb_probe(struct vfio_pci_device *device) 185 { 186 if (!vfio_pci_device_match(device, PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82576)) 187 return -EINVAL; 188 189 return 0; 190 } 191 192 static void igb_reset(struct igb *igb) 193 { 194 int retries = 20; 195 196 igb_write32(igb, E1000_CTRL, igb_read32(igb, E1000_CTRL) | E1000_CTRL_RST); 197 /* 198 * Must wait at least 1 millisecond after setting the reset bit before 199 * checking if this device is ready to be used (82576 datasheet section 200 * 4.2.1.6.1). The delay also ensures the reset has taken effect and 201 * cleared EECD.AUTO_RD before it is polled below. 202 */ 203 usleep(1000); 204 205 /* 206 * Poll NVM Auto Read Done rather than CTRL.RST, matching 207 * igb_get_auto_rd_done() in the igb driver: AUTO_RD implies both that 208 * the reset completed and that the device finished re-reading its 209 * configuration from NVM, which is what actually makes it usable. 210 */ 211 while (retries-- > 0 && !(igb_read32(igb, E1000_EECD) & E1000_EECD_AUTO_RD)) 212 usleep(1000); 213 214 /* 215 * QEMU's igb emulation does not set E1000_EECD_AUTO_RD. If we timed out, 216 * check if CTRL.RST is cleared, which is what QEMU uses to signal reset 217 * completion. 218 */ 219 if (retries < 0) { 220 VFIO_ASSERT_EQ(igb_read32(igb, E1000_CTRL) & E1000_CTRL_RST, 0, 221 "Device reset did not complete (CTRL.RST not cleared)"); 222 } 223 224 igb_write32(igb, E1000_IMC, 0xFFFFFFFF); 225 } 226 227 /* 228 * Program the device into a usable state. Split out of igb_init() so it 229 * can be reused after a device reset to re-program the registers that 230 * CTRL.RST clears. Expects bar0 to be mapped and MSI-X already enabled 231 * via VFIO. 232 */ 233 static void igb_hw_init(struct vfio_pci_device *device) 234 { 235 struct igb *igb = to_igb_state(device); 236 u64 iova_tx, iova_rx; 237 u32 ctrl, rctl; 238 u16 cmd_reg; 239 int retries; 240 241 iova_tx = to_iova(device, igb->tx_ring); 242 iova_rx = to_iova(device, igb->rx_ring); 243 244 245 246 /* Signal that the driver is loaded */ 247 ctrl = igb_read32(igb, E1000_CTRL_EXT); 248 ctrl |= E1000_CTRL_EXT_DRV_LOAD; 249 ctrl &= ~E1000_CTRL_EXT_LINK_MODE_MASK; 250 igb_write32(igb, E1000_CTRL_EXT, ctrl); 251 252 /* Enable PCI Bus Master. */ 253 cmd_reg = vfio_pci_config_readw(device, PCI_COMMAND); 254 if ((cmd_reg & (PCI_COMMAND_MASTER | PCI_COMMAND_MEMORY)) != 255 (PCI_COMMAND_MASTER | PCI_COMMAND_MEMORY)) { 256 cmd_reg |= (PCI_COMMAND_MASTER | PCI_COMMAND_MEMORY); 257 vfio_pci_config_writew(device, PCI_COMMAND, cmd_reg); 258 } 259 260 /* Configure PHY internal loopback for testing. */ 261 igb_setup_loopback(igb); 262 263 /* 264 * Disable DMA re-send on PCIe completion timeout (82576 datasheet 265 * section 8.6.1, GCR.Completion_Timeout_Resend, bit 16). The 266 * mix_and_match test intentionally submits descriptors targeting 267 * unmapped IOVAs; with the default (set) value, the device keeps 268 * retrying the failed read indefinitely, which keeps PCIe AER and 269 * IOMMU error handling busy and interferes with reset recovery. 270 */ 271 ctrl = igb_read32(igb, E1000_GCR); 272 ctrl &= ~E1000_GCR_CMPL_TMOUT_RESEND; 273 igb_write32(igb, E1000_GCR, ctrl); 274 275 /* Configure TX and RX descriptor rings */ 276 igb_write32(igb, E1000_TDBAL(0), (u32)iova_tx); 277 igb_write32(igb, E1000_TDBAH(0), (u32)(iova_tx >> 32)); 278 igb_write32(igb, E1000_TDLEN(0), RING_SIZE * sizeof(struct igb_tx_desc)); 279 igb_write32(igb, E1000_TDH(0), 0); 280 igb_write32(igb, E1000_TDT(0), 0); 281 igb_write32(igb, E1000_TXDCTL(0), E1000_TXDCTL_QUEUE_ENABLE); 282 283 igb_write32(igb, E1000_RDBAL(0), (u32)iova_rx); 284 igb_write32(igb, E1000_RDBAH(0), (u32)(iova_rx >> 32)); 285 igb_write32(igb, E1000_RDLEN(0), RING_SIZE * sizeof(struct igb_rx_desc)); 286 igb_write32(igb, E1000_RDH(0), 0); 287 igb_write32(igb, E1000_RDT(0), 0); 288 289 /* 290 * Select the advanced one-buffer descriptor format. Per 82576 291 * datasheet section 7.1.5.2: "SRRCTL[n].DESCTYPE must be set to a 292 * value other than 000b for the 82576 to write back the special 293 * descriptors." struct igb_rx_desc matches the advanced one-buffer 294 * writeback layout (section 7.1.5.2), so polling rx.wb.status_error 295 * requires this format. Section 8.10.2 specifies DESCTYPE[27:25]. 296 * 297 * The direct write also zeroes SRRCTL.BSIZEPACKET, which is 298 * intentional: per section 7.1.3.1 a zero BSIZEPACKET falls back to 299 * the RCTL.BSIZE buffer size, whose reset default (00b) is 2048 300 * bytes -- ample for the loopback frames here. 301 */ 302 igb_write32(igb, E1000_SRRCTL(0), E1000_SRRCTL_DESCTYPE_ADV_ONEBUF); 303 304 igb_write32(igb, E1000_RXDCTL(0), E1000_RXDCTL_QUEUE_ENABLE); 305 306 /* 307 * Enable Receiver and Transmitter. RCTL.LBM_MAC is set in addition 308 * to PHY loopback as a QEMU-only accommodation: QEMU's emulated igb 309 * does not honor PHY register 0 bit 14 (PHY internal loopback) and 310 * relies on RCTL.LBM_MAC to wrap TX descriptors back to the RX 311 * queue. Datasheet 8.10.1 (RCTL register) advises "When using the 312 * internal PHY, LBM should remain set to 00b", so setting LBM_MAC 313 * here deviates from datasheet guidance; empirically the bit has 314 * no observable effect on real 82576 hardware because MAC loopback 315 * is not implemented (datasheet 3.5.6.2). Setting both lets the 316 * selftest work on both real hardware and QEMU without conditional 317 * code paths. 318 */ 319 rctl = E1000_RCTL_EN | /* Receiver Enable */ 320 E1000_RCTL_UPE | /* Unicast Promiscuous (for dummy MAC) */ 321 E1000_RCTL_MPE | /* Multicast Promiscuous */ 322 E1000_RCTL_BAM | /* Broadcast Accept Mode */ 323 E1000_RCTL_LBM_MAC | /* MAC Loopback - for QEMU emulation only */ 324 E1000_RCTL_SECRC; /* Strip CRC (needed for memcmp) */ 325 igb_write32(igb, E1000_RCTL, rctl); 326 igb_write32(igb, E1000_TCTL, E1000_TCTL_EN | E1000_TCTL_PSP); 327 328 /* 329 * Wait for TX and RX queues to be enabled. Per the RXDCTL/TXDCTL 330 * register definitions (8.10.10/8.12.13), the per-queue enable bit 331 * "remains zero" until the global RCTL.RXEN/TCTL.TXEN are set, so 332 * E1000_RCTL_EN and E1000_TCTL_EN must already be written above. 333 */ 334 retries = 2000; 335 while (retries-- > 0) { 336 if ((igb_read32(igb, E1000_TXDCTL(0)) & E1000_TXDCTL_QUEUE_ENABLE) && 337 (igb_read32(igb, E1000_RXDCTL(0)) & E1000_RXDCTL_QUEUE_ENABLE)) 338 break; 339 usleep(10); 340 } 341 VFIO_ASSERT_GE(retries, 0); 342 343 /* 344 * Program MSI-X interrupt routing per 82576 datasheet: 345 * 346 * GPIE (section 7.3.2.11, Table 7-47): set Multiple_MSIX (bit 4) to 347 * route interrupt causes through IVAR mapping, and EIAME (bit 30) 348 * to apply EIAM on MSI-X assertion (without EIAME, EIAM only 349 * applies on EICR read/write). 350 * 351 * EIAC (section 8.8.5): enable auto-clear of EICR for vector 0. 352 * Without auto-clear the cause stays set after delivery and the 353 * test can see spurious interrupts on the next memcpy batch. 354 * 355 * EIAM (section 8.8.6): enable auto-mask of EIMS for vector 0 on 356 * MSI-X assertion (effective because EIAME is set). 357 * 358 * IVAR (section 7.3.1.2, register definition in 8.8.13): map RX 359 * cause 0 to MSI-X vector 0 and mark the entry valid. 360 */ 361 igb_write32(igb, E1000_GPIE, E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME); 362 igb_write32(igb, E1000_EIAC, MSIX_VECTOR_MASK); 363 igb_write32(igb, E1000_EIAM, MSIX_VECTOR_MASK); 364 365 /* Map vector 0 to interrupt cause 0 and mark it valid */ 366 igb_write32(igb, E1000_IVAR0, E1000_IVAR_VALID); 367 368 /* Enable interrupts on vector 0 */ 369 igb_write32(igb, E1000_EIMS, MSIX_VECTOR_MASK); 370 371 /* Initialize driver state and capability limits */ 372 igb->tx_tail = 0; 373 igb->rx_tail = 0; 374 375 device->driver.max_memcpy_size = IGB_MAX_CHUNK_SIZE; 376 device->driver.max_memcpy_count = RING_SIZE - 1; 377 device->driver.msi = MSIX_VECTOR; 378 } 379 380 static void igb_init(struct vfio_pci_device *device) 381 { 382 struct igb *igb = to_igb_state(device); 383 384 VFIO_ASSERT_GE(device->driver.region.size, sizeof(struct igb)); 385 386 igb->bar0 = device->bars[0].vaddr; 387 388 igb_reset(igb); 389 390 /* 391 * Enable MSI-X via VFIO before device-side register programming. 392 * vfio_pci_msix_enable() only touches the VFIO IRQ machinery and the 393 * PCI MSI-X capability via config space; it has no ordering 394 * dependency on the device-side writes performed by igb_hw_init(). 395 * Placing it here keeps igb_hw_init() reusable from the reset 396 * recovery path (which calls vfio_pci_irq_reenable() instead). 397 */ 398 vfio_pci_msix_enable(device, MSIX_VECTOR, 1); 399 400 igb_hw_init(device); 401 } 402 403 static void igb_remove(struct vfio_pci_device *device) 404 { 405 struct igb *igb = to_igb_state(device); 406 407 igb_write32(igb, E1000_RCTL, 0); 408 igb_write32(igb, E1000_TCTL, 0); 409 igb_reset(igb); 410 411 vfio_pci_msix_disable(device); 412 } 413 414 static void igb_irq_disable(struct igb *igb) 415 { 416 igb_write32(igb, E1000_EIMC, MSIX_VECTOR_MASK); 417 } 418 419 static void igb_irq_enable(struct igb *igb) 420 { 421 igb_write32(igb, E1000_EIMS, MSIX_VECTOR_MASK); 422 } 423 424 static void igb_irq_clear(struct igb *igb) 425 { 426 /* 427 * Use write-to-clear (datasheet 7.3.4.2). In MSI-X mode with EIAC 428 * programmed, section 8.8.5 explicitly states "If any bits are set 429 * in EIAC, the EICR register should not be read", which rules out 430 * the read-to-clear path in 7.3.4.3. Bits not in EIAC are still 431 * cleared by writing 1. 432 */ 433 igb_write32(igb, E1000_EICR, 0xFFFFFFFF); 434 } 435 436 static void igb_memcpy_start(struct vfio_pci_device *device, iova_t src, 437 iova_t dst, u64 size, u64 count) 438 { 439 struct igb *igb = to_igb_state(device); 440 struct igb_rx_desc *rx; 441 struct igb_tx_desc *tx; 442 u32 i; 443 444 VFIO_ASSERT_GE(size, 60, 445 "IGB driver requires memcpy size to be at least 60 bytes (Ethernet minimum payload size)"); 446 447 igb_irq_disable(igb); 448 449 for (i = 0; i < count; i++) { 450 tx = &igb->tx_ring[igb->tx_tail]; 451 rx = &igb->rx_ring[igb->rx_tail]; 452 453 memset(tx, 0, sizeof(struct igb_tx_desc)); 454 memset(rx, 0, sizeof(struct igb_rx_desc)); 455 456 rx->read.pkt_addr = cpu_to_le64(dst); 457 rx->read.hdr_addr = cpu_to_le64(0); 458 459 tx->read.buffer_addr = cpu_to_le64(src); 460 /* 461 * Build an advanced data descriptor per 82576 datasheet 462 * section 7.2.2.3. DEXT marks the descriptor as advanced 463 * (required by hardware); DTYP=data selects the data 464 * descriptor; IFCS asks the MAC to append the Ethernet 465 * FCS (without it the frame is dropped as malformed); 466 * EOP marks end of packet. DTALEN is the buffer length 467 * in bits 15:0 of cmd_type_len. 468 */ 469 tx->read.cmd_type_len = cpu_to_le32((uint32_t)size | 470 E1000_ADVTXD_DTYP_DATA | 471 E1000_ADVTXD_DCMD_DEXT | 472 E1000_ADVTXD_DCMD_IFCS | 473 E1000_ADVTXD_DCMD_EOP); 474 /* 475 * PAYLEN (section 7.2.2.3.11) is the total payload size 476 * in olinfo_status[31:14]. 477 */ 478 tx->read.olinfo_status = 479 cpu_to_le32((uint32_t)size << E1000_ADVTXD_PAYLEN_SHIFT); 480 481 igb->tx_tail = (igb->tx_tail + 1) % RING_SIZE; 482 igb->rx_tail = (igb->rx_tail + 1) % RING_SIZE; 483 } 484 485 igb_write32(igb, E1000_RDT(0), igb->rx_tail); 486 igb_write32(igb, E1000_TDT(0), igb->tx_tail); 487 } 488 489 /* 490 * Reset the device via VFIO_DEVICE_RESET (PCIe FLR on the 82576) and 491 * re-program it. VFIO_DEVICE_RESET tears down the kernel-side MSI-X 492 * trigger but leaves user-side eventfds intact, so re-arm the trigger 493 * via vfio_pci_irq_reenable() before reprogramming so any caller-cached 494 * eventfd remains valid. 495 * 496 * FLR clears device-side state to power-on reset values (datasheet 497 * 4.2.1.5.1: a PF FLR is "equivalent to a D0->D3->D0 transition"), so 498 * EIMS and EICR come back as 0 from their register-defined initial 499 * values, and igb_hw_init() resets tx_tail/rx_tail to 0. The next 500 * igb_memcpy_start() will memset each descriptor it touches before 501 * submission, so no explicit IMC/EICR writes or ring memsets are 502 * needed here. 503 */ 504 static void igb_error_reset_and_reinit(struct vfio_pci_device *device) 505 { 506 vfio_pci_device_reset(device); 507 vfio_pci_msix_reenable(device, MSIX_VECTOR, 1); 508 igb_hw_init(device); 509 } 510 511 static int igb_memcpy_wait(struct vfio_pci_device *device) 512 { 513 struct igb *igb = to_igb_state(device); 514 struct igb_rx_desc *rx; 515 u32 status = 0; 516 u32 prev_tail; 517 int retries; 518 519 prev_tail = (igb->rx_tail + RING_SIZE - 1) % RING_SIZE; 520 rx = &igb->rx_ring[prev_tail]; 521 522 /* 523 * Real 82576 hardware processes the descriptor ring at line rate. 524 * max_memcpy_size = (RING_SIZE - 1) * IGB_MAX_CHUNK_SIZE ~= 4 MB, 525 * split into 4095 1 KB frames. At 1 Gb/s (~125 MB/s) the worst 526 * valid memcpy takes ~32 ms on the wire, plus per-frame preamble, 527 * SFD, IFG and FCS overhead (~3%) and descriptor fetch/writeback 528 * latency. Wait up to ~200 ms before declaring the device hung; 529 * ~6x the line-rate floor leaves comfortable headroom for host 530 * scheduling jitter while keeping the intentional invalid-DMA 531 * tests bounded. 532 */ 533 retries = 200; 534 while (retries-- > 0) { 535 status = le32_to_cpu(READ_ONCE(rx->wb.status_error)); 536 if (status & 1) 537 break; 538 usleep(1000); 539 } 540 541 if (status & 1) 542 /* 543 * Ensure the test code doesn't speculatively read the DMA 544 * destination buffer before we have verified that the 545 * descriptor writeback is complete. 546 */ 547 rmb(); 548 549 igb_irq_clear(igb); 550 551 igb_irq_enable(igb); 552 553 if (status & 1) 554 return 0; 555 556 /* 557 * The descriptor never completed. On real 82576 hardware this 558 * typically follows a DMA-read fault from one of the intentional 559 * unmapped-IOVA tests; the fault leaves the descriptor engine 560 * unable to service subsequent valid descriptors. CTRL.RST alone 561 * reinitializes the queue registers but leaves the engine wedged 562 * for the current process, so a broader VFIO_DEVICE_RESET (FLR) 563 * is required. 564 */ 565 igb_error_reset_and_reinit(device); 566 567 return -ETIMEDOUT; 568 } 569 570 static void igb_send_msi(struct vfio_pci_device *device) 571 { 572 struct igb *igb = to_igb_state(device); 573 574 igb_write32(igb, E1000_EICS, MSIX_VECTOR_MASK); 575 } 576 577 const struct vfio_pci_driver_ops igb_ops = { 578 .name = "igb", 579 .probe = igb_probe, 580 .init = igb_init, 581 .remove = igb_remove, 582 .memcpy_start = igb_memcpy_start, 583 .memcpy_wait = igb_memcpy_wait, 584 .send_msi = igb_send_msi, 585 }; 586