26 static inline struct ioat_state *to_ioat_state(struct vfio_pci_device *device)  in to_ioat_state()  argument
28 	return device->driver.region.vaddr;  in to_ioat_state()
31 static inline void *ioat_channel_registers(struct vfio_pci_device *device)  in ioat_channel_registers()  argument
33 	return device->bars[0].vaddr + IOAT_CHANNEL_MMIO_SIZE;  in ioat_channel_registers()
36 static int ioat_probe(struct vfio_pci_device *device)  in ioat_probe()  argument
41 	if (!vfio_pci_device_match(device, PCI_VENDOR_ID_INTEL,  in ioat_probe()
45 	VFIO_ASSERT_NOT_NULL(device->bars[0].vaddr);  in ioat_probe()
47 	version = readb(device->bars[0].vaddr + IOAT_VER_OFFSET);  in ioat_probe()
54 		dev_err(device, "ioat: Unsupported version: 0x%x\n", version);  in ioat_probe()
65 static void ioat_clear_errors(struct vfio_pci_device *device)  in ioat_clear_errors()  argument
67 	void *registers = ioat_channel_registers(device);  in ioat_clear_errors()
70 	errors = vfio_pci_config_readl(device, IOAT_PCI_CHANERR_INT_OFFSET);  in ioat_clear_errors()
71 	vfio_pci_config_writel(device, IOAT_PCI_CHANERR_INT_OFFSET, errors);  in ioat_clear_errors()
73 	errors = vfio_pci_config_readl(device, IOAT_PCI_DMAUNCERRSTS_OFFSET);  in ioat_clear_errors()
74 	vfio_pci_config_writel(device, IOAT_PCI_CHANERR_INT_OFFSET, errors);  in ioat_clear_errors()
80 static void ioat_reset(struct vfio_pci_device *device)  in ioat_reset()  argument
82 	void *registers = ioat_channel_registers(device);  in ioat_reset()
86 	ioat_clear_errors(device);  in ioat_reset()
102 static void ioat_init(struct vfio_pci_device *device)  in ioat_init()  argument
104 	struct ioat_state *ioat = to_ioat_state(device);  in ioat_init()
107 	VFIO_ASSERT_GE(device->driver.region.size, sizeof(*ioat));  in ioat_init()
109 	vfio_pci_config_writew(device, PCI_COMMAND,  in ioat_init()
114 	ioat_reset(device);  in ioat_init()
118 	writeb(intrctrl, device->bars[0].vaddr + IOAT_INTRCTRL_OFFSET);  in ioat_init()
120 	vfio_pci_msix_enable(device, 0, device->msix_info.count);  in ioat_init()
122 	device->driver.msi = 0;  in ioat_init()
123 	device->driver.max_memcpy_size =  in ioat_init()
124 		1UL << readb(device->bars[0].vaddr + IOAT_XFERCAP_OFFSET);  in ioat_init()
125 	device->driver.max_memcpy_count = IOAT_DMACOUNT_MAX;  in ioat_init()
128 static void ioat_remove(struct vfio_pci_device *device)  in ioat_remove()  argument
130 	ioat_reset(device);  in ioat_remove()
131 	vfio_pci_msix_disable(device);  in ioat_remove()
134 static void ioat_handle_error(struct vfio_pci_device *device)  in ioat_handle_error()  argument
136 	void *registers = ioat_channel_registers(device);  in ioat_handle_error()
138 	dev_err(device, "Error detected during memcpy operation!\n"  in ioat_handle_error()
143 		 vfio_pci_config_readl(device, IOAT_PCI_CHANERR_INT_OFFSET),  in ioat_handle_error()
144 		 vfio_pci_config_readl(device, IOAT_PCI_DMAUNCERRSTS_OFFSET));  in ioat_handle_error()
146 	ioat_reset(device);  in ioat_handle_error()
149 static int ioat_memcpy_wait(struct vfio_pci_device *device)  in ioat_memcpy_wait()  argument
151 	void *registers = ioat_channel_registers(device);  in ioat_memcpy_wait()
162 			ioat_handle_error(device);  in ioat_memcpy_wait()
178 static void __ioat_memcpy_start(struct vfio_pci_device *device,  in __ioat_memcpy_start()  argument
182 	void *registers = ioat_channel_registers(device);  in __ioat_memcpy_start()
183 	struct ioat_state *ioat = to_ioat_state(device);  in __ioat_memcpy_start()
187 	desc_iova = to_iova(device, &ioat->desc);  in __ioat_memcpy_start()
197 	/* Tell the device the address of the descriptor. */  in __ioat_memcpy_start()
208 static void ioat_memcpy_start(struct vfio_pci_device *device,  in ioat_memcpy_start()  argument
212 	__ioat_memcpy_start(device, src, dst, size, count, false);  in ioat_memcpy_start()
215 static void ioat_send_msi(struct vfio_pci_device *device)  in ioat_send_msi()  argument
217 	struct ioat_state *ioat = to_ioat_state(device);  in ioat_send_msi()
219 	__ioat_memcpy_start(device,  in ioat_send_msi()
220 			    to_iova(device, &ioat->send_msi_src),  in ioat_send_msi()
221 			    to_iova(device, &ioat->send_msi_dst),  in ioat_send_msi()
224 	VFIO_ASSERT_EQ(ioat_memcpy_wait(device), 0);  in ioat_send_msi()