1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 289e1f7d4SAlex Williamson /* 389e1f7d4SAlex Williamson * VFIO PCI I/O Port & MMIO access 489e1f7d4SAlex Williamson * 589e1f7d4SAlex Williamson * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 689e1f7d4SAlex Williamson * Author: Alex Williamson <alex.williamson@redhat.com> 789e1f7d4SAlex Williamson * 889e1f7d4SAlex Williamson * Derived from original vfio: 989e1f7d4SAlex Williamson * Copyright 2010 Cisco Systems, Inc. All rights reserved. 1089e1f7d4SAlex Williamson * Author: Tom Lyon, pugs@cisco.com 1189e1f7d4SAlex Williamson */ 1289e1f7d4SAlex Williamson 1389e1f7d4SAlex Williamson #include <linux/fs.h> 1489e1f7d4SAlex Williamson #include <linux/pci.h> 1589e1f7d4SAlex Williamson #include <linux/uaccess.h> 1689e1f7d4SAlex Williamson #include <linux/io.h> 1730656177SAlex Williamson #include <linux/vfio.h> 1884237a82SAlex Williamson #include <linux/vgaarb.h> 1989e1f7d4SAlex Williamson 2089e1f7d4SAlex Williamson #include "vfio_pci_private.h" 2189e1f7d4SAlex Williamson 2207fd7ef3SAlex Williamson #ifdef __LITTLE_ENDIAN 2307fd7ef3SAlex Williamson #define vfio_ioread64 ioread64 2407fd7ef3SAlex Williamson #define vfio_iowrite64 iowrite64 2507fd7ef3SAlex Williamson #define vfio_ioread32 ioread32 2607fd7ef3SAlex Williamson #define vfio_iowrite32 iowrite32 2707fd7ef3SAlex Williamson #define vfio_ioread16 ioread16 2807fd7ef3SAlex Williamson #define vfio_iowrite16 iowrite16 2907fd7ef3SAlex Williamson #else 3007fd7ef3SAlex Williamson #define vfio_ioread64 ioread64be 3107fd7ef3SAlex Williamson #define vfio_iowrite64 iowrite64be 3207fd7ef3SAlex Williamson #define vfio_ioread32 ioread32be 3307fd7ef3SAlex Williamson #define vfio_iowrite32 iowrite32be 3407fd7ef3SAlex Williamson #define vfio_ioread16 ioread16be 3507fd7ef3SAlex Williamson #define vfio_iowrite16 iowrite16be 3607fd7ef3SAlex Williamson #endif 3707fd7ef3SAlex Williamson #define vfio_ioread8 ioread8 3807fd7ef3SAlex Williamson #define vfio_iowrite8 iowrite8 3907fd7ef3SAlex Williamson 4089e1f7d4SAlex Williamson /* 41906ee99dSAlex Williamson * Read or write from an __iomem region (MMIO or I/O port) with an excluded 42906ee99dSAlex Williamson * range which is inaccessible. The excluded range drops writes and fills 43906ee99dSAlex Williamson * reads with -1. This is intended for handling MSI-X vector tables and 44906ee99dSAlex Williamson * leftover space for ROM BARs. 4589e1f7d4SAlex Williamson */ 46906ee99dSAlex Williamson static ssize_t do_io_rw(void __iomem *io, char __user *buf, 47906ee99dSAlex Williamson loff_t off, size_t count, size_t x_start, 48906ee99dSAlex Williamson size_t x_end, bool iswrite) 4989e1f7d4SAlex Williamson { 50906ee99dSAlex Williamson ssize_t done = 0; 5189e1f7d4SAlex Williamson 5289e1f7d4SAlex Williamson while (count) { 5389e1f7d4SAlex Williamson size_t fillable, filled; 5489e1f7d4SAlex Williamson 55906ee99dSAlex Williamson if (off < x_start) 56906ee99dSAlex Williamson fillable = min(count, (size_t)(x_start - off)); 57906ee99dSAlex Williamson else if (off >= x_end) 58906ee99dSAlex Williamson fillable = count; 5989e1f7d4SAlex Williamson else 6089e1f7d4SAlex Williamson fillable = 0; 6189e1f7d4SAlex Williamson 62906ee99dSAlex Williamson if (fillable >= 4 && !(off % 4)) { 6307fd7ef3SAlex Williamson u32 val; 6489e1f7d4SAlex Williamson 6589e1f7d4SAlex Williamson if (iswrite) { 6689e1f7d4SAlex Williamson if (copy_from_user(&val, buf, 4)) 67906ee99dSAlex Williamson return -EFAULT; 6889e1f7d4SAlex Williamson 6907fd7ef3SAlex Williamson vfio_iowrite32(val, io + off); 7089e1f7d4SAlex Williamson } else { 7107fd7ef3SAlex Williamson val = vfio_ioread32(io + off); 7289e1f7d4SAlex Williamson 7389e1f7d4SAlex Williamson if (copy_to_user(buf, &val, 4)) 74906ee99dSAlex Williamson return -EFAULT; 7589e1f7d4SAlex Williamson } 7689e1f7d4SAlex Williamson 7789e1f7d4SAlex Williamson filled = 4; 78906ee99dSAlex Williamson } else if (fillable >= 2 && !(off % 2)) { 7907fd7ef3SAlex Williamson u16 val; 8089e1f7d4SAlex Williamson 8189e1f7d4SAlex Williamson if (iswrite) { 8289e1f7d4SAlex Williamson if (copy_from_user(&val, buf, 2)) 83906ee99dSAlex Williamson return -EFAULT; 8489e1f7d4SAlex Williamson 8507fd7ef3SAlex Williamson vfio_iowrite16(val, io + off); 8689e1f7d4SAlex Williamson } else { 8707fd7ef3SAlex Williamson val = vfio_ioread16(io + off); 8889e1f7d4SAlex Williamson 8989e1f7d4SAlex Williamson if (copy_to_user(buf, &val, 2)) 90906ee99dSAlex Williamson return -EFAULT; 9189e1f7d4SAlex Williamson } 9289e1f7d4SAlex Williamson 9389e1f7d4SAlex Williamson filled = 2; 9489e1f7d4SAlex Williamson } else if (fillable) { 9589e1f7d4SAlex Williamson u8 val; 9689e1f7d4SAlex Williamson 9789e1f7d4SAlex Williamson if (iswrite) { 9889e1f7d4SAlex Williamson if (copy_from_user(&val, buf, 1)) 99906ee99dSAlex Williamson return -EFAULT; 10089e1f7d4SAlex Williamson 10107fd7ef3SAlex Williamson vfio_iowrite8(val, io + off); 10289e1f7d4SAlex Williamson } else { 10307fd7ef3SAlex Williamson val = vfio_ioread8(io + off); 10489e1f7d4SAlex Williamson 10589e1f7d4SAlex Williamson if (copy_to_user(buf, &val, 1)) 106906ee99dSAlex Williamson return -EFAULT; 10789e1f7d4SAlex Williamson } 10889e1f7d4SAlex Williamson 10989e1f7d4SAlex Williamson filled = 1; 11089e1f7d4SAlex Williamson } else { 111906ee99dSAlex Williamson /* Fill reads with -1, drop writes */ 112906ee99dSAlex Williamson filled = min(count, (size_t)(x_end - off)); 11389e1f7d4SAlex Williamson if (!iswrite) { 114906ee99dSAlex Williamson u8 val = 0xFF; 11589e1f7d4SAlex Williamson size_t i; 11689e1f7d4SAlex Williamson 117906ee99dSAlex Williamson for (i = 0; i < filled; i++) 118906ee99dSAlex Williamson if (copy_to_user(buf + i, &val, 1)) 119906ee99dSAlex Williamson return -EFAULT; 12089e1f7d4SAlex Williamson } 12189e1f7d4SAlex Williamson } 12289e1f7d4SAlex Williamson 12389e1f7d4SAlex Williamson count -= filled; 12489e1f7d4SAlex Williamson done += filled; 125906ee99dSAlex Williamson off += filled; 12689e1f7d4SAlex Williamson buf += filled; 12789e1f7d4SAlex Williamson } 12889e1f7d4SAlex Williamson 129906ee99dSAlex Williamson return done; 130906ee99dSAlex Williamson } 131906ee99dSAlex Williamson 1320d77ed35SAlex Williamson static int vfio_pci_setup_barmap(struct vfio_pci_device *vdev, int bar) 1330d77ed35SAlex Williamson { 1340d77ed35SAlex Williamson struct pci_dev *pdev = vdev->pdev; 1350d77ed35SAlex Williamson int ret; 1360d77ed35SAlex Williamson void __iomem *io; 1370d77ed35SAlex Williamson 1380d77ed35SAlex Williamson if (vdev->barmap[bar]) 1390d77ed35SAlex Williamson return 0; 1400d77ed35SAlex Williamson 1410d77ed35SAlex Williamson ret = pci_request_selected_regions(pdev, 1 << bar, "vfio"); 1420d77ed35SAlex Williamson if (ret) 1430d77ed35SAlex Williamson return ret; 1440d77ed35SAlex Williamson 1450d77ed35SAlex Williamson io = pci_iomap(pdev, bar, 0); 1460d77ed35SAlex Williamson if (!io) { 1470d77ed35SAlex Williamson pci_release_selected_regions(pdev, 1 << bar); 1480d77ed35SAlex Williamson return -ENOMEM; 1490d77ed35SAlex Williamson } 1500d77ed35SAlex Williamson 1510d77ed35SAlex Williamson vdev->barmap[bar] = io; 1520d77ed35SAlex Williamson 1530d77ed35SAlex Williamson return 0; 1540d77ed35SAlex Williamson } 1550d77ed35SAlex Williamson 156906ee99dSAlex Williamson ssize_t vfio_pci_bar_rw(struct vfio_pci_device *vdev, char __user *buf, 157906ee99dSAlex Williamson size_t count, loff_t *ppos, bool iswrite) 158906ee99dSAlex Williamson { 159906ee99dSAlex Williamson struct pci_dev *pdev = vdev->pdev; 160906ee99dSAlex Williamson loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK; 161906ee99dSAlex Williamson int bar = VFIO_PCI_OFFSET_TO_INDEX(*ppos); 162906ee99dSAlex Williamson size_t x_start = 0, x_end = 0; 163906ee99dSAlex Williamson resource_size_t end; 164906ee99dSAlex Williamson void __iomem *io; 165906ee99dSAlex Williamson ssize_t done; 166906ee99dSAlex Williamson 167a13b6459SAlex Williamson if (pci_resource_start(pdev, bar)) 168906ee99dSAlex Williamson end = pci_resource_len(pdev, bar); 169a13b6459SAlex Williamson else if (bar == PCI_ROM_RESOURCE && 170a13b6459SAlex Williamson pdev->resource[bar].flags & IORESOURCE_ROM_SHADOW) 171a13b6459SAlex Williamson end = 0x20000; 172a13b6459SAlex Williamson else 173a13b6459SAlex Williamson return -EINVAL; 174906ee99dSAlex Williamson 175906ee99dSAlex Williamson if (pos >= end) 176906ee99dSAlex Williamson return -EINVAL; 177906ee99dSAlex Williamson 178906ee99dSAlex Williamson count = min(count, (size_t)(end - pos)); 179906ee99dSAlex Williamson 180906ee99dSAlex Williamson if (bar == PCI_ROM_RESOURCE) { 181906ee99dSAlex Williamson /* 182906ee99dSAlex Williamson * The ROM can fill less space than the BAR, so we start the 183906ee99dSAlex Williamson * excluded range at the end of the actual ROM. This makes 184906ee99dSAlex Williamson * filling large ROM BARs much faster. 185906ee99dSAlex Williamson */ 186906ee99dSAlex Williamson io = pci_map_rom(pdev, &x_start); 187906ee99dSAlex Williamson if (!io) 188906ee99dSAlex Williamson return -ENOMEM; 189906ee99dSAlex Williamson x_end = end; 1900d77ed35SAlex Williamson } else { 1910d77ed35SAlex Williamson int ret = vfio_pci_setup_barmap(vdev, bar); 192906ee99dSAlex Williamson if (ret) 193906ee99dSAlex Williamson return ret; 194906ee99dSAlex Williamson 195906ee99dSAlex Williamson io = vdev->barmap[bar]; 1960d77ed35SAlex Williamson } 197906ee99dSAlex Williamson 198906ee99dSAlex Williamson if (bar == vdev->msix_bar) { 199906ee99dSAlex Williamson x_start = vdev->msix_offset; 200906ee99dSAlex Williamson x_end = vdev->msix_offset + vdev->msix_size; 201906ee99dSAlex Williamson } 202906ee99dSAlex Williamson 203906ee99dSAlex Williamson done = do_io_rw(io, buf, pos, count, x_start, x_end, iswrite); 204906ee99dSAlex Williamson 205906ee99dSAlex Williamson if (done >= 0) 20689e1f7d4SAlex Williamson *ppos += done; 20789e1f7d4SAlex Williamson 20889e1f7d4SAlex Williamson if (bar == PCI_ROM_RESOURCE) 20989e1f7d4SAlex Williamson pci_unmap_rom(pdev, io); 21089e1f7d4SAlex Williamson 211906ee99dSAlex Williamson return done; 21289e1f7d4SAlex Williamson } 21384237a82SAlex Williamson 21484237a82SAlex Williamson ssize_t vfio_pci_vga_rw(struct vfio_pci_device *vdev, char __user *buf, 21584237a82SAlex Williamson size_t count, loff_t *ppos, bool iswrite) 21684237a82SAlex Williamson { 21784237a82SAlex Williamson int ret; 21884237a82SAlex Williamson loff_t off, pos = *ppos & VFIO_PCI_OFFSET_MASK; 21984237a82SAlex Williamson void __iomem *iomem = NULL; 22084237a82SAlex Williamson unsigned int rsrc; 22184237a82SAlex Williamson bool is_ioport; 22284237a82SAlex Williamson ssize_t done; 22384237a82SAlex Williamson 22484237a82SAlex Williamson if (!vdev->has_vga) 22584237a82SAlex Williamson return -EINVAL; 22684237a82SAlex Williamson 22745e86971SArnd Bergmann if (pos > 0xbfffful) 22845e86971SArnd Bergmann return -EINVAL; 22945e86971SArnd Bergmann 23045e86971SArnd Bergmann switch ((u32)pos) { 23184237a82SAlex Williamson case 0xa0000 ... 0xbffff: 23284237a82SAlex Williamson count = min(count, (size_t)(0xc0000 - pos)); 233*4bdc0d67SChristoph Hellwig iomem = ioremap(0xa0000, 0xbffff - 0xa0000 + 1); 23484237a82SAlex Williamson off = pos - 0xa0000; 23584237a82SAlex Williamson rsrc = VGA_RSRC_LEGACY_MEM; 23684237a82SAlex Williamson is_ioport = false; 23784237a82SAlex Williamson break; 23884237a82SAlex Williamson case 0x3b0 ... 0x3bb: 23984237a82SAlex Williamson count = min(count, (size_t)(0x3bc - pos)); 24084237a82SAlex Williamson iomem = ioport_map(0x3b0, 0x3bb - 0x3b0 + 1); 24184237a82SAlex Williamson off = pos - 0x3b0; 24284237a82SAlex Williamson rsrc = VGA_RSRC_LEGACY_IO; 24384237a82SAlex Williamson is_ioport = true; 24484237a82SAlex Williamson break; 24584237a82SAlex Williamson case 0x3c0 ... 0x3df: 24684237a82SAlex Williamson count = min(count, (size_t)(0x3e0 - pos)); 24784237a82SAlex Williamson iomem = ioport_map(0x3c0, 0x3df - 0x3c0 + 1); 24884237a82SAlex Williamson off = pos - 0x3c0; 24984237a82SAlex Williamson rsrc = VGA_RSRC_LEGACY_IO; 25084237a82SAlex Williamson is_ioport = true; 25184237a82SAlex Williamson break; 25284237a82SAlex Williamson default: 25384237a82SAlex Williamson return -EINVAL; 25484237a82SAlex Williamson } 25584237a82SAlex Williamson 25684237a82SAlex Williamson if (!iomem) 25784237a82SAlex Williamson return -ENOMEM; 25884237a82SAlex Williamson 25984237a82SAlex Williamson ret = vga_get_interruptible(vdev->pdev, rsrc); 26084237a82SAlex Williamson if (ret) { 26184237a82SAlex Williamson is_ioport ? ioport_unmap(iomem) : iounmap(iomem); 26284237a82SAlex Williamson return ret; 26384237a82SAlex Williamson } 26484237a82SAlex Williamson 26584237a82SAlex Williamson done = do_io_rw(iomem, buf, off, count, 0, 0, iswrite); 26684237a82SAlex Williamson 26784237a82SAlex Williamson vga_put(vdev->pdev, rsrc); 26884237a82SAlex Williamson 26984237a82SAlex Williamson is_ioport ? ioport_unmap(iomem) : iounmap(iomem); 27084237a82SAlex Williamson 27184237a82SAlex Williamson if (done >= 0) 27284237a82SAlex Williamson *ppos += done; 27384237a82SAlex Williamson 27484237a82SAlex Williamson return done; 27584237a82SAlex Williamson } 27630656177SAlex Williamson 27730656177SAlex Williamson static int vfio_pci_ioeventfd_handler(void *opaque, void *unused) 27830656177SAlex Williamson { 27930656177SAlex Williamson struct vfio_pci_ioeventfd *ioeventfd = opaque; 28030656177SAlex Williamson 28130656177SAlex Williamson switch (ioeventfd->count) { 28230656177SAlex Williamson case 1: 28330656177SAlex Williamson vfio_iowrite8(ioeventfd->data, ioeventfd->addr); 28430656177SAlex Williamson break; 28530656177SAlex Williamson case 2: 28630656177SAlex Williamson vfio_iowrite16(ioeventfd->data, ioeventfd->addr); 28730656177SAlex Williamson break; 28830656177SAlex Williamson case 4: 28930656177SAlex Williamson vfio_iowrite32(ioeventfd->data, ioeventfd->addr); 29030656177SAlex Williamson break; 29130656177SAlex Williamson #ifdef iowrite64 29230656177SAlex Williamson case 8: 29330656177SAlex Williamson vfio_iowrite64(ioeventfd->data, ioeventfd->addr); 29430656177SAlex Williamson break; 29530656177SAlex Williamson #endif 29630656177SAlex Williamson } 29730656177SAlex Williamson 29830656177SAlex Williamson return 0; 29930656177SAlex Williamson } 30030656177SAlex Williamson 30130656177SAlex Williamson long vfio_pci_ioeventfd(struct vfio_pci_device *vdev, loff_t offset, 30230656177SAlex Williamson uint64_t data, int count, int fd) 30330656177SAlex Williamson { 30430656177SAlex Williamson struct pci_dev *pdev = vdev->pdev; 30530656177SAlex Williamson loff_t pos = offset & VFIO_PCI_OFFSET_MASK; 30630656177SAlex Williamson int ret, bar = VFIO_PCI_OFFSET_TO_INDEX(offset); 30730656177SAlex Williamson struct vfio_pci_ioeventfd *ioeventfd; 30830656177SAlex Williamson 30930656177SAlex Williamson /* Only support ioeventfds into BARs */ 31030656177SAlex Williamson if (bar > VFIO_PCI_BAR5_REGION_INDEX) 31130656177SAlex Williamson return -EINVAL; 31230656177SAlex Williamson 31330656177SAlex Williamson if (pos + count > pci_resource_len(pdev, bar)) 31430656177SAlex Williamson return -EINVAL; 31530656177SAlex Williamson 31630656177SAlex Williamson /* Disallow ioeventfds working around MSI-X table writes */ 31730656177SAlex Williamson if (bar == vdev->msix_bar && 31830656177SAlex Williamson !(pos + count <= vdev->msix_offset || 31930656177SAlex Williamson pos >= vdev->msix_offset + vdev->msix_size)) 32030656177SAlex Williamson return -EINVAL; 32130656177SAlex Williamson 32230656177SAlex Williamson #ifndef iowrite64 32330656177SAlex Williamson if (count == 8) 32430656177SAlex Williamson return -EINVAL; 32530656177SAlex Williamson #endif 32630656177SAlex Williamson 32730656177SAlex Williamson ret = vfio_pci_setup_barmap(vdev, bar); 32830656177SAlex Williamson if (ret) 32930656177SAlex Williamson return ret; 33030656177SAlex Williamson 33130656177SAlex Williamson mutex_lock(&vdev->ioeventfds_lock); 33230656177SAlex Williamson 33330656177SAlex Williamson list_for_each_entry(ioeventfd, &vdev->ioeventfds_list, next) { 33430656177SAlex Williamson if (ioeventfd->pos == pos && ioeventfd->bar == bar && 33530656177SAlex Williamson ioeventfd->data == data && ioeventfd->count == count) { 33630656177SAlex Williamson if (fd == -1) { 33730656177SAlex Williamson vfio_virqfd_disable(&ioeventfd->virqfd); 33830656177SAlex Williamson list_del(&ioeventfd->next); 33930656177SAlex Williamson vdev->ioeventfds_nr--; 34030656177SAlex Williamson kfree(ioeventfd); 34130656177SAlex Williamson ret = 0; 34230656177SAlex Williamson } else 34330656177SAlex Williamson ret = -EEXIST; 34430656177SAlex Williamson 34530656177SAlex Williamson goto out_unlock; 34630656177SAlex Williamson } 34730656177SAlex Williamson } 34830656177SAlex Williamson 34930656177SAlex Williamson if (fd < 0) { 35030656177SAlex Williamson ret = -ENODEV; 35130656177SAlex Williamson goto out_unlock; 35230656177SAlex Williamson } 35330656177SAlex Williamson 35430656177SAlex Williamson if (vdev->ioeventfds_nr >= VFIO_PCI_IOEVENTFD_MAX) { 35530656177SAlex Williamson ret = -ENOSPC; 35630656177SAlex Williamson goto out_unlock; 35730656177SAlex Williamson } 35830656177SAlex Williamson 35930656177SAlex Williamson ioeventfd = kzalloc(sizeof(*ioeventfd), GFP_KERNEL); 36030656177SAlex Williamson if (!ioeventfd) { 36130656177SAlex Williamson ret = -ENOMEM; 36230656177SAlex Williamson goto out_unlock; 36330656177SAlex Williamson } 36430656177SAlex Williamson 36530656177SAlex Williamson ioeventfd->addr = vdev->barmap[bar] + pos; 36630656177SAlex Williamson ioeventfd->data = data; 36730656177SAlex Williamson ioeventfd->pos = pos; 36830656177SAlex Williamson ioeventfd->bar = bar; 36930656177SAlex Williamson ioeventfd->count = count; 37030656177SAlex Williamson 37130656177SAlex Williamson ret = vfio_virqfd_enable(ioeventfd, vfio_pci_ioeventfd_handler, 37230656177SAlex Williamson NULL, NULL, &ioeventfd->virqfd, fd); 37330656177SAlex Williamson if (ret) { 37430656177SAlex Williamson kfree(ioeventfd); 37530656177SAlex Williamson goto out_unlock; 37630656177SAlex Williamson } 37730656177SAlex Williamson 37830656177SAlex Williamson list_add(&ioeventfd->next, &vdev->ioeventfds_list); 37930656177SAlex Williamson vdev->ioeventfds_nr++; 38030656177SAlex Williamson 38130656177SAlex Williamson out_unlock: 38230656177SAlex Williamson mutex_unlock(&vdev->ioeventfds_lock); 38330656177SAlex Williamson 38430656177SAlex Williamson return ret; 38530656177SAlex Williamson } 386