1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved 4 */ 5 6 #include <linux/device.h> 7 #include <linux/module.h> 8 #include <linux/mutex.h> 9 #include <linux/pci.h> 10 #include <linux/pm_runtime.h> 11 #include <linux/types.h> 12 #include <linux/uaccess.h> 13 #include <linux/vfio.h> 14 #include <linux/vfio_pci_core.h> 15 #include <linux/virtio_pci.h> 16 #include <linux/virtio_net.h> 17 #include <linux/virtio_pci_admin.h> 18 19 #include "common.h" 20 21 static int 22 virtiovf_issue_legacy_rw_cmd(struct virtiovf_pci_core_device *virtvdev, 23 loff_t pos, char __user *buf, 24 size_t count, bool read) 25 { 26 bool msix_enabled = 27 (virtvdev->core_device.irq_type == VFIO_PCI_MSIX_IRQ_INDEX); 28 struct pci_dev *pdev = virtvdev->core_device.pdev; 29 u8 *bar0_buf = virtvdev->bar0_virtual_buf; 30 bool common; 31 u8 offset; 32 int ret; 33 34 common = pos < VIRTIO_PCI_CONFIG_OFF(msix_enabled); 35 /* offset within the relevant configuration area */ 36 offset = common ? pos : pos - VIRTIO_PCI_CONFIG_OFF(msix_enabled); 37 mutex_lock(&virtvdev->bar_mutex); 38 if (read) { 39 if (common) 40 ret = virtio_pci_admin_legacy_common_io_read(pdev, offset, 41 count, bar0_buf + pos); 42 else 43 ret = virtio_pci_admin_legacy_device_io_read(pdev, offset, 44 count, bar0_buf + pos); 45 if (ret) 46 goto out; 47 if (copy_to_user(buf, bar0_buf + pos, count)) 48 ret = -EFAULT; 49 } else { 50 if (copy_from_user(bar0_buf + pos, buf, count)) { 51 ret = -EFAULT; 52 goto out; 53 } 54 55 if (common) 56 ret = virtio_pci_admin_legacy_common_io_write(pdev, offset, 57 count, bar0_buf + pos); 58 else 59 ret = virtio_pci_admin_legacy_device_io_write(pdev, offset, 60 count, bar0_buf + pos); 61 } 62 out: 63 mutex_unlock(&virtvdev->bar_mutex); 64 return ret; 65 } 66 67 static int 68 virtiovf_pci_bar0_rw(struct virtiovf_pci_core_device *virtvdev, 69 loff_t pos, char __user *buf, 70 size_t count, bool read) 71 { 72 struct vfio_pci_core_device *core_device = &virtvdev->core_device; 73 struct pci_dev *pdev = core_device->pdev; 74 u16 queue_notify; 75 int ret; 76 77 if (!(le16_to_cpu(virtvdev->pci_cmd) & PCI_COMMAND_IO)) 78 return -EIO; 79 80 if (pos + count > virtvdev->bar0_virtual_buf_size) 81 return -EINVAL; 82 83 ret = pm_runtime_resume_and_get(&pdev->dev); 84 if (ret) { 85 pci_info_ratelimited(pdev, "runtime resume failed %d\n", ret); 86 return -EIO; 87 } 88 89 switch (pos) { 90 case VIRTIO_PCI_QUEUE_NOTIFY: 91 if (count != sizeof(queue_notify)) { 92 ret = -EINVAL; 93 goto end; 94 } 95 if (read) { 96 ret = vfio_pci_core_ioread16(core_device, true, &queue_notify, 97 virtvdev->notify_addr); 98 if (ret) 99 goto end; 100 if (copy_to_user(buf, &queue_notify, 101 sizeof(queue_notify))) { 102 ret = -EFAULT; 103 goto end; 104 } 105 } else { 106 if (copy_from_user(&queue_notify, buf, count)) { 107 ret = -EFAULT; 108 goto end; 109 } 110 ret = vfio_pci_core_iowrite16(core_device, true, queue_notify, 111 virtvdev->notify_addr); 112 } 113 break; 114 default: 115 ret = virtiovf_issue_legacy_rw_cmd(virtvdev, pos, buf, count, 116 read); 117 } 118 119 end: 120 pm_runtime_put(&pdev->dev); 121 return ret ? ret : count; 122 } 123 124 static ssize_t virtiovf_pci_read_config(struct vfio_device *core_vdev, 125 char __user *buf, size_t count, 126 loff_t *ppos) 127 { 128 struct virtiovf_pci_core_device *virtvdev = container_of( 129 core_vdev, struct virtiovf_pci_core_device, core_device.vdev); 130 loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK; 131 size_t register_offset; 132 loff_t copy_offset; 133 size_t copy_count; 134 __le32 val32; 135 __le16 val16; 136 u8 val8; 137 int ret; 138 139 ret = vfio_pci_core_read(core_vdev, buf, count, ppos); 140 if (ret < 0) 141 return ret; 142 143 if (vfio_pci_core_range_intersect_range(pos, count, PCI_DEVICE_ID, 144 sizeof(val16), ©_offset, 145 ©_count, ®ister_offset)) { 146 val16 = cpu_to_le16(VIRTIO_TRANS_ID_NET); 147 if (copy_to_user(buf + copy_offset, (void *)&val16 + register_offset, copy_count)) 148 return -EFAULT; 149 } 150 151 if ((le16_to_cpu(virtvdev->pci_cmd) & PCI_COMMAND_IO) && 152 vfio_pci_core_range_intersect_range(pos, count, PCI_COMMAND, 153 sizeof(val16), ©_offset, 154 ©_count, ®ister_offset)) { 155 if (copy_from_user((void *)&val16 + register_offset, buf + copy_offset, 156 copy_count)) 157 return -EFAULT; 158 val16 |= cpu_to_le16(PCI_COMMAND_IO); 159 if (copy_to_user(buf + copy_offset, (void *)&val16 + register_offset, 160 copy_count)) 161 return -EFAULT; 162 } 163 164 if (vfio_pci_core_range_intersect_range(pos, count, PCI_REVISION_ID, 165 sizeof(val8), ©_offset, 166 ©_count, ®ister_offset)) { 167 /* Transional needs to have revision 0 */ 168 val8 = 0; 169 if (copy_to_user(buf + copy_offset, &val8, copy_count)) 170 return -EFAULT; 171 } 172 173 if (vfio_pci_core_range_intersect_range(pos, count, PCI_BASE_ADDRESS_0, 174 sizeof(val32), ©_offset, 175 ©_count, ®ister_offset)) { 176 u32 bar_mask = ~(virtvdev->bar0_virtual_buf_size - 1); 177 u32 pci_base_addr_0 = le32_to_cpu(virtvdev->pci_base_addr_0); 178 179 val32 = cpu_to_le32((pci_base_addr_0 & bar_mask) | PCI_BASE_ADDRESS_SPACE_IO); 180 if (copy_to_user(buf + copy_offset, (void *)&val32 + register_offset, copy_count)) 181 return -EFAULT; 182 } 183 184 if (vfio_pci_core_range_intersect_range(pos, count, PCI_SUBSYSTEM_ID, 185 sizeof(val16), ©_offset, 186 ©_count, ®ister_offset)) { 187 /* 188 * Transitional devices use the PCI subsystem device id as 189 * virtio device id, same as legacy driver always did. 190 */ 191 val16 = cpu_to_le16(VIRTIO_ID_NET); 192 if (copy_to_user(buf + copy_offset, (void *)&val16 + register_offset, 193 copy_count)) 194 return -EFAULT; 195 } 196 197 if (vfio_pci_core_range_intersect_range(pos, count, PCI_SUBSYSTEM_VENDOR_ID, 198 sizeof(val16), ©_offset, 199 ©_count, ®ister_offset)) { 200 val16 = cpu_to_le16(PCI_VENDOR_ID_REDHAT_QUMRANET); 201 if (copy_to_user(buf + copy_offset, (void *)&val16 + register_offset, 202 copy_count)) 203 return -EFAULT; 204 } 205 206 return count; 207 } 208 209 ssize_t virtiovf_pci_core_read(struct vfio_device *core_vdev, char __user *buf, 210 size_t count, loff_t *ppos) 211 { 212 struct virtiovf_pci_core_device *virtvdev = container_of( 213 core_vdev, struct virtiovf_pci_core_device, core_device.vdev); 214 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); 215 loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK; 216 217 if (!count) 218 return 0; 219 220 if (index == VFIO_PCI_CONFIG_REGION_INDEX) 221 return virtiovf_pci_read_config(core_vdev, buf, count, ppos); 222 223 if (index == VFIO_PCI_BAR0_REGION_INDEX) 224 return virtiovf_pci_bar0_rw(virtvdev, pos, buf, count, true); 225 226 return vfio_pci_core_read(core_vdev, buf, count, ppos); 227 } 228 229 static ssize_t virtiovf_pci_write_config(struct vfio_device *core_vdev, 230 const char __user *buf, size_t count, 231 loff_t *ppos) 232 { 233 struct virtiovf_pci_core_device *virtvdev = container_of( 234 core_vdev, struct virtiovf_pci_core_device, core_device.vdev); 235 loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK; 236 size_t register_offset; 237 loff_t copy_offset; 238 size_t copy_count; 239 240 if (vfio_pci_core_range_intersect_range(pos, count, PCI_COMMAND, 241 sizeof(virtvdev->pci_cmd), 242 ©_offset, ©_count, 243 ®ister_offset)) { 244 if (copy_from_user((void *)&virtvdev->pci_cmd + register_offset, 245 buf + copy_offset, 246 copy_count)) 247 return -EFAULT; 248 } 249 250 if (vfio_pci_core_range_intersect_range(pos, count, PCI_BASE_ADDRESS_0, 251 sizeof(virtvdev->pci_base_addr_0), 252 ©_offset, ©_count, 253 ®ister_offset)) { 254 if (copy_from_user((void *)&virtvdev->pci_base_addr_0 + register_offset, 255 buf + copy_offset, 256 copy_count)) 257 return -EFAULT; 258 } 259 260 return vfio_pci_core_write(core_vdev, buf, count, ppos); 261 } 262 263 ssize_t virtiovf_pci_core_write(struct vfio_device *core_vdev, const char __user *buf, 264 size_t count, loff_t *ppos) 265 { 266 struct virtiovf_pci_core_device *virtvdev = container_of( 267 core_vdev, struct virtiovf_pci_core_device, core_device.vdev); 268 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); 269 loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK; 270 271 if (!count) 272 return 0; 273 274 if (index == VFIO_PCI_CONFIG_REGION_INDEX) 275 return virtiovf_pci_write_config(core_vdev, buf, count, ppos); 276 277 if (index == VFIO_PCI_BAR0_REGION_INDEX) 278 return virtiovf_pci_bar0_rw(virtvdev, pos, (char __user *)buf, count, false); 279 280 return vfio_pci_core_write(core_vdev, buf, count, ppos); 281 } 282 283 int virtiovf_pci_ioctl_get_region_info(struct vfio_device *core_vdev, 284 unsigned int cmd, unsigned long arg) 285 { 286 struct virtiovf_pci_core_device *virtvdev = container_of( 287 core_vdev, struct virtiovf_pci_core_device, core_device.vdev); 288 unsigned long minsz = offsetofend(struct vfio_region_info, offset); 289 void __user *uarg = (void __user *)arg; 290 struct vfio_region_info info = {}; 291 292 if (copy_from_user(&info, uarg, minsz)) 293 return -EFAULT; 294 295 if (info.argsz < minsz) 296 return -EINVAL; 297 298 switch (info.index) { 299 case VFIO_PCI_BAR0_REGION_INDEX: 300 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 301 info.size = virtvdev->bar0_virtual_buf_size; 302 info.flags = VFIO_REGION_INFO_FLAG_READ | 303 VFIO_REGION_INFO_FLAG_WRITE; 304 return copy_to_user(uarg, &info, minsz) ? -EFAULT : 0; 305 default: 306 return vfio_pci_core_ioctl(core_vdev, cmd, arg); 307 } 308 } 309 310 long virtiovf_vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd, 311 unsigned long arg) 312 { 313 switch (cmd) { 314 case VFIO_DEVICE_GET_REGION_INFO: 315 return virtiovf_pci_ioctl_get_region_info(core_vdev, cmd, arg); 316 default: 317 return vfio_pci_core_ioctl(core_vdev, cmd, arg); 318 } 319 } 320 321 static int virtiovf_set_notify_addr(struct virtiovf_pci_core_device *virtvdev) 322 { 323 struct vfio_pci_core_device *core_device = &virtvdev->core_device; 324 int ret; 325 326 /* 327 * Setup the BAR where the 'notify' exists to be used by vfio as well 328 * This will let us mmap it only once and use it when needed. 329 */ 330 ret = vfio_pci_core_setup_barmap(core_device, 331 virtvdev->notify_bar); 332 if (ret) 333 return ret; 334 335 virtvdev->notify_addr = core_device->barmap[virtvdev->notify_bar] + 336 virtvdev->notify_offset; 337 return 0; 338 } 339 340 int virtiovf_open_legacy_io(struct virtiovf_pci_core_device *virtvdev) 341 { 342 if (!virtvdev->bar0_virtual_buf) 343 return 0; 344 345 /* 346 * Upon close_device() the vfio_pci_core_disable() is called 347 * and will close all the previous mmaps, so it seems that the 348 * valid life cycle for the 'notify' addr is per open/close. 349 */ 350 return virtiovf_set_notify_addr(virtvdev); 351 } 352 353 static int virtiovf_get_device_config_size(unsigned short device) 354 { 355 /* Network card */ 356 return offsetofend(struct virtio_net_config, status); 357 } 358 359 static int virtiovf_read_notify_info(struct virtiovf_pci_core_device *virtvdev) 360 { 361 u64 offset; 362 int ret; 363 u8 bar; 364 365 ret = virtio_pci_admin_legacy_io_notify_info(virtvdev->core_device.pdev, 366 VIRTIO_ADMIN_CMD_NOTIFY_INFO_FLAGS_OWNER_MEM, 367 &bar, &offset); 368 if (ret) 369 return ret; 370 371 virtvdev->notify_bar = bar; 372 virtvdev->notify_offset = offset; 373 return 0; 374 } 375 376 static bool virtiovf_bar0_exists(struct pci_dev *pdev) 377 { 378 struct resource *res = pdev->resource; 379 380 return res->flags; 381 } 382 383 bool virtiovf_support_legacy_io(struct pci_dev *pdev) 384 { 385 return virtio_pci_admin_has_legacy_io(pdev) && !virtiovf_bar0_exists(pdev); 386 } 387 388 int virtiovf_init_legacy_io(struct virtiovf_pci_core_device *virtvdev) 389 { 390 struct pci_dev *pdev = virtvdev->core_device.pdev; 391 int ret; 392 393 ret = virtiovf_read_notify_info(virtvdev); 394 if (ret) 395 return ret; 396 397 virtvdev->bar0_virtual_buf_size = VIRTIO_PCI_CONFIG_OFF(true) + 398 virtiovf_get_device_config_size(pdev->device); 399 BUILD_BUG_ON(!is_power_of_2(virtvdev->bar0_virtual_buf_size)); 400 virtvdev->bar0_virtual_buf = kzalloc(virtvdev->bar0_virtual_buf_size, 401 GFP_KERNEL); 402 if (!virtvdev->bar0_virtual_buf) 403 return -ENOMEM; 404 mutex_init(&virtvdev->bar_mutex); 405 return 0; 406 } 407 408 void virtiovf_release_legacy_io(struct virtiovf_pci_core_device *virtvdev) 409 { 410 kfree(virtvdev->bar0_virtual_buf); 411 } 412 413 void virtiovf_legacy_io_reset_done(struct pci_dev *pdev) 414 { 415 struct virtiovf_pci_core_device *virtvdev = dev_get_drvdata(&pdev->dev); 416 417 virtvdev->pci_cmd = 0; 418 } 419