1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Mediated virtual PCI serial host device driver 4 * 5 * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. 6 * Author: Neo Jia <cjia@nvidia.com> 7 * Kirti Wankhede <kwankhede@nvidia.com> 8 * 9 * Sample driver that creates mdev device that simulates serial port over PCI 10 * card. 11 */ 12 13 #include <linux/init.h> 14 #include <linux/module.h> 15 #include <linux/kernel.h> 16 #include <linux/fs.h> 17 #include <linux/poll.h> 18 #include <linux/slab.h> 19 #include <linux/cdev.h> 20 #include <linux/sched.h> 21 #include <linux/wait.h> 22 #include <linux/vfio.h> 23 #include <linux/iommu.h> 24 #include <linux/sysfs.h> 25 #include <linux/ctype.h> 26 #include <linux/file.h> 27 #include <linux/mdev.h> 28 #include <linux/pci.h> 29 #include <linux/serial.h> 30 #include <uapi/linux/serial_reg.h> 31 #include <linux/eventfd.h> 32 #include <linux/anon_inodes.h> 33 34 /* 35 * #defines 36 */ 37 38 #define VERSION_STRING "0.1" 39 #define DRIVER_AUTHOR "NVIDIA Corporation" 40 41 #define MTTY_CLASS_NAME "mtty" 42 43 #define MTTY_NAME "mtty" 44 45 #define MTTY_STRING_LEN 16 46 47 #define MTTY_CONFIG_SPACE_SIZE 0xff 48 #define MTTY_IO_BAR_SIZE 0x8 49 #define MTTY_MMIO_BAR_SIZE 0x100000 50 51 #define STORE_LE16(addr, val) (*(u16 *)addr = val) 52 #define STORE_LE32(addr, val) (*(u32 *)addr = val) 53 54 #define MAX_FIFO_SIZE 16 55 56 #define CIRCULAR_BUF_INC_IDX(idx) (idx = (idx + 1) & (MAX_FIFO_SIZE - 1)) 57 58 #define MTTY_VFIO_PCI_OFFSET_SHIFT 40 59 60 #define MTTY_VFIO_PCI_OFFSET_TO_INDEX(off) (off >> MTTY_VFIO_PCI_OFFSET_SHIFT) 61 #define MTTY_VFIO_PCI_INDEX_TO_OFFSET(index) \ 62 ((u64)(index) << MTTY_VFIO_PCI_OFFSET_SHIFT) 63 #define MTTY_VFIO_PCI_OFFSET_MASK \ 64 (((u64)(1) << MTTY_VFIO_PCI_OFFSET_SHIFT) - 1) 65 #define MAX_MTTYS 24 66 67 /* 68 * Global Structures 69 */ 70 71 static struct mtty_dev { 72 dev_t vd_devt; 73 struct class *vd_class; 74 struct cdev vd_cdev; 75 struct idr vd_idr; 76 struct device dev; 77 struct mdev_parent parent; 78 } mtty_dev; 79 80 struct mdev_region_info { 81 u64 start; 82 u64 phys_start; 83 u32 size; 84 u64 vfio_offset; 85 }; 86 87 #if defined(DEBUG_REGS) 88 static const char *wr_reg[] = { 89 "TX", 90 "IER", 91 "FCR", 92 "LCR", 93 "MCR", 94 "LSR", 95 "MSR", 96 "SCR" 97 }; 98 99 static const char *rd_reg[] = { 100 "RX", 101 "IER", 102 "IIR", 103 "LCR", 104 "MCR", 105 "LSR", 106 "MSR", 107 "SCR" 108 }; 109 #endif 110 111 /* loop back buffer */ 112 struct rxtx { 113 u8 fifo[MAX_FIFO_SIZE]; 114 u8 head, tail; 115 u8 count; 116 }; 117 118 struct serial_port { 119 u8 uart_reg[8]; /* 8 registers */ 120 struct rxtx rxtx; /* loop back buffer */ 121 bool dlab; 122 bool overrun; 123 u16 divisor; 124 u8 fcr; /* FIFO control register */ 125 u8 max_fifo_size; 126 u8 intr_trigger_level; /* interrupt trigger level */ 127 }; 128 129 struct mtty_data { 130 u64 magic; 131 #define MTTY_MAGIC 0x7e9d09898c3e2c4e /* Nothing clever, just random */ 132 u32 major_ver; 133 #define MTTY_MAJOR_VER 1 134 u32 minor_ver; 135 #define MTTY_MINOR_VER 0 136 u32 nr_ports; 137 u32 flags; 138 struct serial_port ports[2]; 139 }; 140 141 struct mdev_state; 142 143 struct mtty_migration_file { 144 struct file *filp; 145 struct mutex lock; 146 struct mdev_state *mdev_state; 147 struct mtty_data data; 148 ssize_t filled_size; 149 u8 disabled:1; 150 }; 151 152 /* State of each mdev device */ 153 struct mdev_state { 154 struct vfio_device vdev; 155 struct eventfd_ctx *intx_evtfd; 156 struct eventfd_ctx *msi_evtfd; 157 int irq_index; 158 u8 *vconfig; 159 struct mutex ops_lock; 160 struct mdev_device *mdev; 161 struct mdev_region_info region_info[VFIO_PCI_NUM_REGIONS]; 162 u32 bar_mask[VFIO_PCI_NUM_REGIONS]; 163 struct list_head next; 164 struct serial_port s[2]; 165 struct mutex rxtx_lock; 166 struct vfio_device_info dev_info; 167 int nr_ports; 168 enum vfio_device_mig_state state; 169 struct mutex state_mutex; 170 struct mutex reset_mutex; 171 struct mtty_migration_file *saving_migf; 172 struct mtty_migration_file *resuming_migf; 173 u8 deferred_reset:1; 174 u8 intx_mask:1; 175 }; 176 177 static struct mtty_type { 178 struct mdev_type type; 179 int nr_ports; 180 } mtty_types[2] = { 181 { .nr_ports = 1, .type.sysfs_name = "1", 182 .type.pretty_name = "Single port serial" }, 183 { .nr_ports = 2, .type.sysfs_name = "2", 184 .type.pretty_name = "Dual port serial" }, 185 }; 186 187 static struct mdev_type *mtty_mdev_types[] = { 188 &mtty_types[0].type, 189 &mtty_types[1].type, 190 }; 191 192 static atomic_t mdev_avail_ports = ATOMIC_INIT(MAX_MTTYS); 193 194 static const struct file_operations vd_fops = { 195 .owner = THIS_MODULE, 196 }; 197 198 static const struct vfio_device_ops mtty_dev_ops; 199 200 /* Helper functions */ 201 202 static void dump_buffer(u8 *buf, uint32_t count) 203 { 204 #if defined(DEBUG) 205 int i; 206 207 pr_info("Buffer:\n"); 208 for (i = 0; i < count; i++) { 209 pr_info("%2x ", *(buf + i)); 210 if ((i + 1) % 16 == 0) 211 pr_info("\n"); 212 } 213 #endif 214 } 215 216 static bool is_intx(struct mdev_state *mdev_state) 217 { 218 return mdev_state->irq_index == VFIO_PCI_INTX_IRQ_INDEX; 219 } 220 221 static bool is_msi(struct mdev_state *mdev_state) 222 { 223 return mdev_state->irq_index == VFIO_PCI_MSI_IRQ_INDEX; 224 } 225 226 static bool is_noirq(struct mdev_state *mdev_state) 227 { 228 return !is_intx(mdev_state) && !is_msi(mdev_state); 229 } 230 231 static void mtty_trigger_interrupt(struct mdev_state *mdev_state) 232 { 233 lockdep_assert_held(&mdev_state->ops_lock); 234 235 if (is_msi(mdev_state)) { 236 if (mdev_state->msi_evtfd) 237 eventfd_signal(mdev_state->msi_evtfd); 238 } else if (is_intx(mdev_state)) { 239 if (mdev_state->intx_evtfd && !mdev_state->intx_mask) { 240 eventfd_signal(mdev_state->intx_evtfd); 241 mdev_state->intx_mask = true; 242 } 243 } 244 } 245 246 static void mtty_create_config_space(struct mdev_state *mdev_state) 247 { 248 /* PCI dev ID */ 249 STORE_LE32((u32 *) &mdev_state->vconfig[0x0], 0x32534348); 250 251 /* Control: I/O+, Mem-, BusMaster- */ 252 STORE_LE16((u16 *) &mdev_state->vconfig[0x4], 0x0001); 253 254 /* Status: capabilities list absent */ 255 STORE_LE16((u16 *) &mdev_state->vconfig[0x6], 0x0200); 256 257 /* Rev ID */ 258 mdev_state->vconfig[0x8] = 0x10; 259 260 /* programming interface class : 16550-compatible serial controller */ 261 mdev_state->vconfig[0x9] = 0x02; 262 263 /* Sub class : 00 */ 264 mdev_state->vconfig[0xa] = 0x00; 265 266 /* Base class : Simple Communication controllers */ 267 mdev_state->vconfig[0xb] = 0x07; 268 269 /* base address registers */ 270 /* BAR0: IO space */ 271 STORE_LE32((u32 *) &mdev_state->vconfig[0x10], 0x000001); 272 mdev_state->bar_mask[0] = ~(MTTY_IO_BAR_SIZE) + 1; 273 274 if (mdev_state->nr_ports == 2) { 275 /* BAR1: IO space */ 276 STORE_LE32((u32 *) &mdev_state->vconfig[0x14], 0x000001); 277 mdev_state->bar_mask[1] = ~(MTTY_IO_BAR_SIZE) + 1; 278 } 279 280 /* Subsystem ID */ 281 STORE_LE32((u32 *) &mdev_state->vconfig[0x2c], 0x32534348); 282 283 mdev_state->vconfig[0x34] = 0x00; /* Cap Ptr */ 284 mdev_state->vconfig[0x3d] = 0x01; /* interrupt pin (INTA#) */ 285 286 /* Vendor specific data */ 287 mdev_state->vconfig[0x40] = 0x23; 288 mdev_state->vconfig[0x43] = 0x80; 289 mdev_state->vconfig[0x44] = 0x23; 290 mdev_state->vconfig[0x48] = 0x23; 291 mdev_state->vconfig[0x4c] = 0x23; 292 293 mdev_state->vconfig[0x60] = 0x50; 294 mdev_state->vconfig[0x61] = 0x43; 295 mdev_state->vconfig[0x62] = 0x49; 296 mdev_state->vconfig[0x63] = 0x20; 297 mdev_state->vconfig[0x64] = 0x53; 298 mdev_state->vconfig[0x65] = 0x65; 299 mdev_state->vconfig[0x66] = 0x72; 300 mdev_state->vconfig[0x67] = 0x69; 301 mdev_state->vconfig[0x68] = 0x61; 302 mdev_state->vconfig[0x69] = 0x6c; 303 mdev_state->vconfig[0x6a] = 0x2f; 304 mdev_state->vconfig[0x6b] = 0x55; 305 mdev_state->vconfig[0x6c] = 0x41; 306 mdev_state->vconfig[0x6d] = 0x52; 307 mdev_state->vconfig[0x6e] = 0x54; 308 } 309 310 static void handle_pci_cfg_write(struct mdev_state *mdev_state, u16 offset, 311 u8 *buf, u32 count) 312 { 313 u32 cfg_addr, bar_mask, bar_index = 0; 314 315 switch (offset) { 316 case 0x04: /* device control */ 317 case 0x06: /* device status */ 318 /* do nothing */ 319 break; 320 case 0x3c: /* interrupt line */ 321 mdev_state->vconfig[0x3c] = buf[0]; 322 break; 323 case 0x3d: 324 /* 325 * Interrupt Pin is hardwired to INTA. 326 * This field is write protected by hardware 327 */ 328 break; 329 case 0x10: /* BAR0 */ 330 case 0x14: /* BAR1 */ 331 if (offset == 0x10) 332 bar_index = 0; 333 else if (offset == 0x14) 334 bar_index = 1; 335 336 if ((mdev_state->nr_ports == 1) && (bar_index == 1)) { 337 STORE_LE32(&mdev_state->vconfig[offset], 0); 338 break; 339 } 340 341 cfg_addr = *(u32 *)buf; 342 pr_info("BAR%d addr 0x%x\n", bar_index, cfg_addr); 343 344 if (cfg_addr == 0xffffffff) { 345 bar_mask = mdev_state->bar_mask[bar_index]; 346 cfg_addr = (cfg_addr & bar_mask); 347 } 348 349 cfg_addr |= (mdev_state->vconfig[offset] & 0x3ul); 350 STORE_LE32(&mdev_state->vconfig[offset], cfg_addr); 351 break; 352 case 0x18: /* BAR2 */ 353 case 0x1c: /* BAR3 */ 354 case 0x20: /* BAR4 */ 355 STORE_LE32(&mdev_state->vconfig[offset], 0); 356 break; 357 default: 358 pr_info("PCI config write @0x%x of %d bytes not handled\n", 359 offset, count); 360 break; 361 } 362 } 363 364 static void handle_bar_write(unsigned int index, struct mdev_state *mdev_state, 365 u16 offset, u8 *buf, u32 count) 366 { 367 u8 data = *buf; 368 369 /* Handle data written by guest */ 370 switch (offset) { 371 case UART_TX: 372 /* if DLAB set, data is LSB of divisor */ 373 if (mdev_state->s[index].dlab) { 374 mdev_state->s[index].divisor |= data; 375 break; 376 } 377 378 mutex_lock(&mdev_state->rxtx_lock); 379 380 /* save in TX buffer */ 381 if (mdev_state->s[index].rxtx.count < 382 mdev_state->s[index].max_fifo_size) { 383 mdev_state->s[index].rxtx.fifo[ 384 mdev_state->s[index].rxtx.head] = data; 385 mdev_state->s[index].rxtx.count++; 386 CIRCULAR_BUF_INC_IDX(mdev_state->s[index].rxtx.head); 387 mdev_state->s[index].overrun = false; 388 389 /* 390 * Trigger interrupt if receive data interrupt is 391 * enabled and fifo reached trigger level 392 */ 393 if ((mdev_state->s[index].uart_reg[UART_IER] & 394 UART_IER_RDI) && 395 (mdev_state->s[index].rxtx.count == 396 mdev_state->s[index].intr_trigger_level)) { 397 /* trigger interrupt */ 398 #if defined(DEBUG_INTR) 399 pr_err("Serial port %d: Fifo level trigger\n", 400 index); 401 #endif 402 mtty_trigger_interrupt(mdev_state); 403 } 404 } else { 405 #if defined(DEBUG_INTR) 406 pr_err("Serial port %d: Buffer Overflow\n", index); 407 #endif 408 mdev_state->s[index].overrun = true; 409 410 /* 411 * Trigger interrupt if receiver line status interrupt 412 * is enabled 413 */ 414 if (mdev_state->s[index].uart_reg[UART_IER] & 415 UART_IER_RLSI) 416 mtty_trigger_interrupt(mdev_state); 417 } 418 mutex_unlock(&mdev_state->rxtx_lock); 419 break; 420 421 case UART_IER: 422 /* if DLAB set, data is MSB of divisor */ 423 if (mdev_state->s[index].dlab) 424 mdev_state->s[index].divisor |= (u16)data << 8; 425 else { 426 mdev_state->s[index].uart_reg[offset] = data; 427 mutex_lock(&mdev_state->rxtx_lock); 428 if ((data & UART_IER_THRI) && 429 (mdev_state->s[index].rxtx.head == 430 mdev_state->s[index].rxtx.tail)) { 431 #if defined(DEBUG_INTR) 432 pr_err("Serial port %d: IER_THRI write\n", 433 index); 434 #endif 435 mtty_trigger_interrupt(mdev_state); 436 } 437 438 mutex_unlock(&mdev_state->rxtx_lock); 439 } 440 441 break; 442 443 case UART_FCR: 444 mdev_state->s[index].fcr = data; 445 446 mutex_lock(&mdev_state->rxtx_lock); 447 if (data & (UART_FCR_CLEAR_RCVR | UART_FCR_CLEAR_XMIT)) { 448 /* clear loop back FIFO */ 449 mdev_state->s[index].rxtx.count = 0; 450 mdev_state->s[index].rxtx.head = 0; 451 mdev_state->s[index].rxtx.tail = 0; 452 } 453 mutex_unlock(&mdev_state->rxtx_lock); 454 455 switch (data & UART_FCR_TRIGGER_MASK) { 456 case UART_FCR_TRIGGER_1: 457 mdev_state->s[index].intr_trigger_level = 1; 458 break; 459 460 case UART_FCR_TRIGGER_4: 461 mdev_state->s[index].intr_trigger_level = 4; 462 break; 463 464 case UART_FCR_TRIGGER_8: 465 mdev_state->s[index].intr_trigger_level = 8; 466 break; 467 468 case UART_FCR_TRIGGER_14: 469 mdev_state->s[index].intr_trigger_level = 14; 470 break; 471 } 472 473 /* 474 * Set trigger level to 1 otherwise or implement timer with 475 * timeout of 4 characters and on expiring that timer set 476 * Recevice data timeout in IIR register 477 */ 478 mdev_state->s[index].intr_trigger_level = 1; 479 if (data & UART_FCR_ENABLE_FIFO) 480 mdev_state->s[index].max_fifo_size = MAX_FIFO_SIZE; 481 else { 482 mdev_state->s[index].max_fifo_size = 1; 483 mdev_state->s[index].intr_trigger_level = 1; 484 } 485 486 break; 487 488 case UART_LCR: 489 if (data & UART_LCR_DLAB) { 490 mdev_state->s[index].dlab = true; 491 mdev_state->s[index].divisor = 0; 492 } else 493 mdev_state->s[index].dlab = false; 494 495 mdev_state->s[index].uart_reg[offset] = data; 496 break; 497 498 case UART_MCR: 499 mdev_state->s[index].uart_reg[offset] = data; 500 501 if ((mdev_state->s[index].uart_reg[UART_IER] & UART_IER_MSI) && 502 (data & UART_MCR_OUT2)) { 503 #if defined(DEBUG_INTR) 504 pr_err("Serial port %d: MCR_OUT2 write\n", index); 505 #endif 506 mtty_trigger_interrupt(mdev_state); 507 } 508 509 if ((mdev_state->s[index].uart_reg[UART_IER] & UART_IER_MSI) && 510 (data & (UART_MCR_RTS | UART_MCR_DTR))) { 511 #if defined(DEBUG_INTR) 512 pr_err("Serial port %d: MCR RTS/DTR write\n", index); 513 #endif 514 mtty_trigger_interrupt(mdev_state); 515 } 516 break; 517 518 case UART_LSR: 519 case UART_MSR: 520 /* do nothing */ 521 break; 522 523 case UART_SCR: 524 mdev_state->s[index].uart_reg[offset] = data; 525 break; 526 527 default: 528 break; 529 } 530 } 531 532 static void handle_bar_read(unsigned int index, struct mdev_state *mdev_state, 533 u16 offset, u8 *buf, u32 count) 534 { 535 /* Handle read requests by guest */ 536 switch (offset) { 537 case UART_RX: 538 /* if DLAB set, data is LSB of divisor */ 539 if (mdev_state->s[index].dlab) { 540 *buf = (u8)mdev_state->s[index].divisor; 541 break; 542 } 543 544 mutex_lock(&mdev_state->rxtx_lock); 545 /* return data in tx buffer */ 546 if (mdev_state->s[index].rxtx.head != 547 mdev_state->s[index].rxtx.tail) { 548 *buf = mdev_state->s[index].rxtx.fifo[ 549 mdev_state->s[index].rxtx.tail]; 550 mdev_state->s[index].rxtx.count--; 551 CIRCULAR_BUF_INC_IDX(mdev_state->s[index].rxtx.tail); 552 } 553 554 if (mdev_state->s[index].rxtx.head == 555 mdev_state->s[index].rxtx.tail) { 556 /* 557 * Trigger interrupt if tx buffer empty interrupt is 558 * enabled and fifo is empty 559 */ 560 #if defined(DEBUG_INTR) 561 pr_err("Serial port %d: Buffer Empty\n", index); 562 #endif 563 if (mdev_state->s[index].uart_reg[UART_IER] & 564 UART_IER_THRI) 565 mtty_trigger_interrupt(mdev_state); 566 } 567 mutex_unlock(&mdev_state->rxtx_lock); 568 569 break; 570 571 case UART_IER: 572 if (mdev_state->s[index].dlab) { 573 *buf = (u8)(mdev_state->s[index].divisor >> 8); 574 break; 575 } 576 *buf = mdev_state->s[index].uart_reg[offset] & 0x0f; 577 break; 578 579 case UART_IIR: 580 { 581 u8 ier = mdev_state->s[index].uart_reg[UART_IER]; 582 *buf = 0; 583 584 mutex_lock(&mdev_state->rxtx_lock); 585 /* Interrupt priority 1: Parity, overrun, framing or break */ 586 if ((ier & UART_IER_RLSI) && mdev_state->s[index].overrun) 587 *buf |= UART_IIR_RLSI; 588 589 /* Interrupt priority 2: Fifo trigger level reached */ 590 if ((ier & UART_IER_RDI) && 591 (mdev_state->s[index].rxtx.count >= 592 mdev_state->s[index].intr_trigger_level)) 593 *buf |= UART_IIR_RDI; 594 595 /* Interrupt priotiry 3: transmitter holding register empty */ 596 if ((ier & UART_IER_THRI) && 597 (mdev_state->s[index].rxtx.head == 598 mdev_state->s[index].rxtx.tail)) 599 *buf |= UART_IIR_THRI; 600 601 /* Interrupt priotiry 4: Modem status: CTS, DSR, RI or DCD */ 602 if ((ier & UART_IER_MSI) && 603 (mdev_state->s[index].uart_reg[UART_MCR] & 604 (UART_MCR_RTS | UART_MCR_DTR))) 605 *buf |= UART_IIR_MSI; 606 607 /* bit0: 0=> interrupt pending, 1=> no interrupt is pending */ 608 if (*buf == 0) 609 *buf = UART_IIR_NO_INT; 610 611 /* set bit 6 & 7 to be 16550 compatible */ 612 *buf |= 0xC0; 613 mutex_unlock(&mdev_state->rxtx_lock); 614 } 615 break; 616 617 case UART_LCR: 618 case UART_MCR: 619 *buf = mdev_state->s[index].uart_reg[offset]; 620 break; 621 622 case UART_LSR: 623 { 624 u8 lsr = 0; 625 626 mutex_lock(&mdev_state->rxtx_lock); 627 /* atleast one char in FIFO */ 628 if (mdev_state->s[index].rxtx.head != 629 mdev_state->s[index].rxtx.tail) 630 lsr |= UART_LSR_DR; 631 632 /* if FIFO overrun */ 633 if (mdev_state->s[index].overrun) 634 lsr |= UART_LSR_OE; 635 636 /* transmit FIFO empty and tramsitter empty */ 637 if (mdev_state->s[index].rxtx.head == 638 mdev_state->s[index].rxtx.tail) 639 lsr |= UART_LSR_TEMT | UART_LSR_THRE; 640 641 mutex_unlock(&mdev_state->rxtx_lock); 642 *buf = lsr; 643 break; 644 } 645 case UART_MSR: 646 *buf = UART_MSR_DSR | UART_MSR_DDSR | UART_MSR_DCD; 647 648 mutex_lock(&mdev_state->rxtx_lock); 649 /* if AFE is 1 and FIFO have space, set CTS bit */ 650 if (mdev_state->s[index].uart_reg[UART_MCR] & 651 UART_MCR_AFE) { 652 if (mdev_state->s[index].rxtx.count < 653 mdev_state->s[index].max_fifo_size) 654 *buf |= UART_MSR_CTS | UART_MSR_DCTS; 655 } else 656 *buf |= UART_MSR_CTS | UART_MSR_DCTS; 657 mutex_unlock(&mdev_state->rxtx_lock); 658 659 break; 660 661 case UART_SCR: 662 *buf = mdev_state->s[index].uart_reg[offset]; 663 break; 664 665 default: 666 break; 667 } 668 } 669 670 static void mdev_read_base(struct mdev_state *mdev_state) 671 { 672 int index, pos; 673 u32 start_lo, start_hi; 674 u32 mem_type; 675 676 pos = PCI_BASE_ADDRESS_0; 677 678 for (index = 0; index <= VFIO_PCI_BAR5_REGION_INDEX; index++) { 679 680 if (!mdev_state->region_info[index].size) 681 continue; 682 683 start_lo = (*(u32 *)(mdev_state->vconfig + pos)) & 684 PCI_BASE_ADDRESS_MEM_MASK; 685 mem_type = (*(u32 *)(mdev_state->vconfig + pos)) & 686 PCI_BASE_ADDRESS_MEM_TYPE_MASK; 687 688 switch (mem_type) { 689 case PCI_BASE_ADDRESS_MEM_TYPE_64: 690 start_hi = (*(u32 *)(mdev_state->vconfig + pos + 4)); 691 pos += 4; 692 break; 693 case PCI_BASE_ADDRESS_MEM_TYPE_32: 694 case PCI_BASE_ADDRESS_MEM_TYPE_1M: 695 /* 1M mem BAR treated as 32-bit BAR */ 696 default: 697 /* mem unknown type treated as 32-bit BAR */ 698 start_hi = 0; 699 break; 700 } 701 pos += 4; 702 mdev_state->region_info[index].start = ((u64)start_hi << 32) | 703 start_lo; 704 } 705 } 706 707 static ssize_t mdev_access(struct mdev_state *mdev_state, u8 *buf, size_t count, 708 loff_t pos, bool is_write) 709 { 710 unsigned int index; 711 loff_t offset; 712 int ret = 0; 713 714 if (!buf) 715 return -EINVAL; 716 717 mutex_lock(&mdev_state->ops_lock); 718 719 index = MTTY_VFIO_PCI_OFFSET_TO_INDEX(pos); 720 offset = pos & MTTY_VFIO_PCI_OFFSET_MASK; 721 switch (index) { 722 case VFIO_PCI_CONFIG_REGION_INDEX: 723 724 #if defined(DEBUG) 725 pr_info("%s: PCI config space %s at offset 0x%llx\n", 726 __func__, is_write ? "write" : "read", offset); 727 #endif 728 if (is_write) { 729 dump_buffer(buf, count); 730 handle_pci_cfg_write(mdev_state, offset, buf, count); 731 } else { 732 memcpy(buf, (mdev_state->vconfig + offset), count); 733 dump_buffer(buf, count); 734 } 735 736 break; 737 738 case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: 739 if (!mdev_state->region_info[index].start) 740 mdev_read_base(mdev_state); 741 742 if (is_write) { 743 dump_buffer(buf, count); 744 745 #if defined(DEBUG_REGS) 746 pr_info("%s: BAR%d WR @0x%llx %s val:0x%02x dlab:%d\n", 747 __func__, index, offset, wr_reg[offset], 748 *buf, mdev_state->s[index].dlab); 749 #endif 750 handle_bar_write(index, mdev_state, offset, buf, count); 751 } else { 752 handle_bar_read(index, mdev_state, offset, buf, count); 753 dump_buffer(buf, count); 754 755 #if defined(DEBUG_REGS) 756 pr_info("%s: BAR%d RD @0x%llx %s val:0x%02x dlab:%d\n", 757 __func__, index, offset, rd_reg[offset], 758 *buf, mdev_state->s[index].dlab); 759 #endif 760 } 761 break; 762 763 default: 764 ret = -1; 765 goto accessfailed; 766 } 767 768 ret = count; 769 770 771 accessfailed: 772 mutex_unlock(&mdev_state->ops_lock); 773 774 return ret; 775 } 776 777 static size_t mtty_data_size(struct mdev_state *mdev_state) 778 { 779 return offsetof(struct mtty_data, ports) + 780 (mdev_state->nr_ports * sizeof(struct serial_port)); 781 } 782 783 static void mtty_disable_file(struct mtty_migration_file *migf) 784 { 785 mutex_lock(&migf->lock); 786 migf->disabled = true; 787 migf->filled_size = 0; 788 migf->filp->f_pos = 0; 789 mutex_unlock(&migf->lock); 790 } 791 792 static void mtty_disable_files(struct mdev_state *mdev_state) 793 { 794 if (mdev_state->saving_migf) { 795 mtty_disable_file(mdev_state->saving_migf); 796 fput(mdev_state->saving_migf->filp); 797 mdev_state->saving_migf = NULL; 798 } 799 800 if (mdev_state->resuming_migf) { 801 mtty_disable_file(mdev_state->resuming_migf); 802 fput(mdev_state->resuming_migf->filp); 803 mdev_state->resuming_migf = NULL; 804 } 805 } 806 807 static void mtty_state_mutex_unlock(struct mdev_state *mdev_state) 808 { 809 again: 810 mutex_lock(&mdev_state->reset_mutex); 811 if (mdev_state->deferred_reset) { 812 mdev_state->deferred_reset = false; 813 mutex_unlock(&mdev_state->reset_mutex); 814 mdev_state->state = VFIO_DEVICE_STATE_RUNNING; 815 mtty_disable_files(mdev_state); 816 goto again; 817 } 818 mutex_unlock(&mdev_state->state_mutex); 819 mutex_unlock(&mdev_state->reset_mutex); 820 } 821 822 static int mtty_release_migf(struct inode *inode, struct file *filp) 823 { 824 struct mtty_migration_file *migf = filp->private_data; 825 826 mtty_disable_file(migf); 827 mutex_destroy(&migf->lock); 828 kfree(migf); 829 830 return 0; 831 } 832 833 static long mtty_precopy_ioctl(struct file *filp, unsigned int cmd, 834 unsigned long arg) 835 { 836 struct mtty_migration_file *migf = filp->private_data; 837 struct mdev_state *mdev_state = migf->mdev_state; 838 loff_t *pos = &filp->f_pos; 839 struct vfio_precopy_info info = {}; 840 unsigned long minsz; 841 int ret; 842 843 if (cmd != VFIO_MIG_GET_PRECOPY_INFO) 844 return -ENOTTY; 845 846 minsz = offsetofend(struct vfio_precopy_info, dirty_bytes); 847 848 if (copy_from_user(&info, (void __user *)arg, minsz)) 849 return -EFAULT; 850 if (info.argsz < minsz) 851 return -EINVAL; 852 853 mutex_lock(&mdev_state->state_mutex); 854 if (mdev_state->state != VFIO_DEVICE_STATE_PRE_COPY && 855 mdev_state->state != VFIO_DEVICE_STATE_PRE_COPY_P2P) { 856 ret = -EINVAL; 857 goto unlock; 858 } 859 860 mutex_lock(&migf->lock); 861 862 if (migf->disabled) { 863 mutex_unlock(&migf->lock); 864 ret = -ENODEV; 865 goto unlock; 866 } 867 868 if (*pos > migf->filled_size) { 869 mutex_unlock(&migf->lock); 870 ret = -EINVAL; 871 goto unlock; 872 } 873 874 info.dirty_bytes = 0; 875 info.initial_bytes = migf->filled_size - *pos; 876 mutex_unlock(&migf->lock); 877 878 ret = copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 879 unlock: 880 mtty_state_mutex_unlock(mdev_state); 881 return ret; 882 } 883 884 static ssize_t mtty_save_read(struct file *filp, char __user *buf, 885 size_t len, loff_t *pos) 886 { 887 struct mtty_migration_file *migf = filp->private_data; 888 ssize_t ret = 0; 889 890 if (pos) 891 return -ESPIPE; 892 893 pos = &filp->f_pos; 894 895 mutex_lock(&migf->lock); 896 897 dev_dbg(migf->mdev_state->vdev.dev, "%s ask %zu\n", __func__, len); 898 899 if (migf->disabled) { 900 ret = -ENODEV; 901 goto out_unlock; 902 } 903 904 if (*pos > migf->filled_size) { 905 ret = -EINVAL; 906 goto out_unlock; 907 } 908 909 len = min_t(size_t, migf->filled_size - *pos, len); 910 if (len) { 911 if (copy_to_user(buf, (void *)&migf->data + *pos, len)) { 912 ret = -EFAULT; 913 goto out_unlock; 914 } 915 *pos += len; 916 ret = len; 917 } 918 out_unlock: 919 dev_dbg(migf->mdev_state->vdev.dev, "%s read %zu\n", __func__, ret); 920 mutex_unlock(&migf->lock); 921 return ret; 922 } 923 924 static const struct file_operations mtty_save_fops = { 925 .owner = THIS_MODULE, 926 .read = mtty_save_read, 927 .unlocked_ioctl = mtty_precopy_ioctl, 928 .compat_ioctl = compat_ptr_ioctl, 929 .release = mtty_release_migf, 930 }; 931 932 static void mtty_save_state(struct mdev_state *mdev_state) 933 { 934 struct mtty_migration_file *migf = mdev_state->saving_migf; 935 int i; 936 937 mutex_lock(&migf->lock); 938 for (i = 0; i < mdev_state->nr_ports; i++) { 939 memcpy(&migf->data.ports[i], 940 &mdev_state->s[i], sizeof(struct serial_port)); 941 migf->filled_size += sizeof(struct serial_port); 942 } 943 dev_dbg(mdev_state->vdev.dev, 944 "%s filled to %zu\n", __func__, migf->filled_size); 945 mutex_unlock(&migf->lock); 946 } 947 948 static int mtty_load_state(struct mdev_state *mdev_state) 949 { 950 struct mtty_migration_file *migf = mdev_state->resuming_migf; 951 int i; 952 953 mutex_lock(&migf->lock); 954 /* magic and version already tested by resume write fn */ 955 if (migf->filled_size < mtty_data_size(mdev_state)) { 956 dev_dbg(mdev_state->vdev.dev, "%s expected %zu, got %zu\n", 957 __func__, mtty_data_size(mdev_state), 958 migf->filled_size); 959 mutex_unlock(&migf->lock); 960 return -EINVAL; 961 } 962 963 for (i = 0; i < mdev_state->nr_ports; i++) 964 memcpy(&mdev_state->s[i], 965 &migf->data.ports[i], sizeof(struct serial_port)); 966 967 mutex_unlock(&migf->lock); 968 return 0; 969 } 970 971 static struct mtty_migration_file * 972 mtty_save_device_data(struct mdev_state *mdev_state, 973 enum vfio_device_mig_state state) 974 { 975 struct mtty_migration_file *migf = mdev_state->saving_migf; 976 struct mtty_migration_file *ret = NULL; 977 978 if (migf) { 979 if (state == VFIO_DEVICE_STATE_STOP_COPY) 980 goto fill_data; 981 return ret; 982 } 983 984 migf = kzalloc(sizeof(*migf), GFP_KERNEL_ACCOUNT); 985 if (!migf) 986 return ERR_PTR(-ENOMEM); 987 988 migf->filp = anon_inode_getfile("mtty_mig", &mtty_save_fops, 989 migf, O_RDONLY); 990 if (IS_ERR(migf->filp)) { 991 int rc = PTR_ERR(migf->filp); 992 993 kfree(migf); 994 return ERR_PTR(rc); 995 } 996 997 stream_open(migf->filp->f_inode, migf->filp); 998 mutex_init(&migf->lock); 999 migf->mdev_state = mdev_state; 1000 1001 migf->data.magic = MTTY_MAGIC; 1002 migf->data.major_ver = MTTY_MAJOR_VER; 1003 migf->data.minor_ver = MTTY_MINOR_VER; 1004 migf->data.nr_ports = mdev_state->nr_ports; 1005 1006 migf->filled_size = offsetof(struct mtty_data, ports); 1007 1008 dev_dbg(mdev_state->vdev.dev, "%s filled header to %zu\n", 1009 __func__, migf->filled_size); 1010 1011 ret = mdev_state->saving_migf = migf; 1012 1013 fill_data: 1014 if (state == VFIO_DEVICE_STATE_STOP_COPY) 1015 mtty_save_state(mdev_state); 1016 1017 return ret; 1018 } 1019 1020 static ssize_t mtty_resume_write(struct file *filp, const char __user *buf, 1021 size_t len, loff_t *pos) 1022 { 1023 struct mtty_migration_file *migf = filp->private_data; 1024 struct mdev_state *mdev_state = migf->mdev_state; 1025 loff_t requested_length; 1026 ssize_t ret = 0; 1027 1028 if (pos) 1029 return -ESPIPE; 1030 1031 pos = &filp->f_pos; 1032 1033 if (*pos < 0 || 1034 check_add_overflow((loff_t)len, *pos, &requested_length)) 1035 return -EINVAL; 1036 1037 if (requested_length > mtty_data_size(mdev_state)) 1038 return -ENOMEM; 1039 1040 mutex_lock(&migf->lock); 1041 1042 if (migf->disabled) { 1043 ret = -ENODEV; 1044 goto out_unlock; 1045 } 1046 1047 if (copy_from_user((void *)&migf->data + *pos, buf, len)) { 1048 ret = -EFAULT; 1049 goto out_unlock; 1050 } 1051 1052 *pos += len; 1053 ret = len; 1054 1055 dev_dbg(migf->mdev_state->vdev.dev, "%s received %zu, total %zu\n", 1056 __func__, len, migf->filled_size + len); 1057 1058 if (migf->filled_size < offsetof(struct mtty_data, ports) && 1059 migf->filled_size + len >= offsetof(struct mtty_data, ports)) { 1060 if (migf->data.magic != MTTY_MAGIC || migf->data.flags || 1061 migf->data.major_ver != MTTY_MAJOR_VER || 1062 migf->data.minor_ver != MTTY_MINOR_VER || 1063 migf->data.nr_ports != mdev_state->nr_ports) { 1064 dev_dbg(migf->mdev_state->vdev.dev, 1065 "%s failed validation\n", __func__); 1066 ret = -EFAULT; 1067 } else { 1068 dev_dbg(migf->mdev_state->vdev.dev, 1069 "%s header validated\n", __func__); 1070 } 1071 } 1072 1073 migf->filled_size += len; 1074 1075 out_unlock: 1076 mutex_unlock(&migf->lock); 1077 return ret; 1078 } 1079 1080 static const struct file_operations mtty_resume_fops = { 1081 .owner = THIS_MODULE, 1082 .write = mtty_resume_write, 1083 .release = mtty_release_migf, 1084 }; 1085 1086 static struct mtty_migration_file * 1087 mtty_resume_device_data(struct mdev_state *mdev_state) 1088 { 1089 struct mtty_migration_file *migf; 1090 int ret; 1091 1092 migf = kzalloc(sizeof(*migf), GFP_KERNEL_ACCOUNT); 1093 if (!migf) 1094 return ERR_PTR(-ENOMEM); 1095 1096 migf->filp = anon_inode_getfile("mtty_mig", &mtty_resume_fops, 1097 migf, O_WRONLY); 1098 if (IS_ERR(migf->filp)) { 1099 ret = PTR_ERR(migf->filp); 1100 kfree(migf); 1101 return ERR_PTR(ret); 1102 } 1103 1104 stream_open(migf->filp->f_inode, migf->filp); 1105 mutex_init(&migf->lock); 1106 migf->mdev_state = mdev_state; 1107 1108 mdev_state->resuming_migf = migf; 1109 1110 return migf; 1111 } 1112 1113 static struct file *mtty_step_state(struct mdev_state *mdev_state, 1114 enum vfio_device_mig_state new) 1115 { 1116 enum vfio_device_mig_state cur = mdev_state->state; 1117 1118 dev_dbg(mdev_state->vdev.dev, "%s: %d -> %d\n", __func__, cur, new); 1119 1120 /* 1121 * The following state transitions are no-op considering 1122 * mtty does not do DMA nor require any explicit start/stop. 1123 * 1124 * RUNNING -> RUNNING_P2P 1125 * RUNNING_P2P -> RUNNING 1126 * RUNNING_P2P -> STOP 1127 * PRE_COPY -> PRE_COPY_P2P 1128 * PRE_COPY_P2P -> PRE_COPY 1129 * STOP -> RUNNING_P2P 1130 */ 1131 if ((cur == VFIO_DEVICE_STATE_RUNNING && 1132 new == VFIO_DEVICE_STATE_RUNNING_P2P) || 1133 (cur == VFIO_DEVICE_STATE_RUNNING_P2P && 1134 (new == VFIO_DEVICE_STATE_RUNNING || 1135 new == VFIO_DEVICE_STATE_STOP)) || 1136 (cur == VFIO_DEVICE_STATE_PRE_COPY && 1137 new == VFIO_DEVICE_STATE_PRE_COPY_P2P) || 1138 (cur == VFIO_DEVICE_STATE_PRE_COPY_P2P && 1139 new == VFIO_DEVICE_STATE_PRE_COPY) || 1140 (cur == VFIO_DEVICE_STATE_STOP && 1141 new == VFIO_DEVICE_STATE_RUNNING_P2P)) 1142 return NULL; 1143 1144 /* 1145 * The following state transitions simply close migration files, 1146 * with the exception of RESUMING -> STOP, which needs to load 1147 * the state first. 1148 * 1149 * RESUMING -> STOP 1150 * PRE_COPY -> RUNNING 1151 * PRE_COPY_P2P -> RUNNING_P2P 1152 * STOP_COPY -> STOP 1153 */ 1154 if (cur == VFIO_DEVICE_STATE_RESUMING && 1155 new == VFIO_DEVICE_STATE_STOP) { 1156 int ret; 1157 1158 ret = mtty_load_state(mdev_state); 1159 if (ret) 1160 return ERR_PTR(ret); 1161 mtty_disable_files(mdev_state); 1162 return NULL; 1163 } 1164 1165 if ((cur == VFIO_DEVICE_STATE_PRE_COPY && 1166 new == VFIO_DEVICE_STATE_RUNNING) || 1167 (cur == VFIO_DEVICE_STATE_PRE_COPY_P2P && 1168 new == VFIO_DEVICE_STATE_RUNNING_P2P) || 1169 (cur == VFIO_DEVICE_STATE_STOP_COPY && 1170 new == VFIO_DEVICE_STATE_STOP)) { 1171 mtty_disable_files(mdev_state); 1172 return NULL; 1173 } 1174 1175 /* 1176 * The following state transitions return migration files. 1177 * 1178 * RUNNING -> PRE_COPY 1179 * RUNNING_P2P -> PRE_COPY_P2P 1180 * STOP -> STOP_COPY 1181 * STOP -> RESUMING 1182 * PRE_COPY_P2P -> STOP_COPY 1183 */ 1184 if ((cur == VFIO_DEVICE_STATE_RUNNING && 1185 new == VFIO_DEVICE_STATE_PRE_COPY) || 1186 (cur == VFIO_DEVICE_STATE_RUNNING_P2P && 1187 new == VFIO_DEVICE_STATE_PRE_COPY_P2P) || 1188 (cur == VFIO_DEVICE_STATE_STOP && 1189 new == VFIO_DEVICE_STATE_STOP_COPY) || 1190 (cur == VFIO_DEVICE_STATE_PRE_COPY_P2P && 1191 new == VFIO_DEVICE_STATE_STOP_COPY)) { 1192 struct mtty_migration_file *migf; 1193 1194 migf = mtty_save_device_data(mdev_state, new); 1195 if (IS_ERR(migf)) 1196 return ERR_CAST(migf); 1197 1198 if (migf) { 1199 get_file(migf->filp); 1200 1201 return migf->filp; 1202 } 1203 return NULL; 1204 } 1205 1206 if (cur == VFIO_DEVICE_STATE_STOP && 1207 new == VFIO_DEVICE_STATE_RESUMING) { 1208 struct mtty_migration_file *migf; 1209 1210 migf = mtty_resume_device_data(mdev_state); 1211 if (IS_ERR(migf)) 1212 return ERR_CAST(migf); 1213 1214 get_file(migf->filp); 1215 1216 return migf->filp; 1217 } 1218 1219 /* vfio_mig_get_next_state() does not use arcs other than the above */ 1220 WARN_ON(true); 1221 return ERR_PTR(-EINVAL); 1222 } 1223 1224 static struct file *mtty_set_state(struct vfio_device *vdev, 1225 enum vfio_device_mig_state new_state) 1226 { 1227 struct mdev_state *mdev_state = 1228 container_of(vdev, struct mdev_state, vdev); 1229 struct file *ret = NULL; 1230 1231 dev_dbg(vdev->dev, "%s -> %d\n", __func__, new_state); 1232 1233 mutex_lock(&mdev_state->state_mutex); 1234 while (mdev_state->state != new_state) { 1235 enum vfio_device_mig_state next_state; 1236 int rc = vfio_mig_get_next_state(vdev, mdev_state->state, 1237 new_state, &next_state); 1238 if (rc) { 1239 ret = ERR_PTR(rc); 1240 break; 1241 } 1242 1243 ret = mtty_step_state(mdev_state, next_state); 1244 if (IS_ERR(ret)) 1245 break; 1246 1247 mdev_state->state = next_state; 1248 1249 if (WARN_ON(ret && new_state != next_state)) { 1250 fput(ret); 1251 ret = ERR_PTR(-EINVAL); 1252 break; 1253 } 1254 } 1255 mtty_state_mutex_unlock(mdev_state); 1256 return ret; 1257 } 1258 1259 static int mtty_get_state(struct vfio_device *vdev, 1260 enum vfio_device_mig_state *current_state) 1261 { 1262 struct mdev_state *mdev_state = 1263 container_of(vdev, struct mdev_state, vdev); 1264 1265 mutex_lock(&mdev_state->state_mutex); 1266 *current_state = mdev_state->state; 1267 mtty_state_mutex_unlock(mdev_state); 1268 return 0; 1269 } 1270 1271 static int mtty_get_data_size(struct vfio_device *vdev, 1272 unsigned long *stop_copy_length) 1273 { 1274 struct mdev_state *mdev_state = 1275 container_of(vdev, struct mdev_state, vdev); 1276 1277 *stop_copy_length = mtty_data_size(mdev_state); 1278 return 0; 1279 } 1280 1281 static const struct vfio_migration_ops mtty_migration_ops = { 1282 .migration_set_state = mtty_set_state, 1283 .migration_get_state = mtty_get_state, 1284 .migration_get_data_size = mtty_get_data_size, 1285 }; 1286 1287 static int mtty_log_start(struct vfio_device *vdev, 1288 struct rb_root_cached *ranges, 1289 u32 nnodes, u64 *page_size) 1290 { 1291 return 0; 1292 } 1293 1294 static int mtty_log_stop(struct vfio_device *vdev) 1295 { 1296 return 0; 1297 } 1298 1299 static int mtty_log_read_and_clear(struct vfio_device *vdev, 1300 unsigned long iova, unsigned long length, 1301 struct iova_bitmap *dirty) 1302 { 1303 return 0; 1304 } 1305 1306 static const struct vfio_log_ops mtty_log_ops = { 1307 .log_start = mtty_log_start, 1308 .log_stop = mtty_log_stop, 1309 .log_read_and_clear = mtty_log_read_and_clear, 1310 }; 1311 1312 static int mtty_init_dev(struct vfio_device *vdev) 1313 { 1314 struct mdev_state *mdev_state = 1315 container_of(vdev, struct mdev_state, vdev); 1316 struct mdev_device *mdev = to_mdev_device(vdev->dev); 1317 struct mtty_type *type = 1318 container_of(mdev->type, struct mtty_type, type); 1319 int avail_ports = atomic_read(&mdev_avail_ports); 1320 int ret; 1321 1322 do { 1323 if (avail_ports < type->nr_ports) 1324 return -ENOSPC; 1325 } while (!atomic_try_cmpxchg(&mdev_avail_ports, 1326 &avail_ports, 1327 avail_ports - type->nr_ports)); 1328 1329 mdev_state->nr_ports = type->nr_ports; 1330 mdev_state->irq_index = -1; 1331 mdev_state->s[0].max_fifo_size = MAX_FIFO_SIZE; 1332 mdev_state->s[1].max_fifo_size = MAX_FIFO_SIZE; 1333 mutex_init(&mdev_state->rxtx_lock); 1334 1335 mdev_state->vconfig = kzalloc(MTTY_CONFIG_SPACE_SIZE, GFP_KERNEL); 1336 if (!mdev_state->vconfig) { 1337 ret = -ENOMEM; 1338 goto err_nr_ports; 1339 } 1340 1341 mutex_init(&mdev_state->ops_lock); 1342 mdev_state->mdev = mdev; 1343 mtty_create_config_space(mdev_state); 1344 1345 mutex_init(&mdev_state->state_mutex); 1346 mutex_init(&mdev_state->reset_mutex); 1347 vdev->migration_flags = VFIO_MIGRATION_STOP_COPY | 1348 VFIO_MIGRATION_P2P | 1349 VFIO_MIGRATION_PRE_COPY; 1350 vdev->mig_ops = &mtty_migration_ops; 1351 vdev->log_ops = &mtty_log_ops; 1352 mdev_state->state = VFIO_DEVICE_STATE_RUNNING; 1353 1354 return 0; 1355 1356 err_nr_ports: 1357 atomic_add(type->nr_ports, &mdev_avail_ports); 1358 return ret; 1359 } 1360 1361 static int mtty_probe(struct mdev_device *mdev) 1362 { 1363 struct mdev_state *mdev_state; 1364 int ret; 1365 1366 mdev_state = vfio_alloc_device(mdev_state, vdev, &mdev->dev, 1367 &mtty_dev_ops); 1368 if (IS_ERR(mdev_state)) 1369 return PTR_ERR(mdev_state); 1370 1371 ret = vfio_register_emulated_iommu_dev(&mdev_state->vdev); 1372 if (ret) 1373 goto err_put_vdev; 1374 dev_set_drvdata(&mdev->dev, mdev_state); 1375 return 0; 1376 1377 err_put_vdev: 1378 vfio_put_device(&mdev_state->vdev); 1379 return ret; 1380 } 1381 1382 static void mtty_release_dev(struct vfio_device *vdev) 1383 { 1384 struct mdev_state *mdev_state = 1385 container_of(vdev, struct mdev_state, vdev); 1386 1387 mutex_destroy(&mdev_state->reset_mutex); 1388 mutex_destroy(&mdev_state->state_mutex); 1389 atomic_add(mdev_state->nr_ports, &mdev_avail_ports); 1390 kfree(mdev_state->vconfig); 1391 } 1392 1393 static void mtty_remove(struct mdev_device *mdev) 1394 { 1395 struct mdev_state *mdev_state = dev_get_drvdata(&mdev->dev); 1396 1397 vfio_unregister_group_dev(&mdev_state->vdev); 1398 vfio_put_device(&mdev_state->vdev); 1399 } 1400 1401 static int mtty_reset(struct mdev_state *mdev_state) 1402 { 1403 pr_info("%s: called\n", __func__); 1404 1405 mutex_lock(&mdev_state->reset_mutex); 1406 mdev_state->deferred_reset = true; 1407 if (!mutex_trylock(&mdev_state->state_mutex)) { 1408 mutex_unlock(&mdev_state->reset_mutex); 1409 return 0; 1410 } 1411 mutex_unlock(&mdev_state->reset_mutex); 1412 mtty_state_mutex_unlock(mdev_state); 1413 1414 return 0; 1415 } 1416 1417 static ssize_t mtty_read(struct vfio_device *vdev, char __user *buf, 1418 size_t count, loff_t *ppos) 1419 { 1420 struct mdev_state *mdev_state = 1421 container_of(vdev, struct mdev_state, vdev); 1422 unsigned int done = 0; 1423 int ret; 1424 1425 while (count) { 1426 size_t filled; 1427 1428 if (count >= 4 && !(*ppos % 4)) { 1429 u32 val; 1430 1431 ret = mdev_access(mdev_state, (u8 *)&val, sizeof(val), 1432 *ppos, false); 1433 if (ret <= 0) 1434 goto read_err; 1435 1436 if (copy_to_user(buf, &val, sizeof(val))) 1437 goto read_err; 1438 1439 filled = 4; 1440 } else if (count >= 2 && !(*ppos % 2)) { 1441 u16 val; 1442 1443 ret = mdev_access(mdev_state, (u8 *)&val, sizeof(val), 1444 *ppos, false); 1445 if (ret <= 0) 1446 goto read_err; 1447 1448 if (copy_to_user(buf, &val, sizeof(val))) 1449 goto read_err; 1450 1451 filled = 2; 1452 } else { 1453 u8 val; 1454 1455 ret = mdev_access(mdev_state, (u8 *)&val, sizeof(val), 1456 *ppos, false); 1457 if (ret <= 0) 1458 goto read_err; 1459 1460 if (copy_to_user(buf, &val, sizeof(val))) 1461 goto read_err; 1462 1463 filled = 1; 1464 } 1465 1466 count -= filled; 1467 done += filled; 1468 *ppos += filled; 1469 buf += filled; 1470 } 1471 1472 return done; 1473 1474 read_err: 1475 return -EFAULT; 1476 } 1477 1478 static ssize_t mtty_write(struct vfio_device *vdev, const char __user *buf, 1479 size_t count, loff_t *ppos) 1480 { 1481 struct mdev_state *mdev_state = 1482 container_of(vdev, struct mdev_state, vdev); 1483 unsigned int done = 0; 1484 int ret; 1485 1486 while (count) { 1487 size_t filled; 1488 1489 if (count >= 4 && !(*ppos % 4)) { 1490 u32 val; 1491 1492 if (copy_from_user(&val, buf, sizeof(val))) 1493 goto write_err; 1494 1495 ret = mdev_access(mdev_state, (u8 *)&val, sizeof(val), 1496 *ppos, true); 1497 if (ret <= 0) 1498 goto write_err; 1499 1500 filled = 4; 1501 } else if (count >= 2 && !(*ppos % 2)) { 1502 u16 val; 1503 1504 if (copy_from_user(&val, buf, sizeof(val))) 1505 goto write_err; 1506 1507 ret = mdev_access(mdev_state, (u8 *)&val, sizeof(val), 1508 *ppos, true); 1509 if (ret <= 0) 1510 goto write_err; 1511 1512 filled = 2; 1513 } else { 1514 u8 val; 1515 1516 if (copy_from_user(&val, buf, sizeof(val))) 1517 goto write_err; 1518 1519 ret = mdev_access(mdev_state, (u8 *)&val, sizeof(val), 1520 *ppos, true); 1521 if (ret <= 0) 1522 goto write_err; 1523 1524 filled = 1; 1525 } 1526 count -= filled; 1527 done += filled; 1528 *ppos += filled; 1529 buf += filled; 1530 } 1531 1532 return done; 1533 write_err: 1534 return -EFAULT; 1535 } 1536 1537 static void mtty_disable_intx(struct mdev_state *mdev_state) 1538 { 1539 if (mdev_state->intx_evtfd) { 1540 eventfd_ctx_put(mdev_state->intx_evtfd); 1541 mdev_state->intx_evtfd = NULL; 1542 mdev_state->intx_mask = false; 1543 mdev_state->irq_index = -1; 1544 } 1545 } 1546 1547 static void mtty_disable_msi(struct mdev_state *mdev_state) 1548 { 1549 if (mdev_state->msi_evtfd) { 1550 eventfd_ctx_put(mdev_state->msi_evtfd); 1551 mdev_state->msi_evtfd = NULL; 1552 mdev_state->irq_index = -1; 1553 } 1554 } 1555 1556 static int mtty_set_irqs(struct mdev_state *mdev_state, uint32_t flags, 1557 unsigned int index, unsigned int start, 1558 unsigned int count, void *data) 1559 { 1560 int ret = 0; 1561 1562 mutex_lock(&mdev_state->ops_lock); 1563 switch (index) { 1564 case VFIO_PCI_INTX_IRQ_INDEX: 1565 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 1566 case VFIO_IRQ_SET_ACTION_MASK: 1567 if (!is_intx(mdev_state) || start != 0 || count != 1) { 1568 ret = -EINVAL; 1569 break; 1570 } 1571 1572 if (flags & VFIO_IRQ_SET_DATA_NONE) { 1573 mdev_state->intx_mask = true; 1574 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 1575 uint8_t mask = *(uint8_t *)data; 1576 1577 if (mask) 1578 mdev_state->intx_mask = true; 1579 } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 1580 ret = -ENOTTY; /* No support for mask fd */ 1581 } 1582 break; 1583 case VFIO_IRQ_SET_ACTION_UNMASK: 1584 if (!is_intx(mdev_state) || start != 0 || count != 1) { 1585 ret = -EINVAL; 1586 break; 1587 } 1588 1589 if (flags & VFIO_IRQ_SET_DATA_NONE) { 1590 mdev_state->intx_mask = false; 1591 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 1592 uint8_t mask = *(uint8_t *)data; 1593 1594 if (mask) 1595 mdev_state->intx_mask = false; 1596 } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 1597 ret = -ENOTTY; /* No support for unmask fd */ 1598 } 1599 break; 1600 case VFIO_IRQ_SET_ACTION_TRIGGER: 1601 if (is_intx(mdev_state) && !count && 1602 (flags & VFIO_IRQ_SET_DATA_NONE)) { 1603 mtty_disable_intx(mdev_state); 1604 break; 1605 } 1606 1607 if (!(is_intx(mdev_state) || is_noirq(mdev_state)) || 1608 start != 0 || count != 1) { 1609 ret = -EINVAL; 1610 break; 1611 } 1612 1613 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 1614 int fd = *(int *)data; 1615 struct eventfd_ctx *evt; 1616 1617 mtty_disable_intx(mdev_state); 1618 1619 if (fd < 0) 1620 break; 1621 1622 evt = eventfd_ctx_fdget(fd); 1623 if (IS_ERR(evt)) { 1624 ret = PTR_ERR(evt); 1625 break; 1626 } 1627 mdev_state->intx_evtfd = evt; 1628 mdev_state->irq_index = index; 1629 break; 1630 } 1631 1632 if (!is_intx(mdev_state)) { 1633 ret = -EINVAL; 1634 break; 1635 } 1636 1637 if (flags & VFIO_IRQ_SET_DATA_NONE) { 1638 mtty_trigger_interrupt(mdev_state); 1639 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 1640 uint8_t trigger = *(uint8_t *)data; 1641 1642 if (trigger) 1643 mtty_trigger_interrupt(mdev_state); 1644 } 1645 break; 1646 } 1647 break; 1648 case VFIO_PCI_MSI_IRQ_INDEX: 1649 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 1650 case VFIO_IRQ_SET_ACTION_MASK: 1651 case VFIO_IRQ_SET_ACTION_UNMASK: 1652 ret = -ENOTTY; 1653 break; 1654 case VFIO_IRQ_SET_ACTION_TRIGGER: 1655 if (is_msi(mdev_state) && !count && 1656 (flags & VFIO_IRQ_SET_DATA_NONE)) { 1657 mtty_disable_msi(mdev_state); 1658 break; 1659 } 1660 1661 if (!(is_msi(mdev_state) || is_noirq(mdev_state)) || 1662 start != 0 || count != 1) { 1663 ret = -EINVAL; 1664 break; 1665 } 1666 1667 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 1668 int fd = *(int *)data; 1669 struct eventfd_ctx *evt; 1670 1671 mtty_disable_msi(mdev_state); 1672 1673 if (fd < 0) 1674 break; 1675 1676 evt = eventfd_ctx_fdget(fd); 1677 if (IS_ERR(evt)) { 1678 ret = PTR_ERR(evt); 1679 break; 1680 } 1681 mdev_state->msi_evtfd = evt; 1682 mdev_state->irq_index = index; 1683 break; 1684 } 1685 1686 if (!is_msi(mdev_state)) { 1687 ret = -EINVAL; 1688 break; 1689 } 1690 1691 if (flags & VFIO_IRQ_SET_DATA_NONE) { 1692 mtty_trigger_interrupt(mdev_state); 1693 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 1694 uint8_t trigger = *(uint8_t *)data; 1695 1696 if (trigger) 1697 mtty_trigger_interrupt(mdev_state); 1698 } 1699 break; 1700 } 1701 break; 1702 case VFIO_PCI_MSIX_IRQ_INDEX: 1703 dev_dbg(mdev_state->vdev.dev, "%s: MSIX_IRQ\n", __func__); 1704 ret = -ENOTTY; 1705 break; 1706 case VFIO_PCI_ERR_IRQ_INDEX: 1707 dev_dbg(mdev_state->vdev.dev, "%s: ERR_IRQ\n", __func__); 1708 ret = -ENOTTY; 1709 break; 1710 case VFIO_PCI_REQ_IRQ_INDEX: 1711 dev_dbg(mdev_state->vdev.dev, "%s: REQ_IRQ\n", __func__); 1712 ret = -ENOTTY; 1713 break; 1714 } 1715 1716 mutex_unlock(&mdev_state->ops_lock); 1717 return ret; 1718 } 1719 1720 static int mtty_get_region_info(struct mdev_state *mdev_state, 1721 struct vfio_region_info *region_info, 1722 u16 *cap_type_id, void **cap_type) 1723 { 1724 unsigned int size = 0; 1725 u32 bar_index; 1726 1727 bar_index = region_info->index; 1728 if (bar_index >= VFIO_PCI_NUM_REGIONS) 1729 return -EINVAL; 1730 1731 mutex_lock(&mdev_state->ops_lock); 1732 1733 switch (bar_index) { 1734 case VFIO_PCI_CONFIG_REGION_INDEX: 1735 size = MTTY_CONFIG_SPACE_SIZE; 1736 break; 1737 case VFIO_PCI_BAR0_REGION_INDEX: 1738 size = MTTY_IO_BAR_SIZE; 1739 break; 1740 case VFIO_PCI_BAR1_REGION_INDEX: 1741 if (mdev_state->nr_ports == 2) 1742 size = MTTY_IO_BAR_SIZE; 1743 break; 1744 default: 1745 size = 0; 1746 break; 1747 } 1748 1749 mdev_state->region_info[bar_index].size = size; 1750 mdev_state->region_info[bar_index].vfio_offset = 1751 MTTY_VFIO_PCI_INDEX_TO_OFFSET(bar_index); 1752 1753 region_info->size = size; 1754 region_info->offset = MTTY_VFIO_PCI_INDEX_TO_OFFSET(bar_index); 1755 region_info->flags = VFIO_REGION_INFO_FLAG_READ | 1756 VFIO_REGION_INFO_FLAG_WRITE; 1757 mutex_unlock(&mdev_state->ops_lock); 1758 return 0; 1759 } 1760 1761 static int mtty_get_irq_info(struct vfio_irq_info *irq_info) 1762 { 1763 if (irq_info->index != VFIO_PCI_INTX_IRQ_INDEX && 1764 irq_info->index != VFIO_PCI_MSI_IRQ_INDEX) 1765 return -EINVAL; 1766 1767 irq_info->flags = VFIO_IRQ_INFO_EVENTFD; 1768 irq_info->count = 1; 1769 1770 if (irq_info->index == VFIO_PCI_INTX_IRQ_INDEX) 1771 irq_info->flags |= VFIO_IRQ_INFO_MASKABLE | 1772 VFIO_IRQ_INFO_AUTOMASKED; 1773 else 1774 irq_info->flags |= VFIO_IRQ_INFO_NORESIZE; 1775 1776 return 0; 1777 } 1778 1779 static int mtty_get_device_info(struct vfio_device_info *dev_info) 1780 { 1781 dev_info->flags = VFIO_DEVICE_FLAGS_PCI; 1782 dev_info->num_regions = VFIO_PCI_NUM_REGIONS; 1783 dev_info->num_irqs = VFIO_PCI_NUM_IRQS; 1784 1785 return 0; 1786 } 1787 1788 static long mtty_ioctl(struct vfio_device *vdev, unsigned int cmd, 1789 unsigned long arg) 1790 { 1791 struct mdev_state *mdev_state = 1792 container_of(vdev, struct mdev_state, vdev); 1793 int ret = 0; 1794 unsigned long minsz; 1795 1796 switch (cmd) { 1797 case VFIO_DEVICE_GET_INFO: 1798 { 1799 struct vfio_device_info info; 1800 1801 minsz = offsetofend(struct vfio_device_info, num_irqs); 1802 1803 if (copy_from_user(&info, (void __user *)arg, minsz)) 1804 return -EFAULT; 1805 1806 if (info.argsz < minsz) 1807 return -EINVAL; 1808 1809 ret = mtty_get_device_info(&info); 1810 if (ret) 1811 return ret; 1812 1813 memcpy(&mdev_state->dev_info, &info, sizeof(info)); 1814 1815 if (copy_to_user((void __user *)arg, &info, minsz)) 1816 return -EFAULT; 1817 1818 return 0; 1819 } 1820 case VFIO_DEVICE_GET_REGION_INFO: 1821 { 1822 struct vfio_region_info info; 1823 u16 cap_type_id = 0; 1824 void *cap_type = NULL; 1825 1826 minsz = offsetofend(struct vfio_region_info, offset); 1827 1828 if (copy_from_user(&info, (void __user *)arg, minsz)) 1829 return -EFAULT; 1830 1831 if (info.argsz < minsz) 1832 return -EINVAL; 1833 1834 ret = mtty_get_region_info(mdev_state, &info, &cap_type_id, 1835 &cap_type); 1836 if (ret) 1837 return ret; 1838 1839 if (copy_to_user((void __user *)arg, &info, minsz)) 1840 return -EFAULT; 1841 1842 return 0; 1843 } 1844 1845 case VFIO_DEVICE_GET_IRQ_INFO: 1846 { 1847 struct vfio_irq_info info; 1848 1849 minsz = offsetofend(struct vfio_irq_info, count); 1850 1851 if (copy_from_user(&info, (void __user *)arg, minsz)) 1852 return -EFAULT; 1853 1854 if ((info.argsz < minsz) || 1855 (info.index >= mdev_state->dev_info.num_irqs)) 1856 return -EINVAL; 1857 1858 ret = mtty_get_irq_info(&info); 1859 if (ret) 1860 return ret; 1861 1862 if (copy_to_user((void __user *)arg, &info, minsz)) 1863 return -EFAULT; 1864 1865 return 0; 1866 } 1867 case VFIO_DEVICE_SET_IRQS: 1868 { 1869 struct vfio_irq_set hdr; 1870 u8 *data = NULL, *ptr = NULL; 1871 size_t data_size = 0; 1872 1873 minsz = offsetofend(struct vfio_irq_set, count); 1874 1875 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 1876 return -EFAULT; 1877 1878 ret = vfio_set_irqs_validate_and_prepare(&hdr, 1879 mdev_state->dev_info.num_irqs, 1880 VFIO_PCI_NUM_IRQS, 1881 &data_size); 1882 if (ret) 1883 return ret; 1884 1885 if (data_size) { 1886 ptr = data = memdup_user((void __user *)(arg + minsz), 1887 data_size); 1888 if (IS_ERR(data)) 1889 return PTR_ERR(data); 1890 } 1891 1892 ret = mtty_set_irqs(mdev_state, hdr.flags, hdr.index, hdr.start, 1893 hdr.count, data); 1894 1895 kfree(ptr); 1896 return ret; 1897 } 1898 case VFIO_DEVICE_RESET: 1899 return mtty_reset(mdev_state); 1900 } 1901 return -ENOTTY; 1902 } 1903 1904 static ssize_t 1905 sample_mdev_dev_show(struct device *dev, struct device_attribute *attr, 1906 char *buf) 1907 { 1908 return sprintf(buf, "This is MDEV %s\n", dev_name(dev)); 1909 } 1910 1911 static DEVICE_ATTR_RO(sample_mdev_dev); 1912 1913 static struct attribute *mdev_dev_attrs[] = { 1914 &dev_attr_sample_mdev_dev.attr, 1915 NULL, 1916 }; 1917 1918 static const struct attribute_group mdev_dev_group = { 1919 .name = "vendor", 1920 .attrs = mdev_dev_attrs, 1921 }; 1922 1923 static const struct attribute_group *mdev_dev_groups[] = { 1924 &mdev_dev_group, 1925 NULL, 1926 }; 1927 1928 static unsigned int mtty_get_available(struct mdev_type *mtype) 1929 { 1930 struct mtty_type *type = container_of(mtype, struct mtty_type, type); 1931 1932 return atomic_read(&mdev_avail_ports) / type->nr_ports; 1933 } 1934 1935 static void mtty_close(struct vfio_device *vdev) 1936 { 1937 struct mdev_state *mdev_state = 1938 container_of(vdev, struct mdev_state, vdev); 1939 1940 mtty_disable_files(mdev_state); 1941 mtty_disable_intx(mdev_state); 1942 mtty_disable_msi(mdev_state); 1943 } 1944 1945 static const struct vfio_device_ops mtty_dev_ops = { 1946 .name = "vfio-mtty", 1947 .init = mtty_init_dev, 1948 .release = mtty_release_dev, 1949 .read = mtty_read, 1950 .write = mtty_write, 1951 .ioctl = mtty_ioctl, 1952 .bind_iommufd = vfio_iommufd_emulated_bind, 1953 .unbind_iommufd = vfio_iommufd_emulated_unbind, 1954 .attach_ioas = vfio_iommufd_emulated_attach_ioas, 1955 .detach_ioas = vfio_iommufd_emulated_detach_ioas, 1956 .close_device = mtty_close, 1957 }; 1958 1959 static struct mdev_driver mtty_driver = { 1960 .device_api = VFIO_DEVICE_API_PCI_STRING, 1961 .driver = { 1962 .name = "mtty", 1963 .owner = THIS_MODULE, 1964 .mod_name = KBUILD_MODNAME, 1965 .dev_groups = mdev_dev_groups, 1966 }, 1967 .probe = mtty_probe, 1968 .remove = mtty_remove, 1969 .get_available = mtty_get_available, 1970 }; 1971 1972 static void mtty_device_release(struct device *dev) 1973 { 1974 dev_dbg(dev, "mtty: released\n"); 1975 } 1976 1977 static int __init mtty_dev_init(void) 1978 { 1979 int ret = 0; 1980 1981 pr_info("mtty_dev: %s\n", __func__); 1982 1983 memset(&mtty_dev, 0, sizeof(mtty_dev)); 1984 1985 idr_init(&mtty_dev.vd_idr); 1986 1987 ret = alloc_chrdev_region(&mtty_dev.vd_devt, 0, MINORMASK + 1, 1988 MTTY_NAME); 1989 1990 if (ret < 0) { 1991 pr_err("Error: failed to register mtty_dev, err:%d\n", ret); 1992 return ret; 1993 } 1994 1995 cdev_init(&mtty_dev.vd_cdev, &vd_fops); 1996 cdev_add(&mtty_dev.vd_cdev, mtty_dev.vd_devt, MINORMASK + 1); 1997 1998 pr_info("major_number:%d\n", MAJOR(mtty_dev.vd_devt)); 1999 2000 ret = mdev_register_driver(&mtty_driver); 2001 if (ret) 2002 goto err_cdev; 2003 2004 mtty_dev.vd_class = class_create(MTTY_CLASS_NAME); 2005 2006 if (IS_ERR(mtty_dev.vd_class)) { 2007 pr_err("Error: failed to register mtty_dev class\n"); 2008 ret = PTR_ERR(mtty_dev.vd_class); 2009 goto err_driver; 2010 } 2011 2012 mtty_dev.dev.class = mtty_dev.vd_class; 2013 mtty_dev.dev.release = mtty_device_release; 2014 dev_set_name(&mtty_dev.dev, "%s", MTTY_NAME); 2015 2016 ret = device_register(&mtty_dev.dev); 2017 if (ret) 2018 goto err_put; 2019 2020 ret = mdev_register_parent(&mtty_dev.parent, &mtty_dev.dev, 2021 &mtty_driver, mtty_mdev_types, 2022 ARRAY_SIZE(mtty_mdev_types)); 2023 if (ret) 2024 goto err_device; 2025 return 0; 2026 2027 err_device: 2028 device_del(&mtty_dev.dev); 2029 err_put: 2030 put_device(&mtty_dev.dev); 2031 class_destroy(mtty_dev.vd_class); 2032 err_driver: 2033 mdev_unregister_driver(&mtty_driver); 2034 err_cdev: 2035 cdev_del(&mtty_dev.vd_cdev); 2036 unregister_chrdev_region(mtty_dev.vd_devt, MINORMASK + 1); 2037 return ret; 2038 } 2039 2040 static void __exit mtty_dev_exit(void) 2041 { 2042 mtty_dev.dev.bus = NULL; 2043 mdev_unregister_parent(&mtty_dev.parent); 2044 2045 device_unregister(&mtty_dev.dev); 2046 idr_destroy(&mtty_dev.vd_idr); 2047 mdev_unregister_driver(&mtty_driver); 2048 cdev_del(&mtty_dev.vd_cdev); 2049 unregister_chrdev_region(mtty_dev.vd_devt, MINORMASK + 1); 2050 class_destroy(mtty_dev.vd_class); 2051 mtty_dev.vd_class = NULL; 2052 pr_info("mtty_dev: Unloaded!\n"); 2053 } 2054 2055 module_init(mtty_dev_init) 2056 module_exit(mtty_dev_exit) 2057 2058 MODULE_LICENSE("GPL v2"); 2059 MODULE_DESCRIPTION("Test driver that simulate serial port over PCI"); 2060 MODULE_VERSION(VERSION_STRING); 2061 MODULE_AUTHOR(DRIVER_AUTHOR); 2062