1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Mediated virtual PCI serial host device driver 4 * 5 * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. 6 * Author: Neo Jia <cjia@nvidia.com> 7 * Kirti Wankhede <kwankhede@nvidia.com> 8 * 9 * Sample driver that creates mdev device that simulates serial port over PCI 10 * card. 11 */ 12 13 #include <linux/init.h> 14 #include <linux/module.h> 15 #include <linux/kernel.h> 16 #include <linux/fs.h> 17 #include <linux/poll.h> 18 #include <linux/slab.h> 19 #include <linux/cdev.h> 20 #include <linux/sched.h> 21 #include <linux/wait.h> 22 #include <linux/vfio.h> 23 #include <linux/iommu.h> 24 #include <linux/sysfs.h> 25 #include <linux/ctype.h> 26 #include <linux/file.h> 27 #include <linux/mdev.h> 28 #include <linux/pci.h> 29 #include <linux/serial.h> 30 #include <uapi/linux/serial_reg.h> 31 #include <linux/eventfd.h> 32 #include <linux/anon_inodes.h> 33 34 /* 35 * #defines 36 */ 37 38 #define VERSION_STRING "0.1" 39 #define DRIVER_AUTHOR "NVIDIA Corporation" 40 41 #define MTTY_CLASS_NAME "mtty" 42 43 #define MTTY_NAME "mtty" 44 45 #define MTTY_STRING_LEN 16 46 47 #define MTTY_CONFIG_SPACE_SIZE 0xff 48 #define MTTY_IO_BAR_SIZE 0x8 49 #define MTTY_MMIO_BAR_SIZE 0x100000 50 51 #define STORE_LE16(addr, val) (*(u16 *)addr = val) 52 #define STORE_LE32(addr, val) (*(u32 *)addr = val) 53 54 #define MAX_FIFO_SIZE 16 55 56 #define CIRCULAR_BUF_INC_IDX(idx) (idx = (idx + 1) & (MAX_FIFO_SIZE - 1)) 57 58 #define MTTY_VFIO_PCI_OFFSET_SHIFT 40 59 60 #define MTTY_VFIO_PCI_OFFSET_TO_INDEX(off) (off >> MTTY_VFIO_PCI_OFFSET_SHIFT) 61 #define MTTY_VFIO_PCI_INDEX_TO_OFFSET(index) \ 62 ((u64)(index) << MTTY_VFIO_PCI_OFFSET_SHIFT) 63 #define MTTY_VFIO_PCI_OFFSET_MASK \ 64 (((u64)(1) << MTTY_VFIO_PCI_OFFSET_SHIFT) - 1) 65 #define MAX_MTTYS 24 66 67 /* 68 * Global Structures 69 */ 70 71 static struct mtty_dev { 72 dev_t vd_devt; 73 struct class *vd_class; 74 struct cdev vd_cdev; 75 struct idr vd_idr; 76 struct device dev; 77 struct mdev_parent parent; 78 } mtty_dev; 79 80 struct mdev_region_info { 81 u64 start; 82 u64 phys_start; 83 u32 size; 84 u64 vfio_offset; 85 }; 86 87 #if defined(DEBUG_REGS) 88 static const char *wr_reg[] = { 89 "TX", 90 "IER", 91 "FCR", 92 "LCR", 93 "MCR", 94 "LSR", 95 "MSR", 96 "SCR" 97 }; 98 99 static const char *rd_reg[] = { 100 "RX", 101 "IER", 102 "IIR", 103 "LCR", 104 "MCR", 105 "LSR", 106 "MSR", 107 "SCR" 108 }; 109 #endif 110 111 /* loop back buffer */ 112 struct rxtx { 113 u8 fifo[MAX_FIFO_SIZE]; 114 u8 head, tail; 115 u8 count; 116 }; 117 118 struct serial_port { 119 u8 uart_reg[8]; /* 8 registers */ 120 struct rxtx rxtx; /* loop back buffer */ 121 bool dlab; 122 bool overrun; 123 u16 divisor; 124 u8 fcr; /* FIFO control register */ 125 u8 max_fifo_size; 126 u8 intr_trigger_level; /* interrupt trigger level */ 127 }; 128 129 struct mtty_data { 130 u64 magic; 131 #define MTTY_MAGIC 0x7e9d09898c3e2c4e /* Nothing clever, just random */ 132 u32 major_ver; 133 #define MTTY_MAJOR_VER 1 134 u32 minor_ver; 135 #define MTTY_MINOR_VER 0 136 u32 nr_ports; 137 u32 flags; 138 struct serial_port ports[2]; 139 }; 140 141 struct mdev_state; 142 143 struct mtty_migration_file { 144 struct file *filp; 145 struct mutex lock; 146 struct mdev_state *mdev_state; 147 struct mtty_data data; 148 ssize_t filled_size; 149 u8 disabled:1; 150 }; 151 152 /* State of each mdev device */ 153 struct mdev_state { 154 struct vfio_device vdev; 155 struct eventfd_ctx *intx_evtfd; 156 struct eventfd_ctx *msi_evtfd; 157 int irq_index; 158 u8 *vconfig; 159 struct mutex ops_lock; 160 struct mdev_device *mdev; 161 struct mdev_region_info region_info[VFIO_PCI_NUM_REGIONS]; 162 u32 bar_mask[VFIO_PCI_NUM_REGIONS]; 163 struct list_head next; 164 struct serial_port s[2]; 165 struct mutex rxtx_lock; 166 struct vfio_device_info dev_info; 167 int nr_ports; 168 enum vfio_device_mig_state state; 169 struct mutex state_mutex; 170 struct mutex reset_mutex; 171 struct mtty_migration_file *saving_migf; 172 struct mtty_migration_file *resuming_migf; 173 u8 deferred_reset:1; 174 u8 intx_mask:1; 175 }; 176 177 static struct mtty_type { 178 struct mdev_type type; 179 int nr_ports; 180 } mtty_types[2] = { 181 { .nr_ports = 1, .type.sysfs_name = "1", 182 .type.pretty_name = "Single port serial" }, 183 { .nr_ports = 2, .type.sysfs_name = "2", 184 .type.pretty_name = "Dual port serial" }, 185 }; 186 187 static struct mdev_type *mtty_mdev_types[] = { 188 &mtty_types[0].type, 189 &mtty_types[1].type, 190 }; 191 192 static atomic_t mdev_avail_ports = ATOMIC_INIT(MAX_MTTYS); 193 194 static const struct file_operations vd_fops = { 195 .owner = THIS_MODULE, 196 }; 197 198 static const struct vfio_device_ops mtty_dev_ops; 199 200 /* Helper functions */ 201 202 static void dump_buffer(u8 *buf, uint32_t count) 203 { 204 #if defined(DEBUG) 205 int i; 206 207 pr_info("Buffer:\n"); 208 for (i = 0; i < count; i++) { 209 pr_info("%2x ", *(buf + i)); 210 if ((i + 1) % 16 == 0) 211 pr_info("\n"); 212 } 213 #endif 214 } 215 216 static bool is_intx(struct mdev_state *mdev_state) 217 { 218 return mdev_state->irq_index == VFIO_PCI_INTX_IRQ_INDEX; 219 } 220 221 static bool is_msi(struct mdev_state *mdev_state) 222 { 223 return mdev_state->irq_index == VFIO_PCI_MSI_IRQ_INDEX; 224 } 225 226 static bool is_noirq(struct mdev_state *mdev_state) 227 { 228 return !is_intx(mdev_state) && !is_msi(mdev_state); 229 } 230 231 static void mtty_trigger_interrupt(struct mdev_state *mdev_state) 232 { 233 lockdep_assert_held(&mdev_state->ops_lock); 234 235 if (is_msi(mdev_state)) { 236 if (mdev_state->msi_evtfd) 237 eventfd_signal(mdev_state->msi_evtfd); 238 } else if (is_intx(mdev_state)) { 239 if (mdev_state->intx_evtfd && !mdev_state->intx_mask) { 240 eventfd_signal(mdev_state->intx_evtfd); 241 mdev_state->intx_mask = true; 242 } 243 } 244 } 245 246 static void mtty_create_config_space(struct mdev_state *mdev_state) 247 { 248 /* PCI dev ID */ 249 STORE_LE32((u32 *) &mdev_state->vconfig[0x0], 0x32534348); 250 251 /* Control: I/O+, Mem-, BusMaster- */ 252 STORE_LE16((u16 *) &mdev_state->vconfig[0x4], 0x0001); 253 254 /* Status: capabilities list absent */ 255 STORE_LE16((u16 *) &mdev_state->vconfig[0x6], 0x0200); 256 257 /* Rev ID */ 258 mdev_state->vconfig[0x8] = 0x10; 259 260 /* programming interface class : 16550-compatible serial controller */ 261 mdev_state->vconfig[0x9] = 0x02; 262 263 /* Sub class : 00 */ 264 mdev_state->vconfig[0xa] = 0x00; 265 266 /* Base class : Simple Communication controllers */ 267 mdev_state->vconfig[0xb] = 0x07; 268 269 /* base address registers */ 270 /* BAR0: IO space */ 271 STORE_LE32((u32 *) &mdev_state->vconfig[0x10], 0x000001); 272 mdev_state->bar_mask[0] = ~(MTTY_IO_BAR_SIZE) + 1; 273 274 if (mdev_state->nr_ports == 2) { 275 /* BAR1: IO space */ 276 STORE_LE32((u32 *) &mdev_state->vconfig[0x14], 0x000001); 277 mdev_state->bar_mask[1] = ~(MTTY_IO_BAR_SIZE) + 1; 278 } 279 280 /* Subsystem ID */ 281 STORE_LE32((u32 *) &mdev_state->vconfig[0x2c], 0x32534348); 282 283 mdev_state->vconfig[0x34] = 0x00; /* Cap Ptr */ 284 mdev_state->vconfig[0x3d] = 0x01; /* interrupt pin (INTA#) */ 285 286 /* Vendor specific data */ 287 mdev_state->vconfig[0x40] = 0x23; 288 mdev_state->vconfig[0x43] = 0x80; 289 mdev_state->vconfig[0x44] = 0x23; 290 mdev_state->vconfig[0x48] = 0x23; 291 mdev_state->vconfig[0x4c] = 0x23; 292 293 mdev_state->vconfig[0x60] = 0x50; 294 mdev_state->vconfig[0x61] = 0x43; 295 mdev_state->vconfig[0x62] = 0x49; 296 mdev_state->vconfig[0x63] = 0x20; 297 mdev_state->vconfig[0x64] = 0x53; 298 mdev_state->vconfig[0x65] = 0x65; 299 mdev_state->vconfig[0x66] = 0x72; 300 mdev_state->vconfig[0x67] = 0x69; 301 mdev_state->vconfig[0x68] = 0x61; 302 mdev_state->vconfig[0x69] = 0x6c; 303 mdev_state->vconfig[0x6a] = 0x2f; 304 mdev_state->vconfig[0x6b] = 0x55; 305 mdev_state->vconfig[0x6c] = 0x41; 306 mdev_state->vconfig[0x6d] = 0x52; 307 mdev_state->vconfig[0x6e] = 0x54; 308 } 309 310 static void handle_pci_cfg_write(struct mdev_state *mdev_state, u16 offset, 311 u8 *buf, u32 count) 312 { 313 u32 cfg_addr, bar_mask, bar_index = 0; 314 315 switch (offset) { 316 case 0x04: /* device control */ 317 case 0x06: /* device status */ 318 /* do nothing */ 319 break; 320 case 0x3c: /* interrupt line */ 321 mdev_state->vconfig[0x3c] = buf[0]; 322 break; 323 case 0x3d: 324 /* 325 * Interrupt Pin is hardwired to INTA. 326 * This field is write protected by hardware 327 */ 328 break; 329 case 0x10: /* BAR0 */ 330 case 0x14: /* BAR1 */ 331 if (offset == 0x10) 332 bar_index = 0; 333 else if (offset == 0x14) 334 bar_index = 1; 335 336 if ((mdev_state->nr_ports == 1) && (bar_index == 1)) { 337 STORE_LE32(&mdev_state->vconfig[offset], 0); 338 break; 339 } 340 341 cfg_addr = *(u32 *)buf; 342 pr_info("BAR%d addr 0x%x\n", bar_index, cfg_addr); 343 344 if (cfg_addr == 0xffffffff) { 345 bar_mask = mdev_state->bar_mask[bar_index]; 346 cfg_addr = (cfg_addr & bar_mask); 347 } 348 349 cfg_addr |= (mdev_state->vconfig[offset] & 0x3ul); 350 STORE_LE32(&mdev_state->vconfig[offset], cfg_addr); 351 break; 352 case 0x18: /* BAR2 */ 353 case 0x1c: /* BAR3 */ 354 case 0x20: /* BAR4 */ 355 STORE_LE32(&mdev_state->vconfig[offset], 0); 356 break; 357 default: 358 pr_info("PCI config write @0x%x of %d bytes not handled\n", 359 offset, count); 360 break; 361 } 362 } 363 364 static void handle_bar_write(unsigned int index, struct mdev_state *mdev_state, 365 u16 offset, u8 *buf, u32 count) 366 { 367 u8 data = *buf; 368 369 /* Handle data written by guest */ 370 switch (offset) { 371 case UART_TX: 372 /* if DLAB set, data is LSB of divisor */ 373 if (mdev_state->s[index].dlab) { 374 mdev_state->s[index].divisor |= data; 375 break; 376 } 377 378 mutex_lock(&mdev_state->rxtx_lock); 379 380 /* save in TX buffer */ 381 if (mdev_state->s[index].rxtx.count < 382 mdev_state->s[index].max_fifo_size) { 383 mdev_state->s[index].rxtx.fifo[ 384 mdev_state->s[index].rxtx.head] = data; 385 mdev_state->s[index].rxtx.count++; 386 CIRCULAR_BUF_INC_IDX(mdev_state->s[index].rxtx.head); 387 mdev_state->s[index].overrun = false; 388 389 /* 390 * Trigger interrupt if receive data interrupt is 391 * enabled and fifo reached trigger level 392 */ 393 if ((mdev_state->s[index].uart_reg[UART_IER] & 394 UART_IER_RDI) && 395 (mdev_state->s[index].rxtx.count == 396 mdev_state->s[index].intr_trigger_level)) { 397 /* trigger interrupt */ 398 #if defined(DEBUG_INTR) 399 pr_err("Serial port %d: Fifo level trigger\n", 400 index); 401 #endif 402 mtty_trigger_interrupt(mdev_state); 403 } 404 } else { 405 #if defined(DEBUG_INTR) 406 pr_err("Serial port %d: Buffer Overflow\n", index); 407 #endif 408 mdev_state->s[index].overrun = true; 409 410 /* 411 * Trigger interrupt if receiver line status interrupt 412 * is enabled 413 */ 414 if (mdev_state->s[index].uart_reg[UART_IER] & 415 UART_IER_RLSI) 416 mtty_trigger_interrupt(mdev_state); 417 } 418 mutex_unlock(&mdev_state->rxtx_lock); 419 break; 420 421 case UART_IER: 422 /* if DLAB set, data is MSB of divisor */ 423 if (mdev_state->s[index].dlab) 424 mdev_state->s[index].divisor |= (u16)data << 8; 425 else { 426 mdev_state->s[index].uart_reg[offset] = data; 427 mutex_lock(&mdev_state->rxtx_lock); 428 if ((data & UART_IER_THRI) && 429 (mdev_state->s[index].rxtx.head == 430 mdev_state->s[index].rxtx.tail)) { 431 #if defined(DEBUG_INTR) 432 pr_err("Serial port %d: IER_THRI write\n", 433 index); 434 #endif 435 mtty_trigger_interrupt(mdev_state); 436 } 437 438 mutex_unlock(&mdev_state->rxtx_lock); 439 } 440 441 break; 442 443 case UART_FCR: 444 mdev_state->s[index].fcr = data; 445 446 mutex_lock(&mdev_state->rxtx_lock); 447 if (data & (UART_FCR_CLEAR_RCVR | UART_FCR_CLEAR_XMIT)) { 448 /* clear loop back FIFO */ 449 mdev_state->s[index].rxtx.count = 0; 450 mdev_state->s[index].rxtx.head = 0; 451 mdev_state->s[index].rxtx.tail = 0; 452 } 453 mutex_unlock(&mdev_state->rxtx_lock); 454 455 switch (data & UART_FCR_TRIGGER_MASK) { 456 case UART_FCR_TRIGGER_1: 457 mdev_state->s[index].intr_trigger_level = 1; 458 break; 459 460 case UART_FCR_TRIGGER_4: 461 mdev_state->s[index].intr_trigger_level = 4; 462 break; 463 464 case UART_FCR_TRIGGER_8: 465 mdev_state->s[index].intr_trigger_level = 8; 466 break; 467 468 case UART_FCR_TRIGGER_14: 469 mdev_state->s[index].intr_trigger_level = 14; 470 break; 471 } 472 473 /* 474 * Set trigger level to 1 otherwise or implement timer with 475 * timeout of 4 characters and on expiring that timer set 476 * Recevice data timeout in IIR register 477 */ 478 mdev_state->s[index].intr_trigger_level = 1; 479 if (data & UART_FCR_ENABLE_FIFO) 480 mdev_state->s[index].max_fifo_size = MAX_FIFO_SIZE; 481 else { 482 mdev_state->s[index].max_fifo_size = 1; 483 mdev_state->s[index].intr_trigger_level = 1; 484 } 485 486 break; 487 488 case UART_LCR: 489 if (data & UART_LCR_DLAB) { 490 mdev_state->s[index].dlab = true; 491 mdev_state->s[index].divisor = 0; 492 } else 493 mdev_state->s[index].dlab = false; 494 495 mdev_state->s[index].uart_reg[offset] = data; 496 break; 497 498 case UART_MCR: 499 mdev_state->s[index].uart_reg[offset] = data; 500 501 if ((mdev_state->s[index].uart_reg[UART_IER] & UART_IER_MSI) && 502 (data & UART_MCR_OUT2)) { 503 #if defined(DEBUG_INTR) 504 pr_err("Serial port %d: MCR_OUT2 write\n", index); 505 #endif 506 mtty_trigger_interrupt(mdev_state); 507 } 508 509 if ((mdev_state->s[index].uart_reg[UART_IER] & UART_IER_MSI) && 510 (data & (UART_MCR_RTS | UART_MCR_DTR))) { 511 #if defined(DEBUG_INTR) 512 pr_err("Serial port %d: MCR RTS/DTR write\n", index); 513 #endif 514 mtty_trigger_interrupt(mdev_state); 515 } 516 break; 517 518 case UART_LSR: 519 case UART_MSR: 520 /* do nothing */ 521 break; 522 523 case UART_SCR: 524 mdev_state->s[index].uart_reg[offset] = data; 525 break; 526 527 default: 528 break; 529 } 530 } 531 532 static void handle_bar_read(unsigned int index, struct mdev_state *mdev_state, 533 u16 offset, u8 *buf, u32 count) 534 { 535 /* Handle read requests by guest */ 536 switch (offset) { 537 case UART_RX: 538 /* if DLAB set, data is LSB of divisor */ 539 if (mdev_state->s[index].dlab) { 540 *buf = (u8)mdev_state->s[index].divisor; 541 break; 542 } 543 544 mutex_lock(&mdev_state->rxtx_lock); 545 /* return data in tx buffer */ 546 if (mdev_state->s[index].rxtx.head != 547 mdev_state->s[index].rxtx.tail) { 548 *buf = mdev_state->s[index].rxtx.fifo[ 549 mdev_state->s[index].rxtx.tail]; 550 mdev_state->s[index].rxtx.count--; 551 CIRCULAR_BUF_INC_IDX(mdev_state->s[index].rxtx.tail); 552 } 553 554 if (mdev_state->s[index].rxtx.head == 555 mdev_state->s[index].rxtx.tail) { 556 /* 557 * Trigger interrupt if tx buffer empty interrupt is 558 * enabled and fifo is empty 559 */ 560 #if defined(DEBUG_INTR) 561 pr_err("Serial port %d: Buffer Empty\n", index); 562 #endif 563 if (mdev_state->s[index].uart_reg[UART_IER] & 564 UART_IER_THRI) 565 mtty_trigger_interrupt(mdev_state); 566 } 567 mutex_unlock(&mdev_state->rxtx_lock); 568 569 break; 570 571 case UART_IER: 572 if (mdev_state->s[index].dlab) { 573 *buf = (u8)(mdev_state->s[index].divisor >> 8); 574 break; 575 } 576 *buf = mdev_state->s[index].uart_reg[offset] & 0x0f; 577 break; 578 579 case UART_IIR: 580 { 581 u8 ier = mdev_state->s[index].uart_reg[UART_IER]; 582 *buf = 0; 583 584 mutex_lock(&mdev_state->rxtx_lock); 585 /* Interrupt priority 1: Parity, overrun, framing or break */ 586 if ((ier & UART_IER_RLSI) && mdev_state->s[index].overrun) 587 *buf |= UART_IIR_RLSI; 588 589 /* Interrupt priority 2: Fifo trigger level reached */ 590 if ((ier & UART_IER_RDI) && 591 (mdev_state->s[index].rxtx.count >= 592 mdev_state->s[index].intr_trigger_level)) 593 *buf |= UART_IIR_RDI; 594 595 /* Interrupt priotiry 3: transmitter holding register empty */ 596 if ((ier & UART_IER_THRI) && 597 (mdev_state->s[index].rxtx.head == 598 mdev_state->s[index].rxtx.tail)) 599 *buf |= UART_IIR_THRI; 600 601 /* Interrupt priotiry 4: Modem status: CTS, DSR, RI or DCD */ 602 if ((ier & UART_IER_MSI) && 603 (mdev_state->s[index].uart_reg[UART_MCR] & 604 (UART_MCR_RTS | UART_MCR_DTR))) 605 *buf |= UART_IIR_MSI; 606 607 /* bit0: 0=> interrupt pending, 1=> no interrupt is pending */ 608 if (*buf == 0) 609 *buf = UART_IIR_NO_INT; 610 611 /* set bit 6 & 7 to be 16550 compatible */ 612 *buf |= 0xC0; 613 mutex_unlock(&mdev_state->rxtx_lock); 614 } 615 break; 616 617 case UART_LCR: 618 case UART_MCR: 619 *buf = mdev_state->s[index].uart_reg[offset]; 620 break; 621 622 case UART_LSR: 623 { 624 u8 lsr = 0; 625 626 mutex_lock(&mdev_state->rxtx_lock); 627 /* atleast one char in FIFO */ 628 if (mdev_state->s[index].rxtx.head != 629 mdev_state->s[index].rxtx.tail) 630 lsr |= UART_LSR_DR; 631 632 /* if FIFO overrun */ 633 if (mdev_state->s[index].overrun) 634 lsr |= UART_LSR_OE; 635 636 /* transmit FIFO empty and tramsitter empty */ 637 if (mdev_state->s[index].rxtx.head == 638 mdev_state->s[index].rxtx.tail) 639 lsr |= UART_LSR_TEMT | UART_LSR_THRE; 640 641 mutex_unlock(&mdev_state->rxtx_lock); 642 *buf = lsr; 643 break; 644 } 645 case UART_MSR: 646 *buf = UART_MSR_DSR | UART_MSR_DDSR | UART_MSR_DCD; 647 648 mutex_lock(&mdev_state->rxtx_lock); 649 /* if AFE is 1 and FIFO have space, set CTS bit */ 650 if (mdev_state->s[index].uart_reg[UART_MCR] & 651 UART_MCR_AFE) { 652 if (mdev_state->s[index].rxtx.count < 653 mdev_state->s[index].max_fifo_size) 654 *buf |= UART_MSR_CTS | UART_MSR_DCTS; 655 } else 656 *buf |= UART_MSR_CTS | UART_MSR_DCTS; 657 mutex_unlock(&mdev_state->rxtx_lock); 658 659 break; 660 661 case UART_SCR: 662 *buf = mdev_state->s[index].uart_reg[offset]; 663 break; 664 665 default: 666 break; 667 } 668 } 669 670 static void mdev_read_base(struct mdev_state *mdev_state) 671 { 672 int index, pos; 673 u32 start_lo, start_hi; 674 u32 mem_type; 675 676 pos = PCI_BASE_ADDRESS_0; 677 678 for (index = 0; index <= VFIO_PCI_BAR5_REGION_INDEX; index++) { 679 680 if (!mdev_state->region_info[index].size) 681 continue; 682 683 start_lo = (*(u32 *)(mdev_state->vconfig + pos)) & 684 PCI_BASE_ADDRESS_MEM_MASK; 685 mem_type = (*(u32 *)(mdev_state->vconfig + pos)) & 686 PCI_BASE_ADDRESS_MEM_TYPE_MASK; 687 688 switch (mem_type) { 689 case PCI_BASE_ADDRESS_MEM_TYPE_64: 690 start_hi = (*(u32 *)(mdev_state->vconfig + pos + 4)); 691 pos += 4; 692 break; 693 case PCI_BASE_ADDRESS_MEM_TYPE_32: 694 case PCI_BASE_ADDRESS_MEM_TYPE_1M: 695 /* 1M mem BAR treated as 32-bit BAR */ 696 default: 697 /* mem unknown type treated as 32-bit BAR */ 698 start_hi = 0; 699 break; 700 } 701 pos += 4; 702 mdev_state->region_info[index].start = ((u64)start_hi << 32) | 703 start_lo; 704 } 705 } 706 707 static ssize_t mdev_access(struct mdev_state *mdev_state, u8 *buf, size_t count, 708 loff_t pos, bool is_write) 709 { 710 unsigned int index; 711 loff_t offset; 712 int ret = 0; 713 714 if (!buf) 715 return -EINVAL; 716 717 mutex_lock(&mdev_state->ops_lock); 718 719 index = MTTY_VFIO_PCI_OFFSET_TO_INDEX(pos); 720 offset = pos & MTTY_VFIO_PCI_OFFSET_MASK; 721 switch (index) { 722 case VFIO_PCI_CONFIG_REGION_INDEX: 723 724 #if defined(DEBUG) 725 pr_info("%s: PCI config space %s at offset 0x%llx\n", 726 __func__, is_write ? "write" : "read", offset); 727 #endif 728 if (is_write) { 729 dump_buffer(buf, count); 730 handle_pci_cfg_write(mdev_state, offset, buf, count); 731 } else { 732 memcpy(buf, (mdev_state->vconfig + offset), count); 733 dump_buffer(buf, count); 734 } 735 736 break; 737 738 case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: 739 if (!mdev_state->region_info[index].start) 740 mdev_read_base(mdev_state); 741 742 if (is_write) { 743 dump_buffer(buf, count); 744 745 #if defined(DEBUG_REGS) 746 pr_info("%s: BAR%d WR @0x%llx %s val:0x%02x dlab:%d\n", 747 __func__, index, offset, wr_reg[offset], 748 *buf, mdev_state->s[index].dlab); 749 #endif 750 handle_bar_write(index, mdev_state, offset, buf, count); 751 } else { 752 handle_bar_read(index, mdev_state, offset, buf, count); 753 dump_buffer(buf, count); 754 755 #if defined(DEBUG_REGS) 756 pr_info("%s: BAR%d RD @0x%llx %s val:0x%02x dlab:%d\n", 757 __func__, index, offset, rd_reg[offset], 758 *buf, mdev_state->s[index].dlab); 759 #endif 760 } 761 break; 762 763 default: 764 ret = -1; 765 goto accessfailed; 766 } 767 768 ret = count; 769 770 771 accessfailed: 772 mutex_unlock(&mdev_state->ops_lock); 773 774 return ret; 775 } 776 777 static size_t mtty_data_size(struct mdev_state *mdev_state) 778 { 779 return offsetof(struct mtty_data, ports) + 780 (mdev_state->nr_ports * sizeof(struct serial_port)); 781 } 782 783 static void mtty_disable_file(struct mtty_migration_file *migf) 784 { 785 mutex_lock(&migf->lock); 786 migf->disabled = true; 787 migf->filled_size = 0; 788 migf->filp->f_pos = 0; 789 mutex_unlock(&migf->lock); 790 } 791 792 static void mtty_disable_files(struct mdev_state *mdev_state) 793 { 794 if (mdev_state->saving_migf) { 795 mtty_disable_file(mdev_state->saving_migf); 796 fput(mdev_state->saving_migf->filp); 797 mdev_state->saving_migf = NULL; 798 } 799 800 if (mdev_state->resuming_migf) { 801 mtty_disable_file(mdev_state->resuming_migf); 802 fput(mdev_state->resuming_migf->filp); 803 mdev_state->resuming_migf = NULL; 804 } 805 } 806 807 static void mtty_state_mutex_unlock(struct mdev_state *mdev_state) 808 { 809 again: 810 mutex_lock(&mdev_state->reset_mutex); 811 if (mdev_state->deferred_reset) { 812 mdev_state->deferred_reset = false; 813 mutex_unlock(&mdev_state->reset_mutex); 814 mdev_state->state = VFIO_DEVICE_STATE_RUNNING; 815 mtty_disable_files(mdev_state); 816 goto again; 817 } 818 mutex_unlock(&mdev_state->state_mutex); 819 mutex_unlock(&mdev_state->reset_mutex); 820 } 821 822 static int mtty_release_migf(struct inode *inode, struct file *filp) 823 { 824 struct mtty_migration_file *migf = filp->private_data; 825 826 mtty_disable_file(migf); 827 mutex_destroy(&migf->lock); 828 kfree(migf); 829 830 return 0; 831 } 832 833 static long mtty_precopy_ioctl(struct file *filp, unsigned int cmd, 834 unsigned long arg) 835 { 836 struct mtty_migration_file *migf = filp->private_data; 837 struct mdev_state *mdev_state = migf->mdev_state; 838 loff_t *pos = &filp->f_pos; 839 struct vfio_precopy_info info = {}; 840 unsigned long minsz; 841 int ret; 842 843 if (cmd != VFIO_MIG_GET_PRECOPY_INFO) 844 return -ENOTTY; 845 846 minsz = offsetofend(struct vfio_precopy_info, dirty_bytes); 847 848 if (copy_from_user(&info, (void __user *)arg, minsz)) 849 return -EFAULT; 850 if (info.argsz < minsz) 851 return -EINVAL; 852 853 mutex_lock(&mdev_state->state_mutex); 854 if (mdev_state->state != VFIO_DEVICE_STATE_PRE_COPY && 855 mdev_state->state != VFIO_DEVICE_STATE_PRE_COPY_P2P) { 856 ret = -EINVAL; 857 goto unlock; 858 } 859 860 mutex_lock(&migf->lock); 861 862 if (migf->disabled) { 863 mutex_unlock(&migf->lock); 864 ret = -ENODEV; 865 goto unlock; 866 } 867 868 if (*pos > migf->filled_size) { 869 mutex_unlock(&migf->lock); 870 ret = -EINVAL; 871 goto unlock; 872 } 873 874 info.dirty_bytes = 0; 875 info.initial_bytes = migf->filled_size - *pos; 876 mutex_unlock(&migf->lock); 877 878 ret = copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; 879 unlock: 880 mtty_state_mutex_unlock(mdev_state); 881 return ret; 882 } 883 884 static ssize_t mtty_save_read(struct file *filp, char __user *buf, 885 size_t len, loff_t *pos) 886 { 887 struct mtty_migration_file *migf = filp->private_data; 888 ssize_t ret = 0; 889 890 if (pos) 891 return -ESPIPE; 892 893 pos = &filp->f_pos; 894 895 mutex_lock(&migf->lock); 896 897 dev_dbg(migf->mdev_state->vdev.dev, "%s ask %zu\n", __func__, len); 898 899 if (migf->disabled) { 900 ret = -ENODEV; 901 goto out_unlock; 902 } 903 904 if (*pos > migf->filled_size) { 905 ret = -EINVAL; 906 goto out_unlock; 907 } 908 909 len = min_t(size_t, migf->filled_size - *pos, len); 910 if (len) { 911 if (copy_to_user(buf, (void *)&migf->data + *pos, len)) { 912 ret = -EFAULT; 913 goto out_unlock; 914 } 915 *pos += len; 916 ret = len; 917 } 918 out_unlock: 919 dev_dbg(migf->mdev_state->vdev.dev, "%s read %zu\n", __func__, ret); 920 mutex_unlock(&migf->lock); 921 return ret; 922 } 923 924 static const struct file_operations mtty_save_fops = { 925 .owner = THIS_MODULE, 926 .read = mtty_save_read, 927 .unlocked_ioctl = mtty_precopy_ioctl, 928 .compat_ioctl = compat_ptr_ioctl, 929 .release = mtty_release_migf, 930 .llseek = no_llseek, 931 }; 932 933 static void mtty_save_state(struct mdev_state *mdev_state) 934 { 935 struct mtty_migration_file *migf = mdev_state->saving_migf; 936 int i; 937 938 mutex_lock(&migf->lock); 939 for (i = 0; i < mdev_state->nr_ports; i++) { 940 memcpy(&migf->data.ports[i], 941 &mdev_state->s[i], sizeof(struct serial_port)); 942 migf->filled_size += sizeof(struct serial_port); 943 } 944 dev_dbg(mdev_state->vdev.dev, 945 "%s filled to %zu\n", __func__, migf->filled_size); 946 mutex_unlock(&migf->lock); 947 } 948 949 static int mtty_load_state(struct mdev_state *mdev_state) 950 { 951 struct mtty_migration_file *migf = mdev_state->resuming_migf; 952 int i; 953 954 mutex_lock(&migf->lock); 955 /* magic and version already tested by resume write fn */ 956 if (migf->filled_size < mtty_data_size(mdev_state)) { 957 dev_dbg(mdev_state->vdev.dev, "%s expected %zu, got %zu\n", 958 __func__, mtty_data_size(mdev_state), 959 migf->filled_size); 960 mutex_unlock(&migf->lock); 961 return -EINVAL; 962 } 963 964 for (i = 0; i < mdev_state->nr_ports; i++) 965 memcpy(&mdev_state->s[i], 966 &migf->data.ports[i], sizeof(struct serial_port)); 967 968 mutex_unlock(&migf->lock); 969 return 0; 970 } 971 972 static struct mtty_migration_file * 973 mtty_save_device_data(struct mdev_state *mdev_state, 974 enum vfio_device_mig_state state) 975 { 976 struct mtty_migration_file *migf = mdev_state->saving_migf; 977 struct mtty_migration_file *ret = NULL; 978 979 if (migf) { 980 if (state == VFIO_DEVICE_STATE_STOP_COPY) 981 goto fill_data; 982 return ret; 983 } 984 985 migf = kzalloc(sizeof(*migf), GFP_KERNEL_ACCOUNT); 986 if (!migf) 987 return ERR_PTR(-ENOMEM); 988 989 migf->filp = anon_inode_getfile("mtty_mig", &mtty_save_fops, 990 migf, O_RDONLY); 991 if (IS_ERR(migf->filp)) { 992 int rc = PTR_ERR(migf->filp); 993 994 kfree(migf); 995 return ERR_PTR(rc); 996 } 997 998 stream_open(migf->filp->f_inode, migf->filp); 999 mutex_init(&migf->lock); 1000 migf->mdev_state = mdev_state; 1001 1002 migf->data.magic = MTTY_MAGIC; 1003 migf->data.major_ver = MTTY_MAJOR_VER; 1004 migf->data.minor_ver = MTTY_MINOR_VER; 1005 migf->data.nr_ports = mdev_state->nr_ports; 1006 1007 migf->filled_size = offsetof(struct mtty_data, ports); 1008 1009 dev_dbg(mdev_state->vdev.dev, "%s filled header to %zu\n", 1010 __func__, migf->filled_size); 1011 1012 ret = mdev_state->saving_migf = migf; 1013 1014 fill_data: 1015 if (state == VFIO_DEVICE_STATE_STOP_COPY) 1016 mtty_save_state(mdev_state); 1017 1018 return ret; 1019 } 1020 1021 static ssize_t mtty_resume_write(struct file *filp, const char __user *buf, 1022 size_t len, loff_t *pos) 1023 { 1024 struct mtty_migration_file *migf = filp->private_data; 1025 struct mdev_state *mdev_state = migf->mdev_state; 1026 loff_t requested_length; 1027 ssize_t ret = 0; 1028 1029 if (pos) 1030 return -ESPIPE; 1031 1032 pos = &filp->f_pos; 1033 1034 if (*pos < 0 || 1035 check_add_overflow((loff_t)len, *pos, &requested_length)) 1036 return -EINVAL; 1037 1038 if (requested_length > mtty_data_size(mdev_state)) 1039 return -ENOMEM; 1040 1041 mutex_lock(&migf->lock); 1042 1043 if (migf->disabled) { 1044 ret = -ENODEV; 1045 goto out_unlock; 1046 } 1047 1048 if (copy_from_user((void *)&migf->data + *pos, buf, len)) { 1049 ret = -EFAULT; 1050 goto out_unlock; 1051 } 1052 1053 *pos += len; 1054 ret = len; 1055 1056 dev_dbg(migf->mdev_state->vdev.dev, "%s received %zu, total %zu\n", 1057 __func__, len, migf->filled_size + len); 1058 1059 if (migf->filled_size < offsetof(struct mtty_data, ports) && 1060 migf->filled_size + len >= offsetof(struct mtty_data, ports)) { 1061 if (migf->data.magic != MTTY_MAGIC || migf->data.flags || 1062 migf->data.major_ver != MTTY_MAJOR_VER || 1063 migf->data.minor_ver != MTTY_MINOR_VER || 1064 migf->data.nr_ports != mdev_state->nr_ports) { 1065 dev_dbg(migf->mdev_state->vdev.dev, 1066 "%s failed validation\n", __func__); 1067 ret = -EFAULT; 1068 } else { 1069 dev_dbg(migf->mdev_state->vdev.dev, 1070 "%s header validated\n", __func__); 1071 } 1072 } 1073 1074 migf->filled_size += len; 1075 1076 out_unlock: 1077 mutex_unlock(&migf->lock); 1078 return ret; 1079 } 1080 1081 static const struct file_operations mtty_resume_fops = { 1082 .owner = THIS_MODULE, 1083 .write = mtty_resume_write, 1084 .release = mtty_release_migf, 1085 .llseek = no_llseek, 1086 }; 1087 1088 static struct mtty_migration_file * 1089 mtty_resume_device_data(struct mdev_state *mdev_state) 1090 { 1091 struct mtty_migration_file *migf; 1092 int ret; 1093 1094 migf = kzalloc(sizeof(*migf), GFP_KERNEL_ACCOUNT); 1095 if (!migf) 1096 return ERR_PTR(-ENOMEM); 1097 1098 migf->filp = anon_inode_getfile("mtty_mig", &mtty_resume_fops, 1099 migf, O_WRONLY); 1100 if (IS_ERR(migf->filp)) { 1101 ret = PTR_ERR(migf->filp); 1102 kfree(migf); 1103 return ERR_PTR(ret); 1104 } 1105 1106 stream_open(migf->filp->f_inode, migf->filp); 1107 mutex_init(&migf->lock); 1108 migf->mdev_state = mdev_state; 1109 1110 mdev_state->resuming_migf = migf; 1111 1112 return migf; 1113 } 1114 1115 static struct file *mtty_step_state(struct mdev_state *mdev_state, 1116 enum vfio_device_mig_state new) 1117 { 1118 enum vfio_device_mig_state cur = mdev_state->state; 1119 1120 dev_dbg(mdev_state->vdev.dev, "%s: %d -> %d\n", __func__, cur, new); 1121 1122 /* 1123 * The following state transitions are no-op considering 1124 * mtty does not do DMA nor require any explicit start/stop. 1125 * 1126 * RUNNING -> RUNNING_P2P 1127 * RUNNING_P2P -> RUNNING 1128 * RUNNING_P2P -> STOP 1129 * PRE_COPY -> PRE_COPY_P2P 1130 * PRE_COPY_P2P -> PRE_COPY 1131 * STOP -> RUNNING_P2P 1132 */ 1133 if ((cur == VFIO_DEVICE_STATE_RUNNING && 1134 new == VFIO_DEVICE_STATE_RUNNING_P2P) || 1135 (cur == VFIO_DEVICE_STATE_RUNNING_P2P && 1136 (new == VFIO_DEVICE_STATE_RUNNING || 1137 new == VFIO_DEVICE_STATE_STOP)) || 1138 (cur == VFIO_DEVICE_STATE_PRE_COPY && 1139 new == VFIO_DEVICE_STATE_PRE_COPY_P2P) || 1140 (cur == VFIO_DEVICE_STATE_PRE_COPY_P2P && 1141 new == VFIO_DEVICE_STATE_PRE_COPY) || 1142 (cur == VFIO_DEVICE_STATE_STOP && 1143 new == VFIO_DEVICE_STATE_RUNNING_P2P)) 1144 return NULL; 1145 1146 /* 1147 * The following state transitions simply close migration files, 1148 * with the exception of RESUMING -> STOP, which needs to load 1149 * the state first. 1150 * 1151 * RESUMING -> STOP 1152 * PRE_COPY -> RUNNING 1153 * PRE_COPY_P2P -> RUNNING_P2P 1154 * STOP_COPY -> STOP 1155 */ 1156 if (cur == VFIO_DEVICE_STATE_RESUMING && 1157 new == VFIO_DEVICE_STATE_STOP) { 1158 int ret; 1159 1160 ret = mtty_load_state(mdev_state); 1161 if (ret) 1162 return ERR_PTR(ret); 1163 mtty_disable_files(mdev_state); 1164 return NULL; 1165 } 1166 1167 if ((cur == VFIO_DEVICE_STATE_PRE_COPY && 1168 new == VFIO_DEVICE_STATE_RUNNING) || 1169 (cur == VFIO_DEVICE_STATE_PRE_COPY_P2P && 1170 new == VFIO_DEVICE_STATE_RUNNING_P2P) || 1171 (cur == VFIO_DEVICE_STATE_STOP_COPY && 1172 new == VFIO_DEVICE_STATE_STOP)) { 1173 mtty_disable_files(mdev_state); 1174 return NULL; 1175 } 1176 1177 /* 1178 * The following state transitions return migration files. 1179 * 1180 * RUNNING -> PRE_COPY 1181 * RUNNING_P2P -> PRE_COPY_P2P 1182 * STOP -> STOP_COPY 1183 * STOP -> RESUMING 1184 * PRE_COPY_P2P -> STOP_COPY 1185 */ 1186 if ((cur == VFIO_DEVICE_STATE_RUNNING && 1187 new == VFIO_DEVICE_STATE_PRE_COPY) || 1188 (cur == VFIO_DEVICE_STATE_RUNNING_P2P && 1189 new == VFIO_DEVICE_STATE_PRE_COPY_P2P) || 1190 (cur == VFIO_DEVICE_STATE_STOP && 1191 new == VFIO_DEVICE_STATE_STOP_COPY) || 1192 (cur == VFIO_DEVICE_STATE_PRE_COPY_P2P && 1193 new == VFIO_DEVICE_STATE_STOP_COPY)) { 1194 struct mtty_migration_file *migf; 1195 1196 migf = mtty_save_device_data(mdev_state, new); 1197 if (IS_ERR(migf)) 1198 return ERR_CAST(migf); 1199 1200 if (migf) { 1201 get_file(migf->filp); 1202 1203 return migf->filp; 1204 } 1205 return NULL; 1206 } 1207 1208 if (cur == VFIO_DEVICE_STATE_STOP && 1209 new == VFIO_DEVICE_STATE_RESUMING) { 1210 struct mtty_migration_file *migf; 1211 1212 migf = mtty_resume_device_data(mdev_state); 1213 if (IS_ERR(migf)) 1214 return ERR_CAST(migf); 1215 1216 get_file(migf->filp); 1217 1218 return migf->filp; 1219 } 1220 1221 /* vfio_mig_get_next_state() does not use arcs other than the above */ 1222 WARN_ON(true); 1223 return ERR_PTR(-EINVAL); 1224 } 1225 1226 static struct file *mtty_set_state(struct vfio_device *vdev, 1227 enum vfio_device_mig_state new_state) 1228 { 1229 struct mdev_state *mdev_state = 1230 container_of(vdev, struct mdev_state, vdev); 1231 struct file *ret = NULL; 1232 1233 dev_dbg(vdev->dev, "%s -> %d\n", __func__, new_state); 1234 1235 mutex_lock(&mdev_state->state_mutex); 1236 while (mdev_state->state != new_state) { 1237 enum vfio_device_mig_state next_state; 1238 int rc = vfio_mig_get_next_state(vdev, mdev_state->state, 1239 new_state, &next_state); 1240 if (rc) { 1241 ret = ERR_PTR(rc); 1242 break; 1243 } 1244 1245 ret = mtty_step_state(mdev_state, next_state); 1246 if (IS_ERR(ret)) 1247 break; 1248 1249 mdev_state->state = next_state; 1250 1251 if (WARN_ON(ret && new_state != next_state)) { 1252 fput(ret); 1253 ret = ERR_PTR(-EINVAL); 1254 break; 1255 } 1256 } 1257 mtty_state_mutex_unlock(mdev_state); 1258 return ret; 1259 } 1260 1261 static int mtty_get_state(struct vfio_device *vdev, 1262 enum vfio_device_mig_state *current_state) 1263 { 1264 struct mdev_state *mdev_state = 1265 container_of(vdev, struct mdev_state, vdev); 1266 1267 mutex_lock(&mdev_state->state_mutex); 1268 *current_state = mdev_state->state; 1269 mtty_state_mutex_unlock(mdev_state); 1270 return 0; 1271 } 1272 1273 static int mtty_get_data_size(struct vfio_device *vdev, 1274 unsigned long *stop_copy_length) 1275 { 1276 struct mdev_state *mdev_state = 1277 container_of(vdev, struct mdev_state, vdev); 1278 1279 *stop_copy_length = mtty_data_size(mdev_state); 1280 return 0; 1281 } 1282 1283 static const struct vfio_migration_ops mtty_migration_ops = { 1284 .migration_set_state = mtty_set_state, 1285 .migration_get_state = mtty_get_state, 1286 .migration_get_data_size = mtty_get_data_size, 1287 }; 1288 1289 static int mtty_log_start(struct vfio_device *vdev, 1290 struct rb_root_cached *ranges, 1291 u32 nnodes, u64 *page_size) 1292 { 1293 return 0; 1294 } 1295 1296 static int mtty_log_stop(struct vfio_device *vdev) 1297 { 1298 return 0; 1299 } 1300 1301 static int mtty_log_read_and_clear(struct vfio_device *vdev, 1302 unsigned long iova, unsigned long length, 1303 struct iova_bitmap *dirty) 1304 { 1305 return 0; 1306 } 1307 1308 static const struct vfio_log_ops mtty_log_ops = { 1309 .log_start = mtty_log_start, 1310 .log_stop = mtty_log_stop, 1311 .log_read_and_clear = mtty_log_read_and_clear, 1312 }; 1313 1314 static int mtty_init_dev(struct vfio_device *vdev) 1315 { 1316 struct mdev_state *mdev_state = 1317 container_of(vdev, struct mdev_state, vdev); 1318 struct mdev_device *mdev = to_mdev_device(vdev->dev); 1319 struct mtty_type *type = 1320 container_of(mdev->type, struct mtty_type, type); 1321 int avail_ports = atomic_read(&mdev_avail_ports); 1322 int ret; 1323 1324 do { 1325 if (avail_ports < type->nr_ports) 1326 return -ENOSPC; 1327 } while (!atomic_try_cmpxchg(&mdev_avail_ports, 1328 &avail_ports, 1329 avail_ports - type->nr_ports)); 1330 1331 mdev_state->nr_ports = type->nr_ports; 1332 mdev_state->irq_index = -1; 1333 mdev_state->s[0].max_fifo_size = MAX_FIFO_SIZE; 1334 mdev_state->s[1].max_fifo_size = MAX_FIFO_SIZE; 1335 mutex_init(&mdev_state->rxtx_lock); 1336 1337 mdev_state->vconfig = kzalloc(MTTY_CONFIG_SPACE_SIZE, GFP_KERNEL); 1338 if (!mdev_state->vconfig) { 1339 ret = -ENOMEM; 1340 goto err_nr_ports; 1341 } 1342 1343 mutex_init(&mdev_state->ops_lock); 1344 mdev_state->mdev = mdev; 1345 mtty_create_config_space(mdev_state); 1346 1347 mutex_init(&mdev_state->state_mutex); 1348 mutex_init(&mdev_state->reset_mutex); 1349 vdev->migration_flags = VFIO_MIGRATION_STOP_COPY | 1350 VFIO_MIGRATION_P2P | 1351 VFIO_MIGRATION_PRE_COPY; 1352 vdev->mig_ops = &mtty_migration_ops; 1353 vdev->log_ops = &mtty_log_ops; 1354 mdev_state->state = VFIO_DEVICE_STATE_RUNNING; 1355 1356 return 0; 1357 1358 err_nr_ports: 1359 atomic_add(type->nr_ports, &mdev_avail_ports); 1360 return ret; 1361 } 1362 1363 static int mtty_probe(struct mdev_device *mdev) 1364 { 1365 struct mdev_state *mdev_state; 1366 int ret; 1367 1368 mdev_state = vfio_alloc_device(mdev_state, vdev, &mdev->dev, 1369 &mtty_dev_ops); 1370 if (IS_ERR(mdev_state)) 1371 return PTR_ERR(mdev_state); 1372 1373 ret = vfio_register_emulated_iommu_dev(&mdev_state->vdev); 1374 if (ret) 1375 goto err_put_vdev; 1376 dev_set_drvdata(&mdev->dev, mdev_state); 1377 return 0; 1378 1379 err_put_vdev: 1380 vfio_put_device(&mdev_state->vdev); 1381 return ret; 1382 } 1383 1384 static void mtty_release_dev(struct vfio_device *vdev) 1385 { 1386 struct mdev_state *mdev_state = 1387 container_of(vdev, struct mdev_state, vdev); 1388 1389 mutex_destroy(&mdev_state->reset_mutex); 1390 mutex_destroy(&mdev_state->state_mutex); 1391 atomic_add(mdev_state->nr_ports, &mdev_avail_ports); 1392 kfree(mdev_state->vconfig); 1393 } 1394 1395 static void mtty_remove(struct mdev_device *mdev) 1396 { 1397 struct mdev_state *mdev_state = dev_get_drvdata(&mdev->dev); 1398 1399 vfio_unregister_group_dev(&mdev_state->vdev); 1400 vfio_put_device(&mdev_state->vdev); 1401 } 1402 1403 static int mtty_reset(struct mdev_state *mdev_state) 1404 { 1405 pr_info("%s: called\n", __func__); 1406 1407 mutex_lock(&mdev_state->reset_mutex); 1408 mdev_state->deferred_reset = true; 1409 if (!mutex_trylock(&mdev_state->state_mutex)) { 1410 mutex_unlock(&mdev_state->reset_mutex); 1411 return 0; 1412 } 1413 mutex_unlock(&mdev_state->reset_mutex); 1414 mtty_state_mutex_unlock(mdev_state); 1415 1416 return 0; 1417 } 1418 1419 static ssize_t mtty_read(struct vfio_device *vdev, char __user *buf, 1420 size_t count, loff_t *ppos) 1421 { 1422 struct mdev_state *mdev_state = 1423 container_of(vdev, struct mdev_state, vdev); 1424 unsigned int done = 0; 1425 int ret; 1426 1427 while (count) { 1428 size_t filled; 1429 1430 if (count >= 4 && !(*ppos % 4)) { 1431 u32 val; 1432 1433 ret = mdev_access(mdev_state, (u8 *)&val, sizeof(val), 1434 *ppos, false); 1435 if (ret <= 0) 1436 goto read_err; 1437 1438 if (copy_to_user(buf, &val, sizeof(val))) 1439 goto read_err; 1440 1441 filled = 4; 1442 } else if (count >= 2 && !(*ppos % 2)) { 1443 u16 val; 1444 1445 ret = mdev_access(mdev_state, (u8 *)&val, sizeof(val), 1446 *ppos, false); 1447 if (ret <= 0) 1448 goto read_err; 1449 1450 if (copy_to_user(buf, &val, sizeof(val))) 1451 goto read_err; 1452 1453 filled = 2; 1454 } else { 1455 u8 val; 1456 1457 ret = mdev_access(mdev_state, (u8 *)&val, sizeof(val), 1458 *ppos, false); 1459 if (ret <= 0) 1460 goto read_err; 1461 1462 if (copy_to_user(buf, &val, sizeof(val))) 1463 goto read_err; 1464 1465 filled = 1; 1466 } 1467 1468 count -= filled; 1469 done += filled; 1470 *ppos += filled; 1471 buf += filled; 1472 } 1473 1474 return done; 1475 1476 read_err: 1477 return -EFAULT; 1478 } 1479 1480 static ssize_t mtty_write(struct vfio_device *vdev, const char __user *buf, 1481 size_t count, loff_t *ppos) 1482 { 1483 struct mdev_state *mdev_state = 1484 container_of(vdev, struct mdev_state, vdev); 1485 unsigned int done = 0; 1486 int ret; 1487 1488 while (count) { 1489 size_t filled; 1490 1491 if (count >= 4 && !(*ppos % 4)) { 1492 u32 val; 1493 1494 if (copy_from_user(&val, buf, sizeof(val))) 1495 goto write_err; 1496 1497 ret = mdev_access(mdev_state, (u8 *)&val, sizeof(val), 1498 *ppos, true); 1499 if (ret <= 0) 1500 goto write_err; 1501 1502 filled = 4; 1503 } else if (count >= 2 && !(*ppos % 2)) { 1504 u16 val; 1505 1506 if (copy_from_user(&val, buf, sizeof(val))) 1507 goto write_err; 1508 1509 ret = mdev_access(mdev_state, (u8 *)&val, sizeof(val), 1510 *ppos, true); 1511 if (ret <= 0) 1512 goto write_err; 1513 1514 filled = 2; 1515 } else { 1516 u8 val; 1517 1518 if (copy_from_user(&val, buf, sizeof(val))) 1519 goto write_err; 1520 1521 ret = mdev_access(mdev_state, (u8 *)&val, sizeof(val), 1522 *ppos, true); 1523 if (ret <= 0) 1524 goto write_err; 1525 1526 filled = 1; 1527 } 1528 count -= filled; 1529 done += filled; 1530 *ppos += filled; 1531 buf += filled; 1532 } 1533 1534 return done; 1535 write_err: 1536 return -EFAULT; 1537 } 1538 1539 static void mtty_disable_intx(struct mdev_state *mdev_state) 1540 { 1541 if (mdev_state->intx_evtfd) { 1542 eventfd_ctx_put(mdev_state->intx_evtfd); 1543 mdev_state->intx_evtfd = NULL; 1544 mdev_state->intx_mask = false; 1545 mdev_state->irq_index = -1; 1546 } 1547 } 1548 1549 static void mtty_disable_msi(struct mdev_state *mdev_state) 1550 { 1551 if (mdev_state->msi_evtfd) { 1552 eventfd_ctx_put(mdev_state->msi_evtfd); 1553 mdev_state->msi_evtfd = NULL; 1554 mdev_state->irq_index = -1; 1555 } 1556 } 1557 1558 static int mtty_set_irqs(struct mdev_state *mdev_state, uint32_t flags, 1559 unsigned int index, unsigned int start, 1560 unsigned int count, void *data) 1561 { 1562 int ret = 0; 1563 1564 mutex_lock(&mdev_state->ops_lock); 1565 switch (index) { 1566 case VFIO_PCI_INTX_IRQ_INDEX: 1567 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 1568 case VFIO_IRQ_SET_ACTION_MASK: 1569 if (!is_intx(mdev_state) || start != 0 || count != 1) { 1570 ret = -EINVAL; 1571 break; 1572 } 1573 1574 if (flags & VFIO_IRQ_SET_DATA_NONE) { 1575 mdev_state->intx_mask = true; 1576 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 1577 uint8_t mask = *(uint8_t *)data; 1578 1579 if (mask) 1580 mdev_state->intx_mask = true; 1581 } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 1582 ret = -ENOTTY; /* No support for mask fd */ 1583 } 1584 break; 1585 case VFIO_IRQ_SET_ACTION_UNMASK: 1586 if (!is_intx(mdev_state) || start != 0 || count != 1) { 1587 ret = -EINVAL; 1588 break; 1589 } 1590 1591 if (flags & VFIO_IRQ_SET_DATA_NONE) { 1592 mdev_state->intx_mask = false; 1593 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 1594 uint8_t mask = *(uint8_t *)data; 1595 1596 if (mask) 1597 mdev_state->intx_mask = false; 1598 } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 1599 ret = -ENOTTY; /* No support for unmask fd */ 1600 } 1601 break; 1602 case VFIO_IRQ_SET_ACTION_TRIGGER: 1603 if (is_intx(mdev_state) && !count && 1604 (flags & VFIO_IRQ_SET_DATA_NONE)) { 1605 mtty_disable_intx(mdev_state); 1606 break; 1607 } 1608 1609 if (!(is_intx(mdev_state) || is_noirq(mdev_state)) || 1610 start != 0 || count != 1) { 1611 ret = -EINVAL; 1612 break; 1613 } 1614 1615 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 1616 int fd = *(int *)data; 1617 struct eventfd_ctx *evt; 1618 1619 mtty_disable_intx(mdev_state); 1620 1621 if (fd < 0) 1622 break; 1623 1624 evt = eventfd_ctx_fdget(fd); 1625 if (IS_ERR(evt)) { 1626 ret = PTR_ERR(evt); 1627 break; 1628 } 1629 mdev_state->intx_evtfd = evt; 1630 mdev_state->irq_index = index; 1631 break; 1632 } 1633 1634 if (!is_intx(mdev_state)) { 1635 ret = -EINVAL; 1636 break; 1637 } 1638 1639 if (flags & VFIO_IRQ_SET_DATA_NONE) { 1640 mtty_trigger_interrupt(mdev_state); 1641 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 1642 uint8_t trigger = *(uint8_t *)data; 1643 1644 if (trigger) 1645 mtty_trigger_interrupt(mdev_state); 1646 } 1647 break; 1648 } 1649 break; 1650 case VFIO_PCI_MSI_IRQ_INDEX: 1651 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 1652 case VFIO_IRQ_SET_ACTION_MASK: 1653 case VFIO_IRQ_SET_ACTION_UNMASK: 1654 ret = -ENOTTY; 1655 break; 1656 case VFIO_IRQ_SET_ACTION_TRIGGER: 1657 if (is_msi(mdev_state) && !count && 1658 (flags & VFIO_IRQ_SET_DATA_NONE)) { 1659 mtty_disable_msi(mdev_state); 1660 break; 1661 } 1662 1663 if (!(is_msi(mdev_state) || is_noirq(mdev_state)) || 1664 start != 0 || count != 1) { 1665 ret = -EINVAL; 1666 break; 1667 } 1668 1669 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 1670 int fd = *(int *)data; 1671 struct eventfd_ctx *evt; 1672 1673 mtty_disable_msi(mdev_state); 1674 1675 if (fd < 0) 1676 break; 1677 1678 evt = eventfd_ctx_fdget(fd); 1679 if (IS_ERR(evt)) { 1680 ret = PTR_ERR(evt); 1681 break; 1682 } 1683 mdev_state->msi_evtfd = evt; 1684 mdev_state->irq_index = index; 1685 break; 1686 } 1687 1688 if (!is_msi(mdev_state)) { 1689 ret = -EINVAL; 1690 break; 1691 } 1692 1693 if (flags & VFIO_IRQ_SET_DATA_NONE) { 1694 mtty_trigger_interrupt(mdev_state); 1695 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 1696 uint8_t trigger = *(uint8_t *)data; 1697 1698 if (trigger) 1699 mtty_trigger_interrupt(mdev_state); 1700 } 1701 break; 1702 } 1703 break; 1704 case VFIO_PCI_MSIX_IRQ_INDEX: 1705 dev_dbg(mdev_state->vdev.dev, "%s: MSIX_IRQ\n", __func__); 1706 ret = -ENOTTY; 1707 break; 1708 case VFIO_PCI_ERR_IRQ_INDEX: 1709 dev_dbg(mdev_state->vdev.dev, "%s: ERR_IRQ\n", __func__); 1710 ret = -ENOTTY; 1711 break; 1712 case VFIO_PCI_REQ_IRQ_INDEX: 1713 dev_dbg(mdev_state->vdev.dev, "%s: REQ_IRQ\n", __func__); 1714 ret = -ENOTTY; 1715 break; 1716 } 1717 1718 mutex_unlock(&mdev_state->ops_lock); 1719 return ret; 1720 } 1721 1722 static int mtty_get_region_info(struct mdev_state *mdev_state, 1723 struct vfio_region_info *region_info, 1724 u16 *cap_type_id, void **cap_type) 1725 { 1726 unsigned int size = 0; 1727 u32 bar_index; 1728 1729 bar_index = region_info->index; 1730 if (bar_index >= VFIO_PCI_NUM_REGIONS) 1731 return -EINVAL; 1732 1733 mutex_lock(&mdev_state->ops_lock); 1734 1735 switch (bar_index) { 1736 case VFIO_PCI_CONFIG_REGION_INDEX: 1737 size = MTTY_CONFIG_SPACE_SIZE; 1738 break; 1739 case VFIO_PCI_BAR0_REGION_INDEX: 1740 size = MTTY_IO_BAR_SIZE; 1741 break; 1742 case VFIO_PCI_BAR1_REGION_INDEX: 1743 if (mdev_state->nr_ports == 2) 1744 size = MTTY_IO_BAR_SIZE; 1745 break; 1746 default: 1747 size = 0; 1748 break; 1749 } 1750 1751 mdev_state->region_info[bar_index].size = size; 1752 mdev_state->region_info[bar_index].vfio_offset = 1753 MTTY_VFIO_PCI_INDEX_TO_OFFSET(bar_index); 1754 1755 region_info->size = size; 1756 region_info->offset = MTTY_VFIO_PCI_INDEX_TO_OFFSET(bar_index); 1757 region_info->flags = VFIO_REGION_INFO_FLAG_READ | 1758 VFIO_REGION_INFO_FLAG_WRITE; 1759 mutex_unlock(&mdev_state->ops_lock); 1760 return 0; 1761 } 1762 1763 static int mtty_get_irq_info(struct vfio_irq_info *irq_info) 1764 { 1765 if (irq_info->index != VFIO_PCI_INTX_IRQ_INDEX && 1766 irq_info->index != VFIO_PCI_MSI_IRQ_INDEX) 1767 return -EINVAL; 1768 1769 irq_info->flags = VFIO_IRQ_INFO_EVENTFD; 1770 irq_info->count = 1; 1771 1772 if (irq_info->index == VFIO_PCI_INTX_IRQ_INDEX) 1773 irq_info->flags |= VFIO_IRQ_INFO_MASKABLE | 1774 VFIO_IRQ_INFO_AUTOMASKED; 1775 else 1776 irq_info->flags |= VFIO_IRQ_INFO_NORESIZE; 1777 1778 return 0; 1779 } 1780 1781 static int mtty_get_device_info(struct vfio_device_info *dev_info) 1782 { 1783 dev_info->flags = VFIO_DEVICE_FLAGS_PCI; 1784 dev_info->num_regions = VFIO_PCI_NUM_REGIONS; 1785 dev_info->num_irqs = VFIO_PCI_NUM_IRQS; 1786 1787 return 0; 1788 } 1789 1790 static long mtty_ioctl(struct vfio_device *vdev, unsigned int cmd, 1791 unsigned long arg) 1792 { 1793 struct mdev_state *mdev_state = 1794 container_of(vdev, struct mdev_state, vdev); 1795 int ret = 0; 1796 unsigned long minsz; 1797 1798 switch (cmd) { 1799 case VFIO_DEVICE_GET_INFO: 1800 { 1801 struct vfio_device_info info; 1802 1803 minsz = offsetofend(struct vfio_device_info, num_irqs); 1804 1805 if (copy_from_user(&info, (void __user *)arg, minsz)) 1806 return -EFAULT; 1807 1808 if (info.argsz < minsz) 1809 return -EINVAL; 1810 1811 ret = mtty_get_device_info(&info); 1812 if (ret) 1813 return ret; 1814 1815 memcpy(&mdev_state->dev_info, &info, sizeof(info)); 1816 1817 if (copy_to_user((void __user *)arg, &info, minsz)) 1818 return -EFAULT; 1819 1820 return 0; 1821 } 1822 case VFIO_DEVICE_GET_REGION_INFO: 1823 { 1824 struct vfio_region_info info; 1825 u16 cap_type_id = 0; 1826 void *cap_type = NULL; 1827 1828 minsz = offsetofend(struct vfio_region_info, offset); 1829 1830 if (copy_from_user(&info, (void __user *)arg, minsz)) 1831 return -EFAULT; 1832 1833 if (info.argsz < minsz) 1834 return -EINVAL; 1835 1836 ret = mtty_get_region_info(mdev_state, &info, &cap_type_id, 1837 &cap_type); 1838 if (ret) 1839 return ret; 1840 1841 if (copy_to_user((void __user *)arg, &info, minsz)) 1842 return -EFAULT; 1843 1844 return 0; 1845 } 1846 1847 case VFIO_DEVICE_GET_IRQ_INFO: 1848 { 1849 struct vfio_irq_info info; 1850 1851 minsz = offsetofend(struct vfio_irq_info, count); 1852 1853 if (copy_from_user(&info, (void __user *)arg, minsz)) 1854 return -EFAULT; 1855 1856 if ((info.argsz < minsz) || 1857 (info.index >= mdev_state->dev_info.num_irqs)) 1858 return -EINVAL; 1859 1860 ret = mtty_get_irq_info(&info); 1861 if (ret) 1862 return ret; 1863 1864 if (copy_to_user((void __user *)arg, &info, minsz)) 1865 return -EFAULT; 1866 1867 return 0; 1868 } 1869 case VFIO_DEVICE_SET_IRQS: 1870 { 1871 struct vfio_irq_set hdr; 1872 u8 *data = NULL, *ptr = NULL; 1873 size_t data_size = 0; 1874 1875 minsz = offsetofend(struct vfio_irq_set, count); 1876 1877 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 1878 return -EFAULT; 1879 1880 ret = vfio_set_irqs_validate_and_prepare(&hdr, 1881 mdev_state->dev_info.num_irqs, 1882 VFIO_PCI_NUM_IRQS, 1883 &data_size); 1884 if (ret) 1885 return ret; 1886 1887 if (data_size) { 1888 ptr = data = memdup_user((void __user *)(arg + minsz), 1889 data_size); 1890 if (IS_ERR(data)) 1891 return PTR_ERR(data); 1892 } 1893 1894 ret = mtty_set_irqs(mdev_state, hdr.flags, hdr.index, hdr.start, 1895 hdr.count, data); 1896 1897 kfree(ptr); 1898 return ret; 1899 } 1900 case VFIO_DEVICE_RESET: 1901 return mtty_reset(mdev_state); 1902 } 1903 return -ENOTTY; 1904 } 1905 1906 static ssize_t 1907 sample_mdev_dev_show(struct device *dev, struct device_attribute *attr, 1908 char *buf) 1909 { 1910 return sprintf(buf, "This is MDEV %s\n", dev_name(dev)); 1911 } 1912 1913 static DEVICE_ATTR_RO(sample_mdev_dev); 1914 1915 static struct attribute *mdev_dev_attrs[] = { 1916 &dev_attr_sample_mdev_dev.attr, 1917 NULL, 1918 }; 1919 1920 static const struct attribute_group mdev_dev_group = { 1921 .name = "vendor", 1922 .attrs = mdev_dev_attrs, 1923 }; 1924 1925 static const struct attribute_group *mdev_dev_groups[] = { 1926 &mdev_dev_group, 1927 NULL, 1928 }; 1929 1930 static unsigned int mtty_get_available(struct mdev_type *mtype) 1931 { 1932 struct mtty_type *type = container_of(mtype, struct mtty_type, type); 1933 1934 return atomic_read(&mdev_avail_ports) / type->nr_ports; 1935 } 1936 1937 static void mtty_close(struct vfio_device *vdev) 1938 { 1939 struct mdev_state *mdev_state = 1940 container_of(vdev, struct mdev_state, vdev); 1941 1942 mtty_disable_files(mdev_state); 1943 mtty_disable_intx(mdev_state); 1944 mtty_disable_msi(mdev_state); 1945 } 1946 1947 static const struct vfio_device_ops mtty_dev_ops = { 1948 .name = "vfio-mtty", 1949 .init = mtty_init_dev, 1950 .release = mtty_release_dev, 1951 .read = mtty_read, 1952 .write = mtty_write, 1953 .ioctl = mtty_ioctl, 1954 .bind_iommufd = vfio_iommufd_emulated_bind, 1955 .unbind_iommufd = vfio_iommufd_emulated_unbind, 1956 .attach_ioas = vfio_iommufd_emulated_attach_ioas, 1957 .detach_ioas = vfio_iommufd_emulated_detach_ioas, 1958 .close_device = mtty_close, 1959 }; 1960 1961 static struct mdev_driver mtty_driver = { 1962 .device_api = VFIO_DEVICE_API_PCI_STRING, 1963 .driver = { 1964 .name = "mtty", 1965 .owner = THIS_MODULE, 1966 .mod_name = KBUILD_MODNAME, 1967 .dev_groups = mdev_dev_groups, 1968 }, 1969 .probe = mtty_probe, 1970 .remove = mtty_remove, 1971 .get_available = mtty_get_available, 1972 }; 1973 1974 static void mtty_device_release(struct device *dev) 1975 { 1976 dev_dbg(dev, "mtty: released\n"); 1977 } 1978 1979 static int __init mtty_dev_init(void) 1980 { 1981 int ret = 0; 1982 1983 pr_info("mtty_dev: %s\n", __func__); 1984 1985 memset(&mtty_dev, 0, sizeof(mtty_dev)); 1986 1987 idr_init(&mtty_dev.vd_idr); 1988 1989 ret = alloc_chrdev_region(&mtty_dev.vd_devt, 0, MINORMASK + 1, 1990 MTTY_NAME); 1991 1992 if (ret < 0) { 1993 pr_err("Error: failed to register mtty_dev, err:%d\n", ret); 1994 return ret; 1995 } 1996 1997 cdev_init(&mtty_dev.vd_cdev, &vd_fops); 1998 cdev_add(&mtty_dev.vd_cdev, mtty_dev.vd_devt, MINORMASK + 1); 1999 2000 pr_info("major_number:%d\n", MAJOR(mtty_dev.vd_devt)); 2001 2002 ret = mdev_register_driver(&mtty_driver); 2003 if (ret) 2004 goto err_cdev; 2005 2006 mtty_dev.vd_class = class_create(MTTY_CLASS_NAME); 2007 2008 if (IS_ERR(mtty_dev.vd_class)) { 2009 pr_err("Error: failed to register mtty_dev class\n"); 2010 ret = PTR_ERR(mtty_dev.vd_class); 2011 goto err_driver; 2012 } 2013 2014 mtty_dev.dev.class = mtty_dev.vd_class; 2015 mtty_dev.dev.release = mtty_device_release; 2016 dev_set_name(&mtty_dev.dev, "%s", MTTY_NAME); 2017 2018 ret = device_register(&mtty_dev.dev); 2019 if (ret) 2020 goto err_put; 2021 2022 ret = mdev_register_parent(&mtty_dev.parent, &mtty_dev.dev, 2023 &mtty_driver, mtty_mdev_types, 2024 ARRAY_SIZE(mtty_mdev_types)); 2025 if (ret) 2026 goto err_device; 2027 return 0; 2028 2029 err_device: 2030 device_del(&mtty_dev.dev); 2031 err_put: 2032 put_device(&mtty_dev.dev); 2033 class_destroy(mtty_dev.vd_class); 2034 err_driver: 2035 mdev_unregister_driver(&mtty_driver); 2036 err_cdev: 2037 cdev_del(&mtty_dev.vd_cdev); 2038 unregister_chrdev_region(mtty_dev.vd_devt, MINORMASK + 1); 2039 return ret; 2040 } 2041 2042 static void __exit mtty_dev_exit(void) 2043 { 2044 mtty_dev.dev.bus = NULL; 2045 mdev_unregister_parent(&mtty_dev.parent); 2046 2047 device_unregister(&mtty_dev.dev); 2048 idr_destroy(&mtty_dev.vd_idr); 2049 mdev_unregister_driver(&mtty_driver); 2050 cdev_del(&mtty_dev.vd_cdev); 2051 unregister_chrdev_region(mtty_dev.vd_devt, MINORMASK + 1); 2052 class_destroy(mtty_dev.vd_class); 2053 mtty_dev.vd_class = NULL; 2054 pr_info("mtty_dev: Unloaded!\n"); 2055 } 2056 2057 module_init(mtty_dev_init) 2058 module_exit(mtty_dev_exit) 2059 2060 MODULE_LICENSE("GPL v2"); 2061 MODULE_DESCRIPTION("Test driver that simulate serial port over PCI"); 2062 MODULE_VERSION(VERSION_STRING); 2063 MODULE_AUTHOR(DRIVER_AUTHOR); 2064