1 /* 2 * KVMGT - the implementation of Intel mediated pass-through framework for KVM 3 * 4 * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 * SOFTWARE. 24 * 25 * Authors: 26 * Kevin Tian <kevin.tian@intel.com> 27 * Jike Song <jike.song@intel.com> 28 * Xiaoguang Chen <xiaoguang.chen@intel.com> 29 * Eddie Dong <eddie.dong@intel.com> 30 * 31 * Contributors: 32 * Niu Bing <bing.niu@intel.com> 33 * Zhi Wang <zhi.a.wang@intel.com> 34 */ 35 36 #include <linux/init.h> 37 #include <linux/mm.h> 38 #include <linux/kthread.h> 39 #include <linux/sched/mm.h> 40 #include <linux/types.h> 41 #include <linux/list.h> 42 #include <linux/rbtree.h> 43 #include <linux/spinlock.h> 44 #include <linux/eventfd.h> 45 #include <linux/mdev.h> 46 #include <linux/debugfs.h> 47 48 #include <linux/nospec.h> 49 50 #include <drm/drm_edid.h> 51 52 #include "i915_drv.h" 53 #include "intel_gvt.h" 54 #include "gvt.h" 55 56 MODULE_IMPORT_NS(DMA_BUF); 57 MODULE_IMPORT_NS(I915_GVT); 58 59 /* helper macros copied from vfio-pci */ 60 #define VFIO_PCI_OFFSET_SHIFT 40 61 #define VFIO_PCI_OFFSET_TO_INDEX(off) (off >> VFIO_PCI_OFFSET_SHIFT) 62 #define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT) 63 #define VFIO_PCI_OFFSET_MASK (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1) 64 65 #define EDID_BLOB_OFFSET (PAGE_SIZE/2) 66 67 #define OPREGION_SIGNATURE "IntelGraphicsMem" 68 69 struct vfio_region; 70 struct intel_vgpu_regops { 71 size_t (*rw)(struct intel_vgpu *vgpu, char *buf, 72 size_t count, loff_t *ppos, bool iswrite); 73 void (*release)(struct intel_vgpu *vgpu, 74 struct vfio_region *region); 75 }; 76 77 struct vfio_region { 78 u32 type; 79 u32 subtype; 80 size_t size; 81 u32 flags; 82 const struct intel_vgpu_regops *ops; 83 void *data; 84 }; 85 86 struct vfio_edid_region { 87 struct vfio_region_gfx_edid vfio_edid_regs; 88 void *edid_blob; 89 }; 90 91 struct kvmgt_pgfn { 92 gfn_t gfn; 93 struct hlist_node hnode; 94 }; 95 96 struct gvt_dma { 97 struct intel_vgpu *vgpu; 98 struct rb_node gfn_node; 99 struct rb_node dma_addr_node; 100 gfn_t gfn; 101 dma_addr_t dma_addr; 102 unsigned long size; 103 struct kref ref; 104 }; 105 106 #define vfio_dev_to_vgpu(vfio_dev) \ 107 container_of((vfio_dev), struct intel_vgpu, vfio_device) 108 109 static void kvmgt_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, 110 const u8 *val, int len, 111 struct kvm_page_track_notifier_node *node); 112 static void kvmgt_page_track_flush_slot(struct kvm *kvm, 113 struct kvm_memory_slot *slot, 114 struct kvm_page_track_notifier_node *node); 115 116 static ssize_t available_instances_show(struct mdev_type *mtype, 117 struct mdev_type_attribute *attr, 118 char *buf) 119 { 120 struct intel_vgpu_type *type = 121 container_of(mtype, struct intel_vgpu_type, type); 122 123 return sprintf(buf, "%u\n", type->avail_instance); 124 } 125 126 static ssize_t device_api_show(struct mdev_type *mtype, 127 struct mdev_type_attribute *attr, char *buf) 128 { 129 return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING); 130 } 131 132 static ssize_t description_show(struct mdev_type *mtype, 133 struct mdev_type_attribute *attr, char *buf) 134 { 135 struct intel_vgpu_type *type = 136 container_of(mtype, struct intel_vgpu_type, type); 137 138 return sprintf(buf, "low_gm_size: %dMB\nhigh_gm_size: %dMB\n" 139 "fence: %d\nresolution: %s\n" 140 "weight: %d\n", 141 BYTES_TO_MB(type->conf->low_mm), 142 BYTES_TO_MB(type->conf->high_mm), 143 type->conf->fence, vgpu_edid_str(type->conf->edid), 144 type->conf->weight); 145 } 146 147 static ssize_t name_show(struct mdev_type *mtype, 148 struct mdev_type_attribute *attr, char *buf) 149 { 150 return sprintf(buf, "%s\n", mtype->sysfs_name); 151 } 152 153 static MDEV_TYPE_ATTR_RO(available_instances); 154 static MDEV_TYPE_ATTR_RO(device_api); 155 static MDEV_TYPE_ATTR_RO(description); 156 static MDEV_TYPE_ATTR_RO(name); 157 158 static const struct attribute *gvt_type_attrs[] = { 159 &mdev_type_attr_available_instances.attr, 160 &mdev_type_attr_device_api.attr, 161 &mdev_type_attr_description.attr, 162 &mdev_type_attr_name.attr, 163 NULL, 164 }; 165 166 static void gvt_unpin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn, 167 unsigned long size) 168 { 169 vfio_unpin_pages(&vgpu->vfio_device, gfn << PAGE_SHIFT, 170 DIV_ROUND_UP(size, PAGE_SIZE)); 171 } 172 173 /* Pin a normal or compound guest page for dma. */ 174 static int gvt_pin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn, 175 unsigned long size, struct page **page) 176 { 177 int total_pages = DIV_ROUND_UP(size, PAGE_SIZE); 178 struct page *base_page = NULL; 179 int npage; 180 int ret; 181 182 /* 183 * We pin the pages one-by-one to avoid allocating a big arrary 184 * on stack to hold pfns. 185 */ 186 for (npage = 0; npage < total_pages; npage++) { 187 dma_addr_t cur_iova = (gfn + npage) << PAGE_SHIFT; 188 struct page *cur_page; 189 190 ret = vfio_pin_pages(&vgpu->vfio_device, cur_iova, 1, 191 IOMMU_READ | IOMMU_WRITE, &cur_page); 192 if (ret != 1) { 193 gvt_vgpu_err("vfio_pin_pages failed for iova %pad, ret %d\n", 194 &cur_iova, ret); 195 goto err; 196 } 197 198 if (npage == 0) 199 base_page = cur_page; 200 else if (base_page + npage != cur_page) { 201 gvt_vgpu_err("The pages are not continuous\n"); 202 ret = -EINVAL; 203 npage++; 204 goto err; 205 } 206 } 207 208 *page = base_page; 209 return 0; 210 err: 211 gvt_unpin_guest_page(vgpu, gfn, npage * PAGE_SIZE); 212 return ret; 213 } 214 215 static int gvt_dma_map_page(struct intel_vgpu *vgpu, unsigned long gfn, 216 dma_addr_t *dma_addr, unsigned long size) 217 { 218 struct device *dev = vgpu->gvt->gt->i915->drm.dev; 219 struct page *page = NULL; 220 int ret; 221 222 ret = gvt_pin_guest_page(vgpu, gfn, size, &page); 223 if (ret) 224 return ret; 225 226 /* Setup DMA mapping. */ 227 *dma_addr = dma_map_page(dev, page, 0, size, DMA_BIDIRECTIONAL); 228 if (dma_mapping_error(dev, *dma_addr)) { 229 gvt_vgpu_err("DMA mapping failed for pfn 0x%lx, ret %d\n", 230 page_to_pfn(page), ret); 231 gvt_unpin_guest_page(vgpu, gfn, size); 232 return -ENOMEM; 233 } 234 235 return 0; 236 } 237 238 static void gvt_dma_unmap_page(struct intel_vgpu *vgpu, unsigned long gfn, 239 dma_addr_t dma_addr, unsigned long size) 240 { 241 struct device *dev = vgpu->gvt->gt->i915->drm.dev; 242 243 dma_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL); 244 gvt_unpin_guest_page(vgpu, gfn, size); 245 } 246 247 static struct gvt_dma *__gvt_cache_find_dma_addr(struct intel_vgpu *vgpu, 248 dma_addr_t dma_addr) 249 { 250 struct rb_node *node = vgpu->dma_addr_cache.rb_node; 251 struct gvt_dma *itr; 252 253 while (node) { 254 itr = rb_entry(node, struct gvt_dma, dma_addr_node); 255 256 if (dma_addr < itr->dma_addr) 257 node = node->rb_left; 258 else if (dma_addr > itr->dma_addr) 259 node = node->rb_right; 260 else 261 return itr; 262 } 263 return NULL; 264 } 265 266 static struct gvt_dma *__gvt_cache_find_gfn(struct intel_vgpu *vgpu, gfn_t gfn) 267 { 268 struct rb_node *node = vgpu->gfn_cache.rb_node; 269 struct gvt_dma *itr; 270 271 while (node) { 272 itr = rb_entry(node, struct gvt_dma, gfn_node); 273 274 if (gfn < itr->gfn) 275 node = node->rb_left; 276 else if (gfn > itr->gfn) 277 node = node->rb_right; 278 else 279 return itr; 280 } 281 return NULL; 282 } 283 284 static int __gvt_cache_add(struct intel_vgpu *vgpu, gfn_t gfn, 285 dma_addr_t dma_addr, unsigned long size) 286 { 287 struct gvt_dma *new, *itr; 288 struct rb_node **link, *parent = NULL; 289 290 new = kzalloc(sizeof(struct gvt_dma), GFP_KERNEL); 291 if (!new) 292 return -ENOMEM; 293 294 new->vgpu = vgpu; 295 new->gfn = gfn; 296 new->dma_addr = dma_addr; 297 new->size = size; 298 kref_init(&new->ref); 299 300 /* gfn_cache maps gfn to struct gvt_dma. */ 301 link = &vgpu->gfn_cache.rb_node; 302 while (*link) { 303 parent = *link; 304 itr = rb_entry(parent, struct gvt_dma, gfn_node); 305 306 if (gfn < itr->gfn) 307 link = &parent->rb_left; 308 else 309 link = &parent->rb_right; 310 } 311 rb_link_node(&new->gfn_node, parent, link); 312 rb_insert_color(&new->gfn_node, &vgpu->gfn_cache); 313 314 /* dma_addr_cache maps dma addr to struct gvt_dma. */ 315 parent = NULL; 316 link = &vgpu->dma_addr_cache.rb_node; 317 while (*link) { 318 parent = *link; 319 itr = rb_entry(parent, struct gvt_dma, dma_addr_node); 320 321 if (dma_addr < itr->dma_addr) 322 link = &parent->rb_left; 323 else 324 link = &parent->rb_right; 325 } 326 rb_link_node(&new->dma_addr_node, parent, link); 327 rb_insert_color(&new->dma_addr_node, &vgpu->dma_addr_cache); 328 329 vgpu->nr_cache_entries++; 330 return 0; 331 } 332 333 static void __gvt_cache_remove_entry(struct intel_vgpu *vgpu, 334 struct gvt_dma *entry) 335 { 336 rb_erase(&entry->gfn_node, &vgpu->gfn_cache); 337 rb_erase(&entry->dma_addr_node, &vgpu->dma_addr_cache); 338 kfree(entry); 339 vgpu->nr_cache_entries--; 340 } 341 342 static void gvt_cache_destroy(struct intel_vgpu *vgpu) 343 { 344 struct gvt_dma *dma; 345 struct rb_node *node = NULL; 346 347 for (;;) { 348 mutex_lock(&vgpu->cache_lock); 349 node = rb_first(&vgpu->gfn_cache); 350 if (!node) { 351 mutex_unlock(&vgpu->cache_lock); 352 break; 353 } 354 dma = rb_entry(node, struct gvt_dma, gfn_node); 355 gvt_dma_unmap_page(vgpu, dma->gfn, dma->dma_addr, dma->size); 356 __gvt_cache_remove_entry(vgpu, dma); 357 mutex_unlock(&vgpu->cache_lock); 358 } 359 } 360 361 static void gvt_cache_init(struct intel_vgpu *vgpu) 362 { 363 vgpu->gfn_cache = RB_ROOT; 364 vgpu->dma_addr_cache = RB_ROOT; 365 vgpu->nr_cache_entries = 0; 366 mutex_init(&vgpu->cache_lock); 367 } 368 369 static void kvmgt_protect_table_init(struct intel_vgpu *info) 370 { 371 hash_init(info->ptable); 372 } 373 374 static void kvmgt_protect_table_destroy(struct intel_vgpu *info) 375 { 376 struct kvmgt_pgfn *p; 377 struct hlist_node *tmp; 378 int i; 379 380 hash_for_each_safe(info->ptable, i, tmp, p, hnode) { 381 hash_del(&p->hnode); 382 kfree(p); 383 } 384 } 385 386 static struct kvmgt_pgfn * 387 __kvmgt_protect_table_find(struct intel_vgpu *info, gfn_t gfn) 388 { 389 struct kvmgt_pgfn *p, *res = NULL; 390 391 hash_for_each_possible(info->ptable, p, hnode, gfn) { 392 if (gfn == p->gfn) { 393 res = p; 394 break; 395 } 396 } 397 398 return res; 399 } 400 401 static bool kvmgt_gfn_is_write_protected(struct intel_vgpu *info, gfn_t gfn) 402 { 403 struct kvmgt_pgfn *p; 404 405 p = __kvmgt_protect_table_find(info, gfn); 406 return !!p; 407 } 408 409 static void kvmgt_protect_table_add(struct intel_vgpu *info, gfn_t gfn) 410 { 411 struct kvmgt_pgfn *p; 412 413 if (kvmgt_gfn_is_write_protected(info, gfn)) 414 return; 415 416 p = kzalloc(sizeof(struct kvmgt_pgfn), GFP_ATOMIC); 417 if (WARN(!p, "gfn: 0x%llx\n", gfn)) 418 return; 419 420 p->gfn = gfn; 421 hash_add(info->ptable, &p->hnode, gfn); 422 } 423 424 static void kvmgt_protect_table_del(struct intel_vgpu *info, gfn_t gfn) 425 { 426 struct kvmgt_pgfn *p; 427 428 p = __kvmgt_protect_table_find(info, gfn); 429 if (p) { 430 hash_del(&p->hnode); 431 kfree(p); 432 } 433 } 434 435 static size_t intel_vgpu_reg_rw_opregion(struct intel_vgpu *vgpu, char *buf, 436 size_t count, loff_t *ppos, bool iswrite) 437 { 438 unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) - 439 VFIO_PCI_NUM_REGIONS; 440 void *base = vgpu->region[i].data; 441 loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK; 442 443 444 if (pos >= vgpu->region[i].size || iswrite) { 445 gvt_vgpu_err("invalid op or offset for Intel vgpu OpRegion\n"); 446 return -EINVAL; 447 } 448 count = min(count, (size_t)(vgpu->region[i].size - pos)); 449 memcpy(buf, base + pos, count); 450 451 return count; 452 } 453 454 static void intel_vgpu_reg_release_opregion(struct intel_vgpu *vgpu, 455 struct vfio_region *region) 456 { 457 } 458 459 static const struct intel_vgpu_regops intel_vgpu_regops_opregion = { 460 .rw = intel_vgpu_reg_rw_opregion, 461 .release = intel_vgpu_reg_release_opregion, 462 }; 463 464 static int handle_edid_regs(struct intel_vgpu *vgpu, 465 struct vfio_edid_region *region, char *buf, 466 size_t count, u16 offset, bool is_write) 467 { 468 struct vfio_region_gfx_edid *regs = ®ion->vfio_edid_regs; 469 unsigned int data; 470 471 if (offset + count > sizeof(*regs)) 472 return -EINVAL; 473 474 if (count != 4) 475 return -EINVAL; 476 477 if (is_write) { 478 data = *((unsigned int *)buf); 479 switch (offset) { 480 case offsetof(struct vfio_region_gfx_edid, link_state): 481 if (data == VFIO_DEVICE_GFX_LINK_STATE_UP) { 482 if (!drm_edid_block_valid( 483 (u8 *)region->edid_blob, 484 0, 485 true, 486 NULL)) { 487 gvt_vgpu_err("invalid EDID blob\n"); 488 return -EINVAL; 489 } 490 intel_vgpu_emulate_hotplug(vgpu, true); 491 } else if (data == VFIO_DEVICE_GFX_LINK_STATE_DOWN) 492 intel_vgpu_emulate_hotplug(vgpu, false); 493 else { 494 gvt_vgpu_err("invalid EDID link state %d\n", 495 regs->link_state); 496 return -EINVAL; 497 } 498 regs->link_state = data; 499 break; 500 case offsetof(struct vfio_region_gfx_edid, edid_size): 501 if (data > regs->edid_max_size) { 502 gvt_vgpu_err("EDID size is bigger than %d!\n", 503 regs->edid_max_size); 504 return -EINVAL; 505 } 506 regs->edid_size = data; 507 break; 508 default: 509 /* read-only regs */ 510 gvt_vgpu_err("write read-only EDID region at offset %d\n", 511 offset); 512 return -EPERM; 513 } 514 } else { 515 memcpy(buf, (char *)regs + offset, count); 516 } 517 518 return count; 519 } 520 521 static int handle_edid_blob(struct vfio_edid_region *region, char *buf, 522 size_t count, u16 offset, bool is_write) 523 { 524 if (offset + count > region->vfio_edid_regs.edid_size) 525 return -EINVAL; 526 527 if (is_write) 528 memcpy(region->edid_blob + offset, buf, count); 529 else 530 memcpy(buf, region->edid_blob + offset, count); 531 532 return count; 533 } 534 535 static size_t intel_vgpu_reg_rw_edid(struct intel_vgpu *vgpu, char *buf, 536 size_t count, loff_t *ppos, bool iswrite) 537 { 538 int ret; 539 unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) - 540 VFIO_PCI_NUM_REGIONS; 541 struct vfio_edid_region *region = vgpu->region[i].data; 542 loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK; 543 544 if (pos < region->vfio_edid_regs.edid_offset) { 545 ret = handle_edid_regs(vgpu, region, buf, count, pos, iswrite); 546 } else { 547 pos -= EDID_BLOB_OFFSET; 548 ret = handle_edid_blob(region, buf, count, pos, iswrite); 549 } 550 551 if (ret < 0) 552 gvt_vgpu_err("failed to access EDID region\n"); 553 554 return ret; 555 } 556 557 static void intel_vgpu_reg_release_edid(struct intel_vgpu *vgpu, 558 struct vfio_region *region) 559 { 560 kfree(region->data); 561 } 562 563 static const struct intel_vgpu_regops intel_vgpu_regops_edid = { 564 .rw = intel_vgpu_reg_rw_edid, 565 .release = intel_vgpu_reg_release_edid, 566 }; 567 568 static int intel_vgpu_register_reg(struct intel_vgpu *vgpu, 569 unsigned int type, unsigned int subtype, 570 const struct intel_vgpu_regops *ops, 571 size_t size, u32 flags, void *data) 572 { 573 struct vfio_region *region; 574 575 region = krealloc(vgpu->region, 576 (vgpu->num_regions + 1) * sizeof(*region), 577 GFP_KERNEL); 578 if (!region) 579 return -ENOMEM; 580 581 vgpu->region = region; 582 vgpu->region[vgpu->num_regions].type = type; 583 vgpu->region[vgpu->num_regions].subtype = subtype; 584 vgpu->region[vgpu->num_regions].ops = ops; 585 vgpu->region[vgpu->num_regions].size = size; 586 vgpu->region[vgpu->num_regions].flags = flags; 587 vgpu->region[vgpu->num_regions].data = data; 588 vgpu->num_regions++; 589 return 0; 590 } 591 592 int intel_gvt_set_opregion(struct intel_vgpu *vgpu) 593 { 594 void *base; 595 int ret; 596 597 /* Each vgpu has its own opregion, although VFIO would create another 598 * one later. This one is used to expose opregion to VFIO. And the 599 * other one created by VFIO later, is used by guest actually. 600 */ 601 base = vgpu_opregion(vgpu)->va; 602 if (!base) 603 return -ENOMEM; 604 605 if (memcmp(base, OPREGION_SIGNATURE, 16)) { 606 memunmap(base); 607 return -EINVAL; 608 } 609 610 ret = intel_vgpu_register_reg(vgpu, 611 PCI_VENDOR_ID_INTEL | VFIO_REGION_TYPE_PCI_VENDOR_TYPE, 612 VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION, 613 &intel_vgpu_regops_opregion, OPREGION_SIZE, 614 VFIO_REGION_INFO_FLAG_READ, base); 615 616 return ret; 617 } 618 619 int intel_gvt_set_edid(struct intel_vgpu *vgpu, int port_num) 620 { 621 struct intel_vgpu_port *port = intel_vgpu_port(vgpu, port_num); 622 struct vfio_edid_region *base; 623 int ret; 624 625 base = kzalloc(sizeof(*base), GFP_KERNEL); 626 if (!base) 627 return -ENOMEM; 628 629 /* TODO: Add multi-port and EDID extension block support */ 630 base->vfio_edid_regs.edid_offset = EDID_BLOB_OFFSET; 631 base->vfio_edid_regs.edid_max_size = EDID_SIZE; 632 base->vfio_edid_regs.edid_size = EDID_SIZE; 633 base->vfio_edid_regs.max_xres = vgpu_edid_xres(port->id); 634 base->vfio_edid_regs.max_yres = vgpu_edid_yres(port->id); 635 base->edid_blob = port->edid->edid_block; 636 637 ret = intel_vgpu_register_reg(vgpu, 638 VFIO_REGION_TYPE_GFX, 639 VFIO_REGION_SUBTYPE_GFX_EDID, 640 &intel_vgpu_regops_edid, EDID_SIZE, 641 VFIO_REGION_INFO_FLAG_READ | 642 VFIO_REGION_INFO_FLAG_WRITE | 643 VFIO_REGION_INFO_FLAG_CAPS, base); 644 645 return ret; 646 } 647 648 static void intel_vgpu_dma_unmap(struct vfio_device *vfio_dev, u64 iova, 649 u64 length) 650 { 651 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev); 652 struct gvt_dma *entry; 653 u64 iov_pfn = iova >> PAGE_SHIFT; 654 u64 end_iov_pfn = iov_pfn + length / PAGE_SIZE; 655 656 mutex_lock(&vgpu->cache_lock); 657 for (; iov_pfn < end_iov_pfn; iov_pfn++) { 658 entry = __gvt_cache_find_gfn(vgpu, iov_pfn); 659 if (!entry) 660 continue; 661 662 gvt_dma_unmap_page(vgpu, entry->gfn, entry->dma_addr, 663 entry->size); 664 __gvt_cache_remove_entry(vgpu, entry); 665 } 666 mutex_unlock(&vgpu->cache_lock); 667 } 668 669 static bool __kvmgt_vgpu_exist(struct intel_vgpu *vgpu) 670 { 671 struct intel_vgpu *itr; 672 int id; 673 bool ret = false; 674 675 mutex_lock(&vgpu->gvt->lock); 676 for_each_active_vgpu(vgpu->gvt, itr, id) { 677 if (!itr->attached) 678 continue; 679 680 if (vgpu->vfio_device.kvm == itr->vfio_device.kvm) { 681 ret = true; 682 goto out; 683 } 684 } 685 out: 686 mutex_unlock(&vgpu->gvt->lock); 687 return ret; 688 } 689 690 static int intel_vgpu_open_device(struct vfio_device *vfio_dev) 691 { 692 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev); 693 694 if (vgpu->attached) 695 return -EEXIST; 696 697 if (!vgpu->vfio_device.kvm || 698 vgpu->vfio_device.kvm->mm != current->mm) { 699 gvt_vgpu_err("KVM is required to use Intel vGPU\n"); 700 return -ESRCH; 701 } 702 703 kvm_get_kvm(vgpu->vfio_device.kvm); 704 705 if (__kvmgt_vgpu_exist(vgpu)) 706 return -EEXIST; 707 708 vgpu->attached = true; 709 710 kvmgt_protect_table_init(vgpu); 711 gvt_cache_init(vgpu); 712 713 vgpu->track_node.track_write = kvmgt_page_track_write; 714 vgpu->track_node.track_flush_slot = kvmgt_page_track_flush_slot; 715 kvm_page_track_register_notifier(vgpu->vfio_device.kvm, 716 &vgpu->track_node); 717 718 debugfs_create_ulong(KVMGT_DEBUGFS_FILENAME, 0444, vgpu->debugfs, 719 &vgpu->nr_cache_entries); 720 721 intel_gvt_activate_vgpu(vgpu); 722 723 atomic_set(&vgpu->released, 0); 724 return 0; 725 } 726 727 static void intel_vgpu_release_msi_eventfd_ctx(struct intel_vgpu *vgpu) 728 { 729 struct eventfd_ctx *trigger; 730 731 trigger = vgpu->msi_trigger; 732 if (trigger) { 733 eventfd_ctx_put(trigger); 734 vgpu->msi_trigger = NULL; 735 } 736 } 737 738 static void intel_vgpu_close_device(struct vfio_device *vfio_dev) 739 { 740 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev); 741 742 if (!vgpu->attached) 743 return; 744 745 if (atomic_cmpxchg(&vgpu->released, 0, 1)) 746 return; 747 748 intel_gvt_release_vgpu(vgpu); 749 750 debugfs_remove(debugfs_lookup(KVMGT_DEBUGFS_FILENAME, vgpu->debugfs)); 751 752 kvm_page_track_unregister_notifier(vgpu->vfio_device.kvm, 753 &vgpu->track_node); 754 kvmgt_protect_table_destroy(vgpu); 755 gvt_cache_destroy(vgpu); 756 757 intel_vgpu_release_msi_eventfd_ctx(vgpu); 758 759 vgpu->attached = false; 760 761 if (vgpu->vfio_device.kvm) 762 kvm_put_kvm(vgpu->vfio_device.kvm); 763 } 764 765 static u64 intel_vgpu_get_bar_addr(struct intel_vgpu *vgpu, int bar) 766 { 767 u32 start_lo, start_hi; 768 u32 mem_type; 769 770 start_lo = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) & 771 PCI_BASE_ADDRESS_MEM_MASK; 772 mem_type = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) & 773 PCI_BASE_ADDRESS_MEM_TYPE_MASK; 774 775 switch (mem_type) { 776 case PCI_BASE_ADDRESS_MEM_TYPE_64: 777 start_hi = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space 778 + bar + 4)); 779 break; 780 case PCI_BASE_ADDRESS_MEM_TYPE_32: 781 case PCI_BASE_ADDRESS_MEM_TYPE_1M: 782 /* 1M mem BAR treated as 32-bit BAR */ 783 default: 784 /* mem unknown type treated as 32-bit BAR */ 785 start_hi = 0; 786 break; 787 } 788 789 return ((u64)start_hi << 32) | start_lo; 790 } 791 792 static int intel_vgpu_bar_rw(struct intel_vgpu *vgpu, int bar, u64 off, 793 void *buf, unsigned int count, bool is_write) 794 { 795 u64 bar_start = intel_vgpu_get_bar_addr(vgpu, bar); 796 int ret; 797 798 if (is_write) 799 ret = intel_vgpu_emulate_mmio_write(vgpu, 800 bar_start + off, buf, count); 801 else 802 ret = intel_vgpu_emulate_mmio_read(vgpu, 803 bar_start + off, buf, count); 804 return ret; 805 } 806 807 static inline bool intel_vgpu_in_aperture(struct intel_vgpu *vgpu, u64 off) 808 { 809 return off >= vgpu_aperture_offset(vgpu) && 810 off < vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu); 811 } 812 813 static int intel_vgpu_aperture_rw(struct intel_vgpu *vgpu, u64 off, 814 void *buf, unsigned long count, bool is_write) 815 { 816 void __iomem *aperture_va; 817 818 if (!intel_vgpu_in_aperture(vgpu, off) || 819 !intel_vgpu_in_aperture(vgpu, off + count)) { 820 gvt_vgpu_err("Invalid aperture offset %llu\n", off); 821 return -EINVAL; 822 } 823 824 aperture_va = io_mapping_map_wc(&vgpu->gvt->gt->ggtt->iomap, 825 ALIGN_DOWN(off, PAGE_SIZE), 826 count + offset_in_page(off)); 827 if (!aperture_va) 828 return -EIO; 829 830 if (is_write) 831 memcpy_toio(aperture_va + offset_in_page(off), buf, count); 832 else 833 memcpy_fromio(buf, aperture_va + offset_in_page(off), count); 834 835 io_mapping_unmap(aperture_va); 836 837 return 0; 838 } 839 840 static ssize_t intel_vgpu_rw(struct intel_vgpu *vgpu, char *buf, 841 size_t count, loff_t *ppos, bool is_write) 842 { 843 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); 844 u64 pos = *ppos & VFIO_PCI_OFFSET_MASK; 845 int ret = -EINVAL; 846 847 848 if (index >= VFIO_PCI_NUM_REGIONS + vgpu->num_regions) { 849 gvt_vgpu_err("invalid index: %u\n", index); 850 return -EINVAL; 851 } 852 853 switch (index) { 854 case VFIO_PCI_CONFIG_REGION_INDEX: 855 if (is_write) 856 ret = intel_vgpu_emulate_cfg_write(vgpu, pos, 857 buf, count); 858 else 859 ret = intel_vgpu_emulate_cfg_read(vgpu, pos, 860 buf, count); 861 break; 862 case VFIO_PCI_BAR0_REGION_INDEX: 863 ret = intel_vgpu_bar_rw(vgpu, PCI_BASE_ADDRESS_0, pos, 864 buf, count, is_write); 865 break; 866 case VFIO_PCI_BAR2_REGION_INDEX: 867 ret = intel_vgpu_aperture_rw(vgpu, pos, buf, count, is_write); 868 break; 869 case VFIO_PCI_BAR1_REGION_INDEX: 870 case VFIO_PCI_BAR3_REGION_INDEX: 871 case VFIO_PCI_BAR4_REGION_INDEX: 872 case VFIO_PCI_BAR5_REGION_INDEX: 873 case VFIO_PCI_VGA_REGION_INDEX: 874 case VFIO_PCI_ROM_REGION_INDEX: 875 break; 876 default: 877 if (index >= VFIO_PCI_NUM_REGIONS + vgpu->num_regions) 878 return -EINVAL; 879 880 index -= VFIO_PCI_NUM_REGIONS; 881 return vgpu->region[index].ops->rw(vgpu, buf, count, 882 ppos, is_write); 883 } 884 885 return ret == 0 ? count : ret; 886 } 887 888 static bool gtt_entry(struct intel_vgpu *vgpu, loff_t *ppos) 889 { 890 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); 891 struct intel_gvt *gvt = vgpu->gvt; 892 int offset; 893 894 /* Only allow MMIO GGTT entry access */ 895 if (index != PCI_BASE_ADDRESS_0) 896 return false; 897 898 offset = (u64)(*ppos & VFIO_PCI_OFFSET_MASK) - 899 intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_0); 900 901 return (offset >= gvt->device_info.gtt_start_offset && 902 offset < gvt->device_info.gtt_start_offset + gvt_ggtt_sz(gvt)) ? 903 true : false; 904 } 905 906 static ssize_t intel_vgpu_read(struct vfio_device *vfio_dev, char __user *buf, 907 size_t count, loff_t *ppos) 908 { 909 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev); 910 unsigned int done = 0; 911 int ret; 912 913 while (count) { 914 size_t filled; 915 916 /* Only support GGTT entry 8 bytes read */ 917 if (count >= 8 && !(*ppos % 8) && 918 gtt_entry(vgpu, ppos)) { 919 u64 val; 920 921 ret = intel_vgpu_rw(vgpu, (char *)&val, sizeof(val), 922 ppos, false); 923 if (ret <= 0) 924 goto read_err; 925 926 if (copy_to_user(buf, &val, sizeof(val))) 927 goto read_err; 928 929 filled = 8; 930 } else if (count >= 4 && !(*ppos % 4)) { 931 u32 val; 932 933 ret = intel_vgpu_rw(vgpu, (char *)&val, sizeof(val), 934 ppos, false); 935 if (ret <= 0) 936 goto read_err; 937 938 if (copy_to_user(buf, &val, sizeof(val))) 939 goto read_err; 940 941 filled = 4; 942 } else if (count >= 2 && !(*ppos % 2)) { 943 u16 val; 944 945 ret = intel_vgpu_rw(vgpu, (char *)&val, sizeof(val), 946 ppos, false); 947 if (ret <= 0) 948 goto read_err; 949 950 if (copy_to_user(buf, &val, sizeof(val))) 951 goto read_err; 952 953 filled = 2; 954 } else { 955 u8 val; 956 957 ret = intel_vgpu_rw(vgpu, &val, sizeof(val), ppos, 958 false); 959 if (ret <= 0) 960 goto read_err; 961 962 if (copy_to_user(buf, &val, sizeof(val))) 963 goto read_err; 964 965 filled = 1; 966 } 967 968 count -= filled; 969 done += filled; 970 *ppos += filled; 971 buf += filled; 972 } 973 974 return done; 975 976 read_err: 977 return -EFAULT; 978 } 979 980 static ssize_t intel_vgpu_write(struct vfio_device *vfio_dev, 981 const char __user *buf, 982 size_t count, loff_t *ppos) 983 { 984 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev); 985 unsigned int done = 0; 986 int ret; 987 988 while (count) { 989 size_t filled; 990 991 /* Only support GGTT entry 8 bytes write */ 992 if (count >= 8 && !(*ppos % 8) && 993 gtt_entry(vgpu, ppos)) { 994 u64 val; 995 996 if (copy_from_user(&val, buf, sizeof(val))) 997 goto write_err; 998 999 ret = intel_vgpu_rw(vgpu, (char *)&val, sizeof(val), 1000 ppos, true); 1001 if (ret <= 0) 1002 goto write_err; 1003 1004 filled = 8; 1005 } else if (count >= 4 && !(*ppos % 4)) { 1006 u32 val; 1007 1008 if (copy_from_user(&val, buf, sizeof(val))) 1009 goto write_err; 1010 1011 ret = intel_vgpu_rw(vgpu, (char *)&val, sizeof(val), 1012 ppos, true); 1013 if (ret <= 0) 1014 goto write_err; 1015 1016 filled = 4; 1017 } else if (count >= 2 && !(*ppos % 2)) { 1018 u16 val; 1019 1020 if (copy_from_user(&val, buf, sizeof(val))) 1021 goto write_err; 1022 1023 ret = intel_vgpu_rw(vgpu, (char *)&val, 1024 sizeof(val), ppos, true); 1025 if (ret <= 0) 1026 goto write_err; 1027 1028 filled = 2; 1029 } else { 1030 u8 val; 1031 1032 if (copy_from_user(&val, buf, sizeof(val))) 1033 goto write_err; 1034 1035 ret = intel_vgpu_rw(vgpu, &val, sizeof(val), 1036 ppos, true); 1037 if (ret <= 0) 1038 goto write_err; 1039 1040 filled = 1; 1041 } 1042 1043 count -= filled; 1044 done += filled; 1045 *ppos += filled; 1046 buf += filled; 1047 } 1048 1049 return done; 1050 write_err: 1051 return -EFAULT; 1052 } 1053 1054 static int intel_vgpu_mmap(struct vfio_device *vfio_dev, 1055 struct vm_area_struct *vma) 1056 { 1057 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev); 1058 unsigned int index; 1059 u64 virtaddr; 1060 unsigned long req_size, pgoff, req_start; 1061 pgprot_t pg_prot; 1062 1063 index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT); 1064 if (index >= VFIO_PCI_ROM_REGION_INDEX) 1065 return -EINVAL; 1066 1067 if (vma->vm_end < vma->vm_start) 1068 return -EINVAL; 1069 if ((vma->vm_flags & VM_SHARED) == 0) 1070 return -EINVAL; 1071 if (index != VFIO_PCI_BAR2_REGION_INDEX) 1072 return -EINVAL; 1073 1074 pg_prot = vma->vm_page_prot; 1075 virtaddr = vma->vm_start; 1076 req_size = vma->vm_end - vma->vm_start; 1077 pgoff = vma->vm_pgoff & 1078 ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1); 1079 req_start = pgoff << PAGE_SHIFT; 1080 1081 if (!intel_vgpu_in_aperture(vgpu, req_start)) 1082 return -EINVAL; 1083 if (req_start + req_size > 1084 vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu)) 1085 return -EINVAL; 1086 1087 pgoff = (gvt_aperture_pa_base(vgpu->gvt) >> PAGE_SHIFT) + pgoff; 1088 1089 return remap_pfn_range(vma, virtaddr, pgoff, req_size, pg_prot); 1090 } 1091 1092 static int intel_vgpu_get_irq_count(struct intel_vgpu *vgpu, int type) 1093 { 1094 if (type == VFIO_PCI_INTX_IRQ_INDEX || type == VFIO_PCI_MSI_IRQ_INDEX) 1095 return 1; 1096 1097 return 0; 1098 } 1099 1100 static int intel_vgpu_set_intx_mask(struct intel_vgpu *vgpu, 1101 unsigned int index, unsigned int start, 1102 unsigned int count, u32 flags, 1103 void *data) 1104 { 1105 return 0; 1106 } 1107 1108 static int intel_vgpu_set_intx_unmask(struct intel_vgpu *vgpu, 1109 unsigned int index, unsigned int start, 1110 unsigned int count, u32 flags, void *data) 1111 { 1112 return 0; 1113 } 1114 1115 static int intel_vgpu_set_intx_trigger(struct intel_vgpu *vgpu, 1116 unsigned int index, unsigned int start, unsigned int count, 1117 u32 flags, void *data) 1118 { 1119 return 0; 1120 } 1121 1122 static int intel_vgpu_set_msi_trigger(struct intel_vgpu *vgpu, 1123 unsigned int index, unsigned int start, unsigned int count, 1124 u32 flags, void *data) 1125 { 1126 struct eventfd_ctx *trigger; 1127 1128 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 1129 int fd = *(int *)data; 1130 1131 trigger = eventfd_ctx_fdget(fd); 1132 if (IS_ERR(trigger)) { 1133 gvt_vgpu_err("eventfd_ctx_fdget failed\n"); 1134 return PTR_ERR(trigger); 1135 } 1136 vgpu->msi_trigger = trigger; 1137 } else if ((flags & VFIO_IRQ_SET_DATA_NONE) && !count) 1138 intel_vgpu_release_msi_eventfd_ctx(vgpu); 1139 1140 return 0; 1141 } 1142 1143 static int intel_vgpu_set_irqs(struct intel_vgpu *vgpu, u32 flags, 1144 unsigned int index, unsigned int start, unsigned int count, 1145 void *data) 1146 { 1147 int (*func)(struct intel_vgpu *vgpu, unsigned int index, 1148 unsigned int start, unsigned int count, u32 flags, 1149 void *data) = NULL; 1150 1151 switch (index) { 1152 case VFIO_PCI_INTX_IRQ_INDEX: 1153 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 1154 case VFIO_IRQ_SET_ACTION_MASK: 1155 func = intel_vgpu_set_intx_mask; 1156 break; 1157 case VFIO_IRQ_SET_ACTION_UNMASK: 1158 func = intel_vgpu_set_intx_unmask; 1159 break; 1160 case VFIO_IRQ_SET_ACTION_TRIGGER: 1161 func = intel_vgpu_set_intx_trigger; 1162 break; 1163 } 1164 break; 1165 case VFIO_PCI_MSI_IRQ_INDEX: 1166 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 1167 case VFIO_IRQ_SET_ACTION_MASK: 1168 case VFIO_IRQ_SET_ACTION_UNMASK: 1169 /* XXX Need masking support exported */ 1170 break; 1171 case VFIO_IRQ_SET_ACTION_TRIGGER: 1172 func = intel_vgpu_set_msi_trigger; 1173 break; 1174 } 1175 break; 1176 } 1177 1178 if (!func) 1179 return -ENOTTY; 1180 1181 return func(vgpu, index, start, count, flags, data); 1182 } 1183 1184 static long intel_vgpu_ioctl(struct vfio_device *vfio_dev, unsigned int cmd, 1185 unsigned long arg) 1186 { 1187 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev); 1188 unsigned long minsz; 1189 1190 gvt_dbg_core("vgpu%d ioctl, cmd: %d\n", vgpu->id, cmd); 1191 1192 if (cmd == VFIO_DEVICE_GET_INFO) { 1193 struct vfio_device_info info; 1194 1195 minsz = offsetofend(struct vfio_device_info, num_irqs); 1196 1197 if (copy_from_user(&info, (void __user *)arg, minsz)) 1198 return -EFAULT; 1199 1200 if (info.argsz < minsz) 1201 return -EINVAL; 1202 1203 info.flags = VFIO_DEVICE_FLAGS_PCI; 1204 info.flags |= VFIO_DEVICE_FLAGS_RESET; 1205 info.num_regions = VFIO_PCI_NUM_REGIONS + 1206 vgpu->num_regions; 1207 info.num_irqs = VFIO_PCI_NUM_IRQS; 1208 1209 return copy_to_user((void __user *)arg, &info, minsz) ? 1210 -EFAULT : 0; 1211 1212 } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) { 1213 struct vfio_region_info info; 1214 struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; 1215 unsigned int i; 1216 int ret; 1217 struct vfio_region_info_cap_sparse_mmap *sparse = NULL; 1218 int nr_areas = 1; 1219 int cap_type_id; 1220 1221 minsz = offsetofend(struct vfio_region_info, offset); 1222 1223 if (copy_from_user(&info, (void __user *)arg, minsz)) 1224 return -EFAULT; 1225 1226 if (info.argsz < minsz) 1227 return -EINVAL; 1228 1229 switch (info.index) { 1230 case VFIO_PCI_CONFIG_REGION_INDEX: 1231 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1232 info.size = vgpu->gvt->device_info.cfg_space_size; 1233 info.flags = VFIO_REGION_INFO_FLAG_READ | 1234 VFIO_REGION_INFO_FLAG_WRITE; 1235 break; 1236 case VFIO_PCI_BAR0_REGION_INDEX: 1237 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1238 info.size = vgpu->cfg_space.bar[info.index].size; 1239 if (!info.size) { 1240 info.flags = 0; 1241 break; 1242 } 1243 1244 info.flags = VFIO_REGION_INFO_FLAG_READ | 1245 VFIO_REGION_INFO_FLAG_WRITE; 1246 break; 1247 case VFIO_PCI_BAR1_REGION_INDEX: 1248 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1249 info.size = 0; 1250 info.flags = 0; 1251 break; 1252 case VFIO_PCI_BAR2_REGION_INDEX: 1253 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1254 info.flags = VFIO_REGION_INFO_FLAG_CAPS | 1255 VFIO_REGION_INFO_FLAG_MMAP | 1256 VFIO_REGION_INFO_FLAG_READ | 1257 VFIO_REGION_INFO_FLAG_WRITE; 1258 info.size = gvt_aperture_sz(vgpu->gvt); 1259 1260 sparse = kzalloc(struct_size(sparse, areas, nr_areas), 1261 GFP_KERNEL); 1262 if (!sparse) 1263 return -ENOMEM; 1264 1265 sparse->header.id = VFIO_REGION_INFO_CAP_SPARSE_MMAP; 1266 sparse->header.version = 1; 1267 sparse->nr_areas = nr_areas; 1268 cap_type_id = VFIO_REGION_INFO_CAP_SPARSE_MMAP; 1269 sparse->areas[0].offset = 1270 PAGE_ALIGN(vgpu_aperture_offset(vgpu)); 1271 sparse->areas[0].size = vgpu_aperture_sz(vgpu); 1272 break; 1273 1274 case VFIO_PCI_BAR3_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: 1275 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1276 info.size = 0; 1277 info.flags = 0; 1278 1279 gvt_dbg_core("get region info bar:%d\n", info.index); 1280 break; 1281 1282 case VFIO_PCI_ROM_REGION_INDEX: 1283 case VFIO_PCI_VGA_REGION_INDEX: 1284 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1285 info.size = 0; 1286 info.flags = 0; 1287 1288 gvt_dbg_core("get region info index:%d\n", info.index); 1289 break; 1290 default: 1291 { 1292 struct vfio_region_info_cap_type cap_type = { 1293 .header.id = VFIO_REGION_INFO_CAP_TYPE, 1294 .header.version = 1 }; 1295 1296 if (info.index >= VFIO_PCI_NUM_REGIONS + 1297 vgpu->num_regions) 1298 return -EINVAL; 1299 info.index = 1300 array_index_nospec(info.index, 1301 VFIO_PCI_NUM_REGIONS + 1302 vgpu->num_regions); 1303 1304 i = info.index - VFIO_PCI_NUM_REGIONS; 1305 1306 info.offset = 1307 VFIO_PCI_INDEX_TO_OFFSET(info.index); 1308 info.size = vgpu->region[i].size; 1309 info.flags = vgpu->region[i].flags; 1310 1311 cap_type.type = vgpu->region[i].type; 1312 cap_type.subtype = vgpu->region[i].subtype; 1313 1314 ret = vfio_info_add_capability(&caps, 1315 &cap_type.header, 1316 sizeof(cap_type)); 1317 if (ret) 1318 return ret; 1319 } 1320 } 1321 1322 if ((info.flags & VFIO_REGION_INFO_FLAG_CAPS) && sparse) { 1323 switch (cap_type_id) { 1324 case VFIO_REGION_INFO_CAP_SPARSE_MMAP: 1325 ret = vfio_info_add_capability(&caps, 1326 &sparse->header, 1327 struct_size(sparse, areas, 1328 sparse->nr_areas)); 1329 if (ret) { 1330 kfree(sparse); 1331 return ret; 1332 } 1333 break; 1334 default: 1335 kfree(sparse); 1336 return -EINVAL; 1337 } 1338 } 1339 1340 if (caps.size) { 1341 info.flags |= VFIO_REGION_INFO_FLAG_CAPS; 1342 if (info.argsz < sizeof(info) + caps.size) { 1343 info.argsz = sizeof(info) + caps.size; 1344 info.cap_offset = 0; 1345 } else { 1346 vfio_info_cap_shift(&caps, sizeof(info)); 1347 if (copy_to_user((void __user *)arg + 1348 sizeof(info), caps.buf, 1349 caps.size)) { 1350 kfree(caps.buf); 1351 kfree(sparse); 1352 return -EFAULT; 1353 } 1354 info.cap_offset = sizeof(info); 1355 } 1356 1357 kfree(caps.buf); 1358 } 1359 1360 kfree(sparse); 1361 return copy_to_user((void __user *)arg, &info, minsz) ? 1362 -EFAULT : 0; 1363 } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) { 1364 struct vfio_irq_info info; 1365 1366 minsz = offsetofend(struct vfio_irq_info, count); 1367 1368 if (copy_from_user(&info, (void __user *)arg, minsz)) 1369 return -EFAULT; 1370 1371 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS) 1372 return -EINVAL; 1373 1374 switch (info.index) { 1375 case VFIO_PCI_INTX_IRQ_INDEX: 1376 case VFIO_PCI_MSI_IRQ_INDEX: 1377 break; 1378 default: 1379 return -EINVAL; 1380 } 1381 1382 info.flags = VFIO_IRQ_INFO_EVENTFD; 1383 1384 info.count = intel_vgpu_get_irq_count(vgpu, info.index); 1385 1386 if (info.index == VFIO_PCI_INTX_IRQ_INDEX) 1387 info.flags |= (VFIO_IRQ_INFO_MASKABLE | 1388 VFIO_IRQ_INFO_AUTOMASKED); 1389 else 1390 info.flags |= VFIO_IRQ_INFO_NORESIZE; 1391 1392 return copy_to_user((void __user *)arg, &info, minsz) ? 1393 -EFAULT : 0; 1394 } else if (cmd == VFIO_DEVICE_SET_IRQS) { 1395 struct vfio_irq_set hdr; 1396 u8 *data = NULL; 1397 int ret = 0; 1398 size_t data_size = 0; 1399 1400 minsz = offsetofend(struct vfio_irq_set, count); 1401 1402 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 1403 return -EFAULT; 1404 1405 if (!(hdr.flags & VFIO_IRQ_SET_DATA_NONE)) { 1406 int max = intel_vgpu_get_irq_count(vgpu, hdr.index); 1407 1408 ret = vfio_set_irqs_validate_and_prepare(&hdr, max, 1409 VFIO_PCI_NUM_IRQS, &data_size); 1410 if (ret) { 1411 gvt_vgpu_err("intel:vfio_set_irqs_validate_and_prepare failed\n"); 1412 return -EINVAL; 1413 } 1414 if (data_size) { 1415 data = memdup_user((void __user *)(arg + minsz), 1416 data_size); 1417 if (IS_ERR(data)) 1418 return PTR_ERR(data); 1419 } 1420 } 1421 1422 ret = intel_vgpu_set_irqs(vgpu, hdr.flags, hdr.index, 1423 hdr.start, hdr.count, data); 1424 kfree(data); 1425 1426 return ret; 1427 } else if (cmd == VFIO_DEVICE_RESET) { 1428 intel_gvt_reset_vgpu(vgpu); 1429 return 0; 1430 } else if (cmd == VFIO_DEVICE_QUERY_GFX_PLANE) { 1431 struct vfio_device_gfx_plane_info dmabuf; 1432 int ret = 0; 1433 1434 minsz = offsetofend(struct vfio_device_gfx_plane_info, 1435 dmabuf_id); 1436 if (copy_from_user(&dmabuf, (void __user *)arg, minsz)) 1437 return -EFAULT; 1438 if (dmabuf.argsz < minsz) 1439 return -EINVAL; 1440 1441 ret = intel_vgpu_query_plane(vgpu, &dmabuf); 1442 if (ret != 0) 1443 return ret; 1444 1445 return copy_to_user((void __user *)arg, &dmabuf, minsz) ? 1446 -EFAULT : 0; 1447 } else if (cmd == VFIO_DEVICE_GET_GFX_DMABUF) { 1448 __u32 dmabuf_id; 1449 1450 if (get_user(dmabuf_id, (__u32 __user *)arg)) 1451 return -EFAULT; 1452 return intel_vgpu_get_dmabuf(vgpu, dmabuf_id); 1453 } 1454 1455 return -ENOTTY; 1456 } 1457 1458 static ssize_t 1459 vgpu_id_show(struct device *dev, struct device_attribute *attr, 1460 char *buf) 1461 { 1462 struct intel_vgpu *vgpu = dev_get_drvdata(dev); 1463 1464 return sprintf(buf, "%d\n", vgpu->id); 1465 } 1466 1467 static DEVICE_ATTR_RO(vgpu_id); 1468 1469 static struct attribute *intel_vgpu_attrs[] = { 1470 &dev_attr_vgpu_id.attr, 1471 NULL 1472 }; 1473 1474 static const struct attribute_group intel_vgpu_group = { 1475 .name = "intel_vgpu", 1476 .attrs = intel_vgpu_attrs, 1477 }; 1478 1479 static const struct attribute_group *intel_vgpu_groups[] = { 1480 &intel_vgpu_group, 1481 NULL, 1482 }; 1483 1484 static int intel_vgpu_init_dev(struct vfio_device *vfio_dev) 1485 { 1486 struct mdev_device *mdev = to_mdev_device(vfio_dev->dev); 1487 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev); 1488 struct intel_vgpu_type *type = 1489 container_of(mdev->type, struct intel_vgpu_type, type); 1490 1491 vgpu->gvt = kdev_to_i915(mdev->type->parent->dev)->gvt; 1492 return intel_gvt_create_vgpu(vgpu, type->conf); 1493 } 1494 1495 static void intel_vgpu_release_dev(struct vfio_device *vfio_dev) 1496 { 1497 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev); 1498 1499 intel_gvt_destroy_vgpu(vgpu); 1500 vfio_free_device(vfio_dev); 1501 } 1502 1503 static const struct vfio_device_ops intel_vgpu_dev_ops = { 1504 .init = intel_vgpu_init_dev, 1505 .release = intel_vgpu_release_dev, 1506 .open_device = intel_vgpu_open_device, 1507 .close_device = intel_vgpu_close_device, 1508 .read = intel_vgpu_read, 1509 .write = intel_vgpu_write, 1510 .mmap = intel_vgpu_mmap, 1511 .ioctl = intel_vgpu_ioctl, 1512 .dma_unmap = intel_vgpu_dma_unmap, 1513 }; 1514 1515 static int intel_vgpu_probe(struct mdev_device *mdev) 1516 { 1517 struct intel_vgpu *vgpu; 1518 int ret; 1519 1520 vgpu = vfio_alloc_device(intel_vgpu, vfio_device, &mdev->dev, 1521 &intel_vgpu_dev_ops); 1522 if (IS_ERR(vgpu)) { 1523 gvt_err("failed to create intel vgpu: %ld\n", PTR_ERR(vgpu)); 1524 return PTR_ERR(vgpu); 1525 } 1526 1527 dev_set_drvdata(&mdev->dev, vgpu); 1528 ret = vfio_register_emulated_iommu_dev(&vgpu->vfio_device); 1529 if (ret) 1530 goto out_put_vdev; 1531 1532 gvt_dbg_core("intel_vgpu_create succeeded for mdev: %s\n", 1533 dev_name(mdev_dev(mdev))); 1534 return 0; 1535 1536 out_put_vdev: 1537 vfio_put_device(&vgpu->vfio_device); 1538 return ret; 1539 } 1540 1541 static void intel_vgpu_remove(struct mdev_device *mdev) 1542 { 1543 struct intel_vgpu *vgpu = dev_get_drvdata(&mdev->dev); 1544 1545 if (WARN_ON_ONCE(vgpu->attached)) 1546 return; 1547 1548 vfio_unregister_group_dev(&vgpu->vfio_device); 1549 vfio_put_device(&vgpu->vfio_device); 1550 } 1551 1552 static struct mdev_driver intel_vgpu_mdev_driver = { 1553 .driver = { 1554 .name = "intel_vgpu_mdev", 1555 .owner = THIS_MODULE, 1556 .dev_groups = intel_vgpu_groups, 1557 }, 1558 .probe = intel_vgpu_probe, 1559 .remove = intel_vgpu_remove, 1560 .types_attrs = gvt_type_attrs, 1561 }; 1562 1563 int intel_gvt_page_track_add(struct intel_vgpu *info, u64 gfn) 1564 { 1565 struct kvm *kvm = info->vfio_device.kvm; 1566 struct kvm_memory_slot *slot; 1567 int idx; 1568 1569 if (!info->attached) 1570 return -ESRCH; 1571 1572 idx = srcu_read_lock(&kvm->srcu); 1573 slot = gfn_to_memslot(kvm, gfn); 1574 if (!slot) { 1575 srcu_read_unlock(&kvm->srcu, idx); 1576 return -EINVAL; 1577 } 1578 1579 write_lock(&kvm->mmu_lock); 1580 1581 if (kvmgt_gfn_is_write_protected(info, gfn)) 1582 goto out; 1583 1584 kvm_slot_page_track_add_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE); 1585 kvmgt_protect_table_add(info, gfn); 1586 1587 out: 1588 write_unlock(&kvm->mmu_lock); 1589 srcu_read_unlock(&kvm->srcu, idx); 1590 return 0; 1591 } 1592 1593 int intel_gvt_page_track_remove(struct intel_vgpu *info, u64 gfn) 1594 { 1595 struct kvm *kvm = info->vfio_device.kvm; 1596 struct kvm_memory_slot *slot; 1597 int idx; 1598 1599 if (!info->attached) 1600 return 0; 1601 1602 idx = srcu_read_lock(&kvm->srcu); 1603 slot = gfn_to_memslot(kvm, gfn); 1604 if (!slot) { 1605 srcu_read_unlock(&kvm->srcu, idx); 1606 return -EINVAL; 1607 } 1608 1609 write_lock(&kvm->mmu_lock); 1610 1611 if (!kvmgt_gfn_is_write_protected(info, gfn)) 1612 goto out; 1613 1614 kvm_slot_page_track_remove_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE); 1615 kvmgt_protect_table_del(info, gfn); 1616 1617 out: 1618 write_unlock(&kvm->mmu_lock); 1619 srcu_read_unlock(&kvm->srcu, idx); 1620 return 0; 1621 } 1622 1623 static void kvmgt_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, 1624 const u8 *val, int len, 1625 struct kvm_page_track_notifier_node *node) 1626 { 1627 struct intel_vgpu *info = 1628 container_of(node, struct intel_vgpu, track_node); 1629 1630 if (kvmgt_gfn_is_write_protected(info, gpa_to_gfn(gpa))) 1631 intel_vgpu_page_track_handler(info, gpa, 1632 (void *)val, len); 1633 } 1634 1635 static void kvmgt_page_track_flush_slot(struct kvm *kvm, 1636 struct kvm_memory_slot *slot, 1637 struct kvm_page_track_notifier_node *node) 1638 { 1639 int i; 1640 gfn_t gfn; 1641 struct intel_vgpu *info = 1642 container_of(node, struct intel_vgpu, track_node); 1643 1644 write_lock(&kvm->mmu_lock); 1645 for (i = 0; i < slot->npages; i++) { 1646 gfn = slot->base_gfn + i; 1647 if (kvmgt_gfn_is_write_protected(info, gfn)) { 1648 kvm_slot_page_track_remove_page(kvm, slot, gfn, 1649 KVM_PAGE_TRACK_WRITE); 1650 kvmgt_protect_table_del(info, gfn); 1651 } 1652 } 1653 write_unlock(&kvm->mmu_lock); 1654 } 1655 1656 void intel_vgpu_detach_regions(struct intel_vgpu *vgpu) 1657 { 1658 int i; 1659 1660 if (!vgpu->region) 1661 return; 1662 1663 for (i = 0; i < vgpu->num_regions; i++) 1664 if (vgpu->region[i].ops->release) 1665 vgpu->region[i].ops->release(vgpu, 1666 &vgpu->region[i]); 1667 vgpu->num_regions = 0; 1668 kfree(vgpu->region); 1669 vgpu->region = NULL; 1670 } 1671 1672 int intel_gvt_dma_map_guest_page(struct intel_vgpu *vgpu, unsigned long gfn, 1673 unsigned long size, dma_addr_t *dma_addr) 1674 { 1675 struct gvt_dma *entry; 1676 int ret; 1677 1678 if (!vgpu->attached) 1679 return -EINVAL; 1680 1681 mutex_lock(&vgpu->cache_lock); 1682 1683 entry = __gvt_cache_find_gfn(vgpu, gfn); 1684 if (!entry) { 1685 ret = gvt_dma_map_page(vgpu, gfn, dma_addr, size); 1686 if (ret) 1687 goto err_unlock; 1688 1689 ret = __gvt_cache_add(vgpu, gfn, *dma_addr, size); 1690 if (ret) 1691 goto err_unmap; 1692 } else if (entry->size != size) { 1693 /* the same gfn with different size: unmap and re-map */ 1694 gvt_dma_unmap_page(vgpu, gfn, entry->dma_addr, entry->size); 1695 __gvt_cache_remove_entry(vgpu, entry); 1696 1697 ret = gvt_dma_map_page(vgpu, gfn, dma_addr, size); 1698 if (ret) 1699 goto err_unlock; 1700 1701 ret = __gvt_cache_add(vgpu, gfn, *dma_addr, size); 1702 if (ret) 1703 goto err_unmap; 1704 } else { 1705 kref_get(&entry->ref); 1706 *dma_addr = entry->dma_addr; 1707 } 1708 1709 mutex_unlock(&vgpu->cache_lock); 1710 return 0; 1711 1712 err_unmap: 1713 gvt_dma_unmap_page(vgpu, gfn, *dma_addr, size); 1714 err_unlock: 1715 mutex_unlock(&vgpu->cache_lock); 1716 return ret; 1717 } 1718 1719 int intel_gvt_dma_pin_guest_page(struct intel_vgpu *vgpu, dma_addr_t dma_addr) 1720 { 1721 struct gvt_dma *entry; 1722 int ret = 0; 1723 1724 if (!vgpu->attached) 1725 return -ENODEV; 1726 1727 mutex_lock(&vgpu->cache_lock); 1728 entry = __gvt_cache_find_dma_addr(vgpu, dma_addr); 1729 if (entry) 1730 kref_get(&entry->ref); 1731 else 1732 ret = -ENOMEM; 1733 mutex_unlock(&vgpu->cache_lock); 1734 1735 return ret; 1736 } 1737 1738 static void __gvt_dma_release(struct kref *ref) 1739 { 1740 struct gvt_dma *entry = container_of(ref, typeof(*entry), ref); 1741 1742 gvt_dma_unmap_page(entry->vgpu, entry->gfn, entry->dma_addr, 1743 entry->size); 1744 __gvt_cache_remove_entry(entry->vgpu, entry); 1745 } 1746 1747 void intel_gvt_dma_unmap_guest_page(struct intel_vgpu *vgpu, 1748 dma_addr_t dma_addr) 1749 { 1750 struct gvt_dma *entry; 1751 1752 if (!vgpu->attached) 1753 return; 1754 1755 mutex_lock(&vgpu->cache_lock); 1756 entry = __gvt_cache_find_dma_addr(vgpu, dma_addr); 1757 if (entry) 1758 kref_put(&entry->ref, __gvt_dma_release); 1759 mutex_unlock(&vgpu->cache_lock); 1760 } 1761 1762 static void init_device_info(struct intel_gvt *gvt) 1763 { 1764 struct intel_gvt_device_info *info = &gvt->device_info; 1765 struct pci_dev *pdev = to_pci_dev(gvt->gt->i915->drm.dev); 1766 1767 info->max_support_vgpus = 8; 1768 info->cfg_space_size = PCI_CFG_SPACE_EXP_SIZE; 1769 info->mmio_size = 2 * 1024 * 1024; 1770 info->mmio_bar = 0; 1771 info->gtt_start_offset = 8 * 1024 * 1024; 1772 info->gtt_entry_size = 8; 1773 info->gtt_entry_size_shift = 3; 1774 info->gmadr_bytes_in_cmd = 8; 1775 info->max_surface_size = 36 * 1024 * 1024; 1776 info->msi_cap_offset = pdev->msi_cap; 1777 } 1778 1779 static void intel_gvt_test_and_emulate_vblank(struct intel_gvt *gvt) 1780 { 1781 struct intel_vgpu *vgpu; 1782 int id; 1783 1784 mutex_lock(&gvt->lock); 1785 idr_for_each_entry((&(gvt)->vgpu_idr), (vgpu), (id)) { 1786 if (test_and_clear_bit(INTEL_GVT_REQUEST_EMULATE_VBLANK + id, 1787 (void *)&gvt->service_request)) { 1788 if (vgpu->active) 1789 intel_vgpu_emulate_vblank(vgpu); 1790 } 1791 } 1792 mutex_unlock(&gvt->lock); 1793 } 1794 1795 static int gvt_service_thread(void *data) 1796 { 1797 struct intel_gvt *gvt = (struct intel_gvt *)data; 1798 int ret; 1799 1800 gvt_dbg_core("service thread start\n"); 1801 1802 while (!kthread_should_stop()) { 1803 ret = wait_event_interruptible(gvt->service_thread_wq, 1804 kthread_should_stop() || gvt->service_request); 1805 1806 if (kthread_should_stop()) 1807 break; 1808 1809 if (WARN_ONCE(ret, "service thread is waken up by signal.\n")) 1810 continue; 1811 1812 intel_gvt_test_and_emulate_vblank(gvt); 1813 1814 if (test_bit(INTEL_GVT_REQUEST_SCHED, 1815 (void *)&gvt->service_request) || 1816 test_bit(INTEL_GVT_REQUEST_EVENT_SCHED, 1817 (void *)&gvt->service_request)) { 1818 intel_gvt_schedule(gvt); 1819 } 1820 } 1821 1822 return 0; 1823 } 1824 1825 static void clean_service_thread(struct intel_gvt *gvt) 1826 { 1827 kthread_stop(gvt->service_thread); 1828 } 1829 1830 static int init_service_thread(struct intel_gvt *gvt) 1831 { 1832 init_waitqueue_head(&gvt->service_thread_wq); 1833 1834 gvt->service_thread = kthread_run(gvt_service_thread, 1835 gvt, "gvt_service_thread"); 1836 if (IS_ERR(gvt->service_thread)) { 1837 gvt_err("fail to start service thread.\n"); 1838 return PTR_ERR(gvt->service_thread); 1839 } 1840 return 0; 1841 } 1842 1843 /** 1844 * intel_gvt_clean_device - clean a GVT device 1845 * @i915: i915 private 1846 * 1847 * This function is called at the driver unloading stage, to free the 1848 * resources owned by a GVT device. 1849 * 1850 */ 1851 static void intel_gvt_clean_device(struct drm_i915_private *i915) 1852 { 1853 struct intel_gvt *gvt = fetch_and_zero(&i915->gvt); 1854 1855 if (drm_WARN_ON(&i915->drm, !gvt)) 1856 return; 1857 1858 mdev_unregister_parent(&gvt->parent); 1859 intel_gvt_destroy_idle_vgpu(gvt->idle_vgpu); 1860 intel_gvt_clean_vgpu_types(gvt); 1861 1862 intel_gvt_debugfs_clean(gvt); 1863 clean_service_thread(gvt); 1864 intel_gvt_clean_cmd_parser(gvt); 1865 intel_gvt_clean_sched_policy(gvt); 1866 intel_gvt_clean_workload_scheduler(gvt); 1867 intel_gvt_clean_gtt(gvt); 1868 intel_gvt_free_firmware(gvt); 1869 intel_gvt_clean_mmio_info(gvt); 1870 idr_destroy(&gvt->vgpu_idr); 1871 1872 kfree(i915->gvt); 1873 } 1874 1875 /** 1876 * intel_gvt_init_device - initialize a GVT device 1877 * @i915: drm i915 private data 1878 * 1879 * This function is called at the initialization stage, to initialize 1880 * necessary GVT components. 1881 * 1882 * Returns: 1883 * Zero on success, negative error code if failed. 1884 * 1885 */ 1886 static int intel_gvt_init_device(struct drm_i915_private *i915) 1887 { 1888 struct intel_gvt *gvt; 1889 struct intel_vgpu *vgpu; 1890 int ret; 1891 1892 if (drm_WARN_ON(&i915->drm, i915->gvt)) 1893 return -EEXIST; 1894 1895 gvt = kzalloc(sizeof(struct intel_gvt), GFP_KERNEL); 1896 if (!gvt) 1897 return -ENOMEM; 1898 1899 gvt_dbg_core("init gvt device\n"); 1900 1901 idr_init_base(&gvt->vgpu_idr, 1); 1902 spin_lock_init(&gvt->scheduler.mmio_context_lock); 1903 mutex_init(&gvt->lock); 1904 mutex_init(&gvt->sched_lock); 1905 gvt->gt = to_gt(i915); 1906 i915->gvt = gvt; 1907 1908 init_device_info(gvt); 1909 1910 ret = intel_gvt_setup_mmio_info(gvt); 1911 if (ret) 1912 goto out_clean_idr; 1913 1914 intel_gvt_init_engine_mmio_context(gvt); 1915 1916 ret = intel_gvt_load_firmware(gvt); 1917 if (ret) 1918 goto out_clean_mmio_info; 1919 1920 ret = intel_gvt_init_irq(gvt); 1921 if (ret) 1922 goto out_free_firmware; 1923 1924 ret = intel_gvt_init_gtt(gvt); 1925 if (ret) 1926 goto out_free_firmware; 1927 1928 ret = intel_gvt_init_workload_scheduler(gvt); 1929 if (ret) 1930 goto out_clean_gtt; 1931 1932 ret = intel_gvt_init_sched_policy(gvt); 1933 if (ret) 1934 goto out_clean_workload_scheduler; 1935 1936 ret = intel_gvt_init_cmd_parser(gvt); 1937 if (ret) 1938 goto out_clean_sched_policy; 1939 1940 ret = init_service_thread(gvt); 1941 if (ret) 1942 goto out_clean_cmd_parser; 1943 1944 ret = intel_gvt_init_vgpu_types(gvt); 1945 if (ret) 1946 goto out_clean_thread; 1947 1948 vgpu = intel_gvt_create_idle_vgpu(gvt); 1949 if (IS_ERR(vgpu)) { 1950 ret = PTR_ERR(vgpu); 1951 gvt_err("failed to create idle vgpu\n"); 1952 goto out_clean_types; 1953 } 1954 gvt->idle_vgpu = vgpu; 1955 1956 intel_gvt_debugfs_init(gvt); 1957 1958 ret = mdev_register_parent(&gvt->parent, i915->drm.dev, 1959 &intel_vgpu_mdev_driver, 1960 gvt->mdev_types, gvt->num_types); 1961 if (ret) 1962 goto out_destroy_idle_vgpu; 1963 1964 gvt_dbg_core("gvt device initialization is done\n"); 1965 return 0; 1966 1967 out_destroy_idle_vgpu: 1968 intel_gvt_destroy_idle_vgpu(gvt->idle_vgpu); 1969 intel_gvt_debugfs_clean(gvt); 1970 out_clean_types: 1971 intel_gvt_clean_vgpu_types(gvt); 1972 out_clean_thread: 1973 clean_service_thread(gvt); 1974 out_clean_cmd_parser: 1975 intel_gvt_clean_cmd_parser(gvt); 1976 out_clean_sched_policy: 1977 intel_gvt_clean_sched_policy(gvt); 1978 out_clean_workload_scheduler: 1979 intel_gvt_clean_workload_scheduler(gvt); 1980 out_clean_gtt: 1981 intel_gvt_clean_gtt(gvt); 1982 out_free_firmware: 1983 intel_gvt_free_firmware(gvt); 1984 out_clean_mmio_info: 1985 intel_gvt_clean_mmio_info(gvt); 1986 out_clean_idr: 1987 idr_destroy(&gvt->vgpu_idr); 1988 kfree(gvt); 1989 i915->gvt = NULL; 1990 return ret; 1991 } 1992 1993 static void intel_gvt_pm_resume(struct drm_i915_private *i915) 1994 { 1995 struct intel_gvt *gvt = i915->gvt; 1996 1997 intel_gvt_restore_fence(gvt); 1998 intel_gvt_restore_mmio(gvt); 1999 intel_gvt_restore_ggtt(gvt); 2000 } 2001 2002 static const struct intel_vgpu_ops intel_gvt_vgpu_ops = { 2003 .init_device = intel_gvt_init_device, 2004 .clean_device = intel_gvt_clean_device, 2005 .pm_resume = intel_gvt_pm_resume, 2006 }; 2007 2008 static int __init kvmgt_init(void) 2009 { 2010 int ret; 2011 2012 ret = intel_gvt_set_ops(&intel_gvt_vgpu_ops); 2013 if (ret) 2014 return ret; 2015 2016 ret = mdev_register_driver(&intel_vgpu_mdev_driver); 2017 if (ret) 2018 intel_gvt_clear_ops(&intel_gvt_vgpu_ops); 2019 return ret; 2020 } 2021 2022 static void __exit kvmgt_exit(void) 2023 { 2024 mdev_unregister_driver(&intel_vgpu_mdev_driver); 2025 intel_gvt_clear_ops(&intel_gvt_vgpu_ops); 2026 } 2027 2028 module_init(kvmgt_init); 2029 module_exit(kvmgt_exit); 2030 2031 MODULE_LICENSE("GPL and additional rights"); 2032 MODULE_AUTHOR("Intel Corporation"); 2033