1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Virtio PCI driver - modern (virtio 1.0) device support 4 * 5 * This module allows virtio devices to be used over a virtual PCI device. 6 * This can be used with QEMU based VMMs like KVM or Xen. 7 * 8 * Copyright IBM Corp. 2007 9 * Copyright Red Hat, Inc. 2014 10 * 11 * Authors: 12 * Anthony Liguori <aliguori@us.ibm.com> 13 * Rusty Russell <rusty@rustcorp.com.au> 14 * Michael S. Tsirkin <mst@redhat.com> 15 */ 16 17 #include <linux/delay.h> 18 #include <linux/virtio_pci_admin.h> 19 #define VIRTIO_PCI_NO_LEGACY 20 #define VIRTIO_RING_NO_LEGACY 21 #include "virtio_pci_common.h" 22 23 #define VIRTIO_AVQ_SGS_MAX 4 24 25 static void vp_get_features(struct virtio_device *vdev, u64 *features) 26 { 27 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 28 29 vp_modern_get_extended_features(&vp_dev->mdev, features); 30 } 31 32 static int vp_avq_index(struct virtio_device *vdev, u16 *index, u16 *num) 33 { 34 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 35 36 *num = 0; 37 if (!virtio_has_feature(vdev, VIRTIO_F_ADMIN_VQ)) 38 return 0; 39 40 *num = vp_modern_avq_num(&vp_dev->mdev); 41 if (!(*num)) 42 return -EINVAL; 43 *index = vp_modern_avq_index(&vp_dev->mdev); 44 return 0; 45 } 46 47 void vp_modern_avq_done(struct virtqueue *vq) 48 { 49 struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); 50 struct virtio_pci_admin_vq *admin_vq = &vp_dev->admin_vq; 51 unsigned int status_size = sizeof(struct virtio_admin_cmd_status); 52 struct virtio_admin_cmd *cmd; 53 unsigned long flags; 54 unsigned int len; 55 56 spin_lock_irqsave(&admin_vq->lock, flags); 57 do { 58 virtqueue_disable_cb(vq); 59 while ((cmd = virtqueue_get_buf(vq, &len))) { 60 /* If the number of bytes written by the device is less 61 * than the size of struct virtio_admin_cmd_status, the 62 * remaining status bytes will remain zero-initialized, 63 * since the buffer was zeroed during allocation. 64 * In this case, set the size of command_specific_result 65 * to 0. 66 */ 67 if (len < status_size) 68 cmd->result_sg_size = 0; 69 else 70 cmd->result_sg_size = len - status_size; 71 complete(&cmd->completion); 72 } 73 } while (!virtqueue_enable_cb(vq)); 74 spin_unlock_irqrestore(&admin_vq->lock, flags); 75 } 76 77 static int virtqueue_exec_admin_cmd(struct virtio_pci_admin_vq *admin_vq, 78 u16 opcode, 79 struct scatterlist **sgs, 80 unsigned int out_num, 81 unsigned int in_num, 82 struct virtio_admin_cmd *cmd) 83 { 84 struct virtqueue *vq; 85 unsigned long flags; 86 int ret; 87 88 vq = admin_vq->info->vq; 89 if (!vq) 90 return -EIO; 91 92 if (opcode != VIRTIO_ADMIN_CMD_LIST_QUERY && 93 opcode != VIRTIO_ADMIN_CMD_LIST_USE && 94 !((1ULL << opcode) & admin_vq->supported_cmds)) 95 return -EOPNOTSUPP; 96 97 init_completion(&cmd->completion); 98 99 again: 100 if (virtqueue_is_broken(vq)) 101 return -EIO; 102 103 spin_lock_irqsave(&admin_vq->lock, flags); 104 ret = virtqueue_add_sgs(vq, sgs, out_num, in_num, cmd, GFP_KERNEL); 105 if (ret < 0) { 106 if (ret == -ENOSPC) { 107 spin_unlock_irqrestore(&admin_vq->lock, flags); 108 cpu_relax(); 109 goto again; 110 } 111 goto unlock_err; 112 } 113 if (!virtqueue_kick(vq)) 114 goto unlock_err; 115 spin_unlock_irqrestore(&admin_vq->lock, flags); 116 117 wait_for_completion(&cmd->completion); 118 119 return cmd->ret; 120 121 unlock_err: 122 spin_unlock_irqrestore(&admin_vq->lock, flags); 123 return -EIO; 124 } 125 126 int vp_modern_admin_cmd_exec(struct virtio_device *vdev, 127 struct virtio_admin_cmd *cmd) 128 { 129 struct scatterlist *sgs[VIRTIO_AVQ_SGS_MAX], hdr, stat; 130 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 131 struct virtio_admin_cmd_status *va_status; 132 unsigned int out_num = 0, in_num = 0; 133 struct virtio_admin_cmd_hdr *va_hdr; 134 u16 status; 135 int ret; 136 137 if (!virtio_has_feature(vdev, VIRTIO_F_ADMIN_VQ)) 138 return -EOPNOTSUPP; 139 140 va_status = kzalloc_obj(*va_status); 141 if (!va_status) 142 return -ENOMEM; 143 144 va_hdr = kzalloc_obj(*va_hdr); 145 if (!va_hdr) { 146 ret = -ENOMEM; 147 goto err_alloc; 148 } 149 150 va_hdr->opcode = cmd->opcode; 151 va_hdr->group_type = cmd->group_type; 152 va_hdr->group_member_id = cmd->group_member_id; 153 154 /* Add header */ 155 sg_init_one(&hdr, va_hdr, sizeof(*va_hdr)); 156 sgs[out_num] = &hdr; 157 out_num++; 158 159 if (cmd->data_sg) { 160 sgs[out_num] = cmd->data_sg; 161 out_num++; 162 } 163 164 /* Add return status */ 165 sg_init_one(&stat, va_status, sizeof(*va_status)); 166 sgs[out_num + in_num] = &stat; 167 in_num++; 168 169 if (cmd->result_sg) { 170 sgs[out_num + in_num] = cmd->result_sg; 171 in_num++; 172 } 173 174 ret = virtqueue_exec_admin_cmd(&vp_dev->admin_vq, 175 le16_to_cpu(cmd->opcode), 176 sgs, out_num, in_num, cmd); 177 if (ret) { 178 dev_err(&vdev->dev, 179 "Failed to execute command on admin vq: %d\n.", ret); 180 goto err_cmd_exec; 181 } 182 183 status = le16_to_cpu(va_status->status); 184 if (status != VIRTIO_ADMIN_STATUS_OK) { 185 dev_err(&vdev->dev, 186 "admin command error: status(%#x) qualifier(%#x)\n", 187 status, le16_to_cpu(va_status->status_qualifier)); 188 ret = -status; 189 } 190 191 err_cmd_exec: 192 kfree(va_hdr); 193 err_alloc: 194 kfree(va_status); 195 return ret; 196 } 197 198 static void virtio_pci_admin_cmd_list_init(struct virtio_device *virtio_dev) 199 { 200 struct virtio_pci_device *vp_dev = to_vp_device(virtio_dev); 201 struct virtio_admin_cmd cmd = {}; 202 struct scatterlist result_sg; 203 struct scatterlist data_sg; 204 __le64 *data; 205 int ret; 206 207 data = kzalloc_obj(*data); 208 if (!data) 209 return; 210 211 sg_init_one(&result_sg, data, sizeof(*data)); 212 cmd.opcode = cpu_to_le16(VIRTIO_ADMIN_CMD_LIST_QUERY); 213 cmd.group_type = cpu_to_le16(VIRTIO_ADMIN_GROUP_TYPE_SRIOV); 214 cmd.result_sg = &result_sg; 215 216 ret = vp_modern_admin_cmd_exec(virtio_dev, &cmd); 217 if (ret) 218 goto end; 219 220 *data &= cpu_to_le64(VIRTIO_ADMIN_CMD_BITMAP); 221 sg_init_one(&data_sg, data, sizeof(*data)); 222 cmd.opcode = cpu_to_le16(VIRTIO_ADMIN_CMD_LIST_USE); 223 cmd.data_sg = &data_sg; 224 cmd.result_sg = NULL; 225 226 ret = vp_modern_admin_cmd_exec(virtio_dev, &cmd); 227 if (ret) 228 goto end; 229 230 vp_dev->admin_vq.supported_cmds = le64_to_cpu(*data); 231 end: 232 kfree(data); 233 } 234 235 static void 236 virtio_pci_admin_cmd_dev_parts_objects_enable(struct virtio_device *virtio_dev) 237 { 238 struct virtio_pci_device *vp_dev = to_vp_device(virtio_dev); 239 struct virtio_admin_cmd_cap_get_data *get_data; 240 struct virtio_admin_cmd_cap_set_data *set_data; 241 struct virtio_dev_parts_cap *result; 242 struct virtio_admin_cmd cmd = {}; 243 struct scatterlist result_sg; 244 struct scatterlist data_sg; 245 u8 resource_objects_limit; 246 u16 set_data_size; 247 int ret; 248 249 get_data = kzalloc_obj(*get_data); 250 if (!get_data) 251 return; 252 253 result = kzalloc_obj(*result); 254 if (!result) 255 goto end; 256 257 get_data->id = cpu_to_le16(VIRTIO_DEV_PARTS_CAP); 258 sg_init_one(&data_sg, get_data, sizeof(*get_data)); 259 sg_init_one(&result_sg, result, sizeof(*result)); 260 cmd.opcode = cpu_to_le16(VIRTIO_ADMIN_CMD_DEVICE_CAP_GET); 261 cmd.group_type = cpu_to_le16(VIRTIO_ADMIN_GROUP_TYPE_SELF); 262 cmd.data_sg = &data_sg; 263 cmd.result_sg = &result_sg; 264 ret = vp_modern_admin_cmd_exec(virtio_dev, &cmd); 265 if (ret) 266 goto err_get; 267 268 set_data_size = sizeof(*set_data) + sizeof(*result); 269 set_data = kzalloc(set_data_size, GFP_KERNEL); 270 if (!set_data) 271 goto err_get; 272 273 set_data->id = cpu_to_le16(VIRTIO_DEV_PARTS_CAP); 274 275 /* Set the limit to the minimum value between the GET and SET values 276 * supported by the device. Since the obj_id for VIRTIO_DEV_PARTS_CAP 277 * is a globally unique value per PF, there is no possibility of 278 * overlap between GET and SET operations. 279 */ 280 resource_objects_limit = min(result->get_parts_resource_objects_limit, 281 result->set_parts_resource_objects_limit); 282 result->get_parts_resource_objects_limit = resource_objects_limit; 283 result->set_parts_resource_objects_limit = resource_objects_limit; 284 memcpy(set_data->cap_specific_data, result, sizeof(*result)); 285 sg_init_one(&data_sg, set_data, set_data_size); 286 cmd.data_sg = &data_sg; 287 cmd.result_sg = NULL; 288 cmd.opcode = cpu_to_le16(VIRTIO_ADMIN_CMD_DRIVER_CAP_SET); 289 ret = vp_modern_admin_cmd_exec(virtio_dev, &cmd); 290 if (ret) 291 goto err_set; 292 293 /* Allocate IDR to manage the dev caps objects */ 294 ida_init(&vp_dev->admin_vq.dev_parts_ida); 295 vp_dev->admin_vq.max_dev_parts_objects = resource_objects_limit; 296 297 err_set: 298 kfree(set_data); 299 err_get: 300 kfree(result); 301 end: 302 kfree(get_data); 303 } 304 305 static void virtio_pci_admin_cmd_cap_init(struct virtio_device *virtio_dev) 306 { 307 struct virtio_pci_device *vp_dev = to_vp_device(virtio_dev); 308 struct virtio_admin_cmd_query_cap_id_result *data; 309 struct virtio_admin_cmd cmd = {}; 310 struct scatterlist result_sg; 311 int ret; 312 313 data = kzalloc_obj(*data); 314 if (!data) 315 return; 316 317 sg_init_one(&result_sg, data, sizeof(*data)); 318 cmd.opcode = cpu_to_le16(VIRTIO_ADMIN_CMD_CAP_ID_LIST_QUERY); 319 cmd.group_type = cpu_to_le16(VIRTIO_ADMIN_GROUP_TYPE_SELF); 320 cmd.result_sg = &result_sg; 321 322 ret = vp_modern_admin_cmd_exec(virtio_dev, &cmd); 323 if (ret) 324 goto end; 325 326 /* Max number of caps fits into a single u64 */ 327 BUILD_BUG_ON(sizeof(data->supported_caps) > sizeof(u64)); 328 329 vp_dev->admin_vq.supported_caps = le64_to_cpu(data->supported_caps[0]); 330 331 if (!(vp_dev->admin_vq.supported_caps & (1 << VIRTIO_DEV_PARTS_CAP))) 332 goto end; 333 334 virtio_pci_admin_cmd_dev_parts_objects_enable(virtio_dev); 335 end: 336 kfree(data); 337 } 338 339 static void vp_modern_avq_activate(struct virtio_device *vdev) 340 { 341 if (!virtio_has_feature(vdev, VIRTIO_F_ADMIN_VQ)) 342 return; 343 344 virtio_pci_admin_cmd_list_init(vdev); 345 virtio_pci_admin_cmd_cap_init(vdev); 346 } 347 348 static void vp_modern_avq_cleanup(struct virtio_device *vdev) 349 { 350 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 351 struct virtio_admin_cmd *cmd; 352 struct virtqueue *vq; 353 354 if (!virtio_has_feature(vdev, VIRTIO_F_ADMIN_VQ)) 355 return; 356 357 vq = vp_dev->admin_vq.info->vq; 358 if (!vq) 359 return; 360 361 while ((cmd = virtqueue_detach_unused_buf(vq))) { 362 cmd->ret = -EIO; 363 complete(&cmd->completion); 364 } 365 } 366 367 static void vp_modern_del_vqs(struct virtio_device *vdev) 368 { 369 vp_modern_avq_cleanup(vdev); 370 vp_del_vqs(vdev); 371 } 372 373 static void vp_transport_features(struct virtio_device *vdev, u64 features) 374 { 375 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 376 struct pci_dev *pci_dev = vp_dev->pci_dev; 377 378 if ((features & BIT_ULL(VIRTIO_F_SR_IOV)) && 379 pci_find_ext_capability(pci_dev, PCI_EXT_CAP_ID_SRIOV)) 380 __virtio_set_bit(vdev, VIRTIO_F_SR_IOV); 381 382 if (features & BIT_ULL(VIRTIO_F_RING_RESET)) 383 __virtio_set_bit(vdev, VIRTIO_F_RING_RESET); 384 385 if (features & BIT_ULL(VIRTIO_F_ADMIN_VQ)) 386 __virtio_set_bit(vdev, VIRTIO_F_ADMIN_VQ); 387 } 388 389 static int __vp_check_common_size_one_feature(struct virtio_device *vdev, u32 fbit, 390 u32 offset, const char *fname) 391 { 392 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 393 394 if (!__virtio_test_bit(vdev, fbit)) 395 return 0; 396 397 if (likely(vp_dev->mdev.common_len >= offset)) 398 return 0; 399 400 dev_err(&vdev->dev, 401 "virtio: common cfg size(%zu) does not match the feature %s\n", 402 vp_dev->mdev.common_len, fname); 403 404 return -EINVAL; 405 } 406 407 #define vp_check_common_size_one_feature(vdev, fbit, field) \ 408 __vp_check_common_size_one_feature(vdev, fbit, \ 409 offsetofend(struct virtio_pci_modern_common_cfg, field), #fbit) 410 411 static int vp_check_common_size(struct virtio_device *vdev) 412 { 413 if (vp_check_common_size_one_feature(vdev, VIRTIO_F_NOTIF_CONFIG_DATA, queue_notify_data)) 414 return -EINVAL; 415 416 if (vp_check_common_size_one_feature(vdev, VIRTIO_F_RING_RESET, queue_reset)) 417 return -EINVAL; 418 419 if (vp_check_common_size_one_feature(vdev, VIRTIO_F_ADMIN_VQ, admin_queue_num)) 420 return -EINVAL; 421 422 return 0; 423 } 424 425 /* virtio config->finalize_features() implementation */ 426 static int vp_finalize_features(struct virtio_device *vdev) 427 { 428 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 429 u64 features = vdev->features; 430 431 /* Give virtio_ring a chance to accept features. */ 432 vring_transport_features(vdev); 433 434 /* Give virtio_pci a chance to accept features. */ 435 vp_transport_features(vdev, features); 436 437 if (!__virtio_test_bit(vdev, VIRTIO_F_VERSION_1)) { 438 dev_err(&vdev->dev, "virtio: device uses modern interface " 439 "but does not have VIRTIO_F_VERSION_1\n"); 440 return -EINVAL; 441 } 442 443 if (vp_check_common_size(vdev)) 444 return -EINVAL; 445 446 vp_modern_set_extended_features(&vp_dev->mdev, vdev->features_array); 447 448 return 0; 449 } 450 451 /* virtio config->get() implementation */ 452 static void vp_get(struct virtio_device *vdev, unsigned int offset, 453 void *buf, unsigned int len) 454 { 455 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 456 struct virtio_pci_modern_device *mdev = &vp_dev->mdev; 457 void __iomem *device = mdev->device; 458 u8 b; 459 __le16 w; 460 __le32 l; 461 462 BUG_ON(offset + len > mdev->device_len); 463 464 switch (len) { 465 case 1: 466 b = ioread8(device + offset); 467 memcpy(buf, &b, sizeof b); 468 break; 469 case 2: 470 w = cpu_to_le16(ioread16(device + offset)); 471 memcpy(buf, &w, sizeof w); 472 break; 473 case 4: 474 l = cpu_to_le32(ioread32(device + offset)); 475 memcpy(buf, &l, sizeof l); 476 break; 477 case 8: 478 l = cpu_to_le32(ioread32(device + offset)); 479 memcpy(buf, &l, sizeof l); 480 l = cpu_to_le32(ioread32(device + offset + sizeof l)); 481 memcpy(buf + sizeof l, &l, sizeof l); 482 break; 483 default: 484 BUG(); 485 } 486 } 487 488 /* the config->set() implementation. it's symmetric to the config->get() 489 * implementation */ 490 static void vp_set(struct virtio_device *vdev, unsigned int offset, 491 const void *buf, unsigned int len) 492 { 493 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 494 struct virtio_pci_modern_device *mdev = &vp_dev->mdev; 495 void __iomem *device = mdev->device; 496 u8 b; 497 __le16 w; 498 __le32 l; 499 500 BUG_ON(offset + len > mdev->device_len); 501 502 switch (len) { 503 case 1: 504 memcpy(&b, buf, sizeof b); 505 iowrite8(b, device + offset); 506 break; 507 case 2: 508 memcpy(&w, buf, sizeof w); 509 iowrite16(le16_to_cpu(w), device + offset); 510 break; 511 case 4: 512 memcpy(&l, buf, sizeof l); 513 iowrite32(le32_to_cpu(l), device + offset); 514 break; 515 case 8: 516 memcpy(&l, buf, sizeof l); 517 iowrite32(le32_to_cpu(l), device + offset); 518 memcpy(&l, buf + sizeof l, sizeof l); 519 iowrite32(le32_to_cpu(l), device + offset + sizeof l); 520 break; 521 default: 522 BUG(); 523 } 524 } 525 526 static u32 vp_generation(struct virtio_device *vdev) 527 { 528 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 529 530 return vp_modern_generation(&vp_dev->mdev); 531 } 532 533 /* config->{get,set}_status() implementations */ 534 static u8 vp_get_status(struct virtio_device *vdev) 535 { 536 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 537 538 return vp_modern_get_status(&vp_dev->mdev); 539 } 540 541 static void vp_set_status(struct virtio_device *vdev, u8 status) 542 { 543 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 544 545 /* We should never be setting status to 0. */ 546 BUG_ON(status == 0); 547 vp_modern_set_status(&vp_dev->mdev, status); 548 if (status & VIRTIO_CONFIG_S_DRIVER_OK) 549 vp_modern_avq_activate(vdev); 550 } 551 552 static void vp_reset(struct virtio_device *vdev) 553 { 554 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 555 struct virtio_pci_modern_device *mdev = &vp_dev->mdev; 556 557 /* 0 status means a reset. */ 558 vp_modern_set_status(mdev, 0); 559 /* After writing 0 to device_status, the driver MUST wait for a read of 560 * device_status to return 0 before reinitializing the device. 561 * This will flush out the status write, and flush in device writes, 562 * including MSI-X interrupts, if any. 563 */ 564 while (vp_modern_get_status(mdev)) 565 msleep(1); 566 } 567 568 static int vp_active_vq(struct virtqueue *vq, u16 msix_vec) 569 { 570 struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); 571 struct virtio_pci_modern_device *mdev = &vp_dev->mdev; 572 unsigned long index; 573 574 index = vq->index; 575 576 /* activate the queue */ 577 vp_modern_set_queue_size(mdev, index, virtqueue_get_vring_size(vq)); 578 vp_modern_queue_address(mdev, index, virtqueue_get_desc_addr(vq), 579 virtqueue_get_avail_addr(vq), 580 virtqueue_get_used_addr(vq)); 581 582 if (msix_vec != VIRTIO_MSI_NO_VECTOR) { 583 msix_vec = vp_modern_queue_vector(mdev, index, msix_vec); 584 if (msix_vec == VIRTIO_MSI_NO_VECTOR) 585 return -EBUSY; 586 } 587 588 return 0; 589 } 590 591 static int vp_modern_disable_vq_and_reset(struct virtqueue *vq) 592 { 593 struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); 594 struct virtio_pci_modern_device *mdev = &vp_dev->mdev; 595 struct virtio_pci_vq_info *info; 596 unsigned long flags; 597 598 if (!virtio_has_feature(vq->vdev, VIRTIO_F_RING_RESET)) 599 return -ENOENT; 600 601 vp_modern_set_queue_reset(mdev, vq->index); 602 603 info = vp_dev->vqs[vq->index]; 604 605 /* delete vq from irq handler */ 606 spin_lock_irqsave(&vp_dev->lock, flags); 607 list_del(&info->node); 608 spin_unlock_irqrestore(&vp_dev->lock, flags); 609 610 INIT_LIST_HEAD(&info->node); 611 612 #ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION 613 __virtqueue_break(vq); 614 #endif 615 616 /* For the case where vq has an exclusive irq, call synchronize_irq() to 617 * wait for completion. 618 * 619 * note: We can't use disable_irq() since it conflicts with the affinity 620 * managed IRQ that is used by some drivers. 621 */ 622 if (vp_dev->per_vq_vectors && info->msix_vector != VIRTIO_MSI_NO_VECTOR) 623 synchronize_irq(pci_irq_vector(vp_dev->pci_dev, info->msix_vector)); 624 625 vq->reset = true; 626 627 return 0; 628 } 629 630 static int vp_modern_enable_vq_after_reset(struct virtqueue *vq) 631 { 632 struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); 633 struct virtio_pci_modern_device *mdev = &vp_dev->mdev; 634 struct virtio_pci_vq_info *info; 635 unsigned long flags, index; 636 int err; 637 638 if (!vq->reset) 639 return -EBUSY; 640 641 index = vq->index; 642 info = vp_dev->vqs[index]; 643 644 if (vp_modern_get_queue_reset(mdev, index)) 645 return -EBUSY; 646 647 if (vp_modern_get_queue_enable(mdev, index)) 648 return -EBUSY; 649 650 err = vp_active_vq(vq, info->msix_vector); 651 if (err) 652 return err; 653 654 if (vq->callback) { 655 spin_lock_irqsave(&vp_dev->lock, flags); 656 list_add(&info->node, &vp_dev->virtqueues); 657 spin_unlock_irqrestore(&vp_dev->lock, flags); 658 } else { 659 INIT_LIST_HEAD(&info->node); 660 } 661 662 #ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION 663 __virtqueue_unbreak(vq); 664 #endif 665 666 vp_modern_set_queue_enable(&vp_dev->mdev, index, true); 667 vq->reset = false; 668 669 return 0; 670 } 671 672 static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector) 673 { 674 return vp_modern_config_vector(&vp_dev->mdev, vector); 675 } 676 677 static bool vp_notify_with_data(struct virtqueue *vq) 678 { 679 u32 data = vring_notification_data(vq); 680 681 iowrite32(data, (void __iomem *)vq->priv); 682 683 return true; 684 } 685 686 static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, 687 struct virtio_pci_vq_info *info, 688 unsigned int index, 689 void (*callback)(struct virtqueue *vq), 690 const char *name, 691 bool ctx, 692 u16 msix_vec) 693 { 694 695 struct virtio_pci_modern_device *mdev = &vp_dev->mdev; 696 bool (*notify)(struct virtqueue *vq); 697 struct virtqueue *vq; 698 bool is_avq; 699 u16 num; 700 int err; 701 702 if (__virtio_test_bit(&vp_dev->vdev, VIRTIO_F_NOTIFICATION_DATA)) 703 notify = vp_notify_with_data; 704 else 705 notify = vp_notify; 706 707 is_avq = vp_is_avq(&vp_dev->vdev, index); 708 if (index >= vp_modern_get_num_queues(mdev) && !is_avq) 709 return ERR_PTR(-EINVAL); 710 711 num = vp_modern_get_queue_size(mdev, index); 712 /* Check if queue is either not available or already active. */ 713 if (!num || vp_modern_get_queue_enable(mdev, index)) 714 return ERR_PTR(-ENOENT); 715 716 info->msix_vector = msix_vec; 717 718 /* create the vring */ 719 vq = vring_create_virtqueue(index, num, 720 SMP_CACHE_BYTES, &vp_dev->vdev, 721 true, true, ctx, 722 notify, callback, name); 723 if (!vq) 724 return ERR_PTR(-ENOMEM); 725 726 vq->num_max = num; 727 728 err = vp_active_vq(vq, msix_vec); 729 if (err) 730 goto err; 731 732 vq->priv = (void __force *)vp_modern_map_vq_notify(mdev, index, NULL); 733 if (!vq->priv) { 734 err = -ENOMEM; 735 goto err; 736 } 737 738 return vq; 739 740 err: 741 vring_del_virtqueue(vq); 742 return ERR_PTR(err); 743 } 744 745 static int vp_modern_find_vqs(struct virtio_device *vdev, unsigned int nvqs, 746 struct virtqueue *vqs[], 747 struct virtqueue_info vqs_info[], 748 struct irq_affinity *desc) 749 { 750 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 751 struct virtqueue *vq; 752 int rc = vp_find_vqs(vdev, nvqs, vqs, vqs_info, desc); 753 754 if (rc) 755 return rc; 756 757 /* Select and activate all queues. Has to be done last: once we do 758 * this, there's no way to go back except reset. 759 */ 760 list_for_each_entry(vq, &vdev->vqs, list) 761 vp_modern_set_queue_enable(&vp_dev->mdev, vq->index, true); 762 763 return 0; 764 } 765 766 static void del_vq(struct virtio_pci_vq_info *info) 767 { 768 struct virtqueue *vq = info->vq; 769 struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); 770 struct virtio_pci_modern_device *mdev = &vp_dev->mdev; 771 772 if (vp_dev->msix_enabled) 773 vp_modern_queue_vector(mdev, vq->index, 774 VIRTIO_MSI_NO_VECTOR); 775 776 if (!mdev->notify_base) 777 pci_iounmap(mdev->pci_dev, (void __force __iomem *)vq->priv); 778 779 vring_del_virtqueue(vq); 780 } 781 782 static int virtio_pci_find_shm_cap(struct pci_dev *dev, u8 required_id, 783 u8 *bar, u64 *offset, u64 *len) 784 { 785 int pos; 786 787 for (pos = pci_find_capability(dev, PCI_CAP_ID_VNDR); pos > 0; 788 pos = pci_find_next_capability(dev, pos, PCI_CAP_ID_VNDR)) { 789 u8 type, cap_len, id, res_bar; 790 u32 tmp32; 791 u64 res_offset, res_length; 792 793 pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap, 794 cfg_type), &type); 795 if (type != VIRTIO_PCI_CAP_SHARED_MEMORY_CFG) 796 continue; 797 798 pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap, 799 cap_len), &cap_len); 800 if (cap_len != sizeof(struct virtio_pci_cap64)) { 801 dev_err(&dev->dev, "%s: shm cap with bad size offset:" 802 " %d size: %d\n", __func__, pos, cap_len); 803 continue; 804 } 805 806 pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap, 807 id), &id); 808 if (id != required_id) 809 continue; 810 811 pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap, 812 bar), &res_bar); 813 if (res_bar >= PCI_STD_NUM_BARS) 814 continue; 815 816 /* Type and ID match, and the BAR value isn't reserved. 817 * Looks good. 818 */ 819 820 /* Read the lower 32bit of length and offset */ 821 pci_read_config_dword(dev, pos + offsetof(struct virtio_pci_cap, 822 offset), &tmp32); 823 res_offset = tmp32; 824 pci_read_config_dword(dev, pos + offsetof(struct virtio_pci_cap, 825 length), &tmp32); 826 res_length = tmp32; 827 828 /* and now the top half */ 829 pci_read_config_dword(dev, 830 pos + offsetof(struct virtio_pci_cap64, 831 offset_hi), &tmp32); 832 res_offset |= ((u64)tmp32) << 32; 833 pci_read_config_dword(dev, 834 pos + offsetof(struct virtio_pci_cap64, 835 length_hi), &tmp32); 836 res_length |= ((u64)tmp32) << 32; 837 838 *bar = res_bar; 839 *offset = res_offset; 840 *len = res_length; 841 842 return pos; 843 } 844 return 0; 845 } 846 847 static bool vp_get_shm_region(struct virtio_device *vdev, 848 struct virtio_shm_region *region, u8 id) 849 { 850 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 851 struct pci_dev *pci_dev = vp_dev->pci_dev; 852 u8 bar; 853 u64 offset, len; 854 phys_addr_t phys_addr; 855 size_t bar_len; 856 857 if (!virtio_pci_find_shm_cap(pci_dev, id, &bar, &offset, &len)) 858 return false; 859 860 phys_addr = pci_resource_start(pci_dev, bar); 861 bar_len = pci_resource_len(pci_dev, bar); 862 863 if ((offset + len) < offset) { 864 dev_err(&pci_dev->dev, "%s: cap offset+len overflow detected\n", 865 __func__); 866 return false; 867 } 868 869 if (offset + len > bar_len) { 870 dev_err(&pci_dev->dev, "%s: bar shorter than cap offset+len\n", 871 __func__); 872 return false; 873 } 874 875 region->len = len; 876 region->addr = (u64) phys_addr + offset; 877 878 return true; 879 } 880 881 /* 882 * virtio_pci_admin_has_dev_parts - Checks whether the device parts 883 * functionality is supported 884 * @pdev: VF pci_dev 885 * 886 * Returns true on success. 887 */ 888 bool virtio_pci_admin_has_dev_parts(struct pci_dev *pdev) 889 { 890 struct virtio_device *virtio_dev = virtio_pci_vf_get_pf_dev(pdev); 891 struct virtio_pci_device *vp_dev; 892 893 if (!virtio_dev) 894 return false; 895 896 if (!virtio_has_feature(virtio_dev, VIRTIO_F_ADMIN_VQ)) 897 return false; 898 899 vp_dev = to_vp_device(virtio_dev); 900 901 if (!((vp_dev->admin_vq.supported_cmds & VIRTIO_DEV_PARTS_ADMIN_CMD_BITMAP) == 902 VIRTIO_DEV_PARTS_ADMIN_CMD_BITMAP)) 903 return false; 904 905 return vp_dev->admin_vq.max_dev_parts_objects; 906 } 907 EXPORT_SYMBOL_GPL(virtio_pci_admin_has_dev_parts); 908 909 /* 910 * virtio_pci_admin_mode_set - Sets the mode of a member device 911 * @pdev: VF pci_dev 912 * @flags: device mode's flags 913 * 914 * Note: caller must serialize access for the given device. 915 * Returns 0 on success, or negative on failure. 916 */ 917 int virtio_pci_admin_mode_set(struct pci_dev *pdev, u8 flags) 918 { 919 struct virtio_device *virtio_dev = virtio_pci_vf_get_pf_dev(pdev); 920 struct virtio_admin_cmd_dev_mode_set_data *data; 921 struct virtio_admin_cmd cmd = {}; 922 struct scatterlist data_sg; 923 int vf_id; 924 int ret; 925 926 if (!virtio_dev) 927 return -ENODEV; 928 929 vf_id = pci_iov_vf_id(pdev); 930 if (vf_id < 0) 931 return vf_id; 932 933 data = kzalloc_obj(*data); 934 if (!data) 935 return -ENOMEM; 936 937 data->flags = flags; 938 sg_init_one(&data_sg, data, sizeof(*data)); 939 cmd.opcode = cpu_to_le16(VIRTIO_ADMIN_CMD_DEV_MODE_SET); 940 cmd.group_type = cpu_to_le16(VIRTIO_ADMIN_GROUP_TYPE_SRIOV); 941 cmd.group_member_id = cpu_to_le64(vf_id + 1); 942 cmd.data_sg = &data_sg; 943 ret = vp_modern_admin_cmd_exec(virtio_dev, &cmd); 944 945 kfree(data); 946 return ret; 947 } 948 EXPORT_SYMBOL_GPL(virtio_pci_admin_mode_set); 949 950 /* 951 * virtio_pci_admin_obj_create - Creates an object for a given type and operation, 952 * following the max objects that can be created for that request. 953 * @pdev: VF pci_dev 954 * @obj_type: Object type 955 * @operation_type: Operation type 956 * @obj_id: Output unique object id 957 * 958 * Note: caller must serialize access for the given device. 959 * Returns 0 on success, or negative on failure. 960 */ 961 int virtio_pci_admin_obj_create(struct pci_dev *pdev, u16 obj_type, u8 operation_type, 962 u32 *obj_id) 963 { 964 struct virtio_device *virtio_dev = virtio_pci_vf_get_pf_dev(pdev); 965 u16 data_size = sizeof(struct virtio_admin_cmd_resource_obj_create_data); 966 struct virtio_admin_cmd_resource_obj_create_data *obj_create_data; 967 struct virtio_resource_obj_dev_parts obj_dev_parts = {}; 968 struct virtio_pci_admin_vq *avq; 969 struct virtio_admin_cmd cmd = {}; 970 struct scatterlist data_sg; 971 void *data; 972 int id = -1; 973 int vf_id; 974 int ret; 975 976 if (!virtio_dev) 977 return -ENODEV; 978 979 vf_id = pci_iov_vf_id(pdev); 980 if (vf_id < 0) 981 return vf_id; 982 983 if (obj_type != VIRTIO_RESOURCE_OBJ_DEV_PARTS) 984 return -EOPNOTSUPP; 985 986 if (operation_type != VIRTIO_RESOURCE_OBJ_DEV_PARTS_TYPE_GET && 987 operation_type != VIRTIO_RESOURCE_OBJ_DEV_PARTS_TYPE_SET) 988 return -EINVAL; 989 990 avq = &to_vp_device(virtio_dev)->admin_vq; 991 if (!avq->max_dev_parts_objects) 992 return -EOPNOTSUPP; 993 994 id = ida_alloc_range(&avq->dev_parts_ida, 0, 995 avq->max_dev_parts_objects - 1, GFP_KERNEL); 996 if (id < 0) 997 return id; 998 999 *obj_id = id; 1000 data_size += sizeof(obj_dev_parts); 1001 data = kzalloc(data_size, GFP_KERNEL); 1002 if (!data) { 1003 ret = -ENOMEM; 1004 goto end; 1005 } 1006 1007 obj_create_data = data; 1008 obj_create_data->hdr.type = cpu_to_le16(obj_type); 1009 obj_create_data->hdr.id = cpu_to_le32(*obj_id); 1010 obj_dev_parts.type = operation_type; 1011 memcpy(obj_create_data->resource_obj_specific_data, &obj_dev_parts, 1012 sizeof(obj_dev_parts)); 1013 sg_init_one(&data_sg, data, data_size); 1014 cmd.opcode = cpu_to_le16(VIRTIO_ADMIN_CMD_RESOURCE_OBJ_CREATE); 1015 cmd.group_type = cpu_to_le16(VIRTIO_ADMIN_GROUP_TYPE_SRIOV); 1016 cmd.group_member_id = cpu_to_le64(vf_id + 1); 1017 cmd.data_sg = &data_sg; 1018 ret = vp_modern_admin_cmd_exec(virtio_dev, &cmd); 1019 1020 kfree(data); 1021 end: 1022 if (ret) 1023 ida_free(&avq->dev_parts_ida, id); 1024 1025 return ret; 1026 } 1027 EXPORT_SYMBOL_GPL(virtio_pci_admin_obj_create); 1028 1029 /* 1030 * virtio_pci_admin_obj_destroy - Destroys an object of a given type and id 1031 * @pdev: VF pci_dev 1032 * @obj_type: Object type 1033 * @id: Object id 1034 * 1035 * Note: caller must serialize access for the given device. 1036 * Returns 0 on success, or negative on failure. 1037 */ 1038 int virtio_pci_admin_obj_destroy(struct pci_dev *pdev, u16 obj_type, u32 id) 1039 { 1040 struct virtio_device *virtio_dev = virtio_pci_vf_get_pf_dev(pdev); 1041 struct virtio_admin_cmd_resource_obj_cmd_hdr *data; 1042 struct virtio_pci_device *vp_dev; 1043 struct virtio_admin_cmd cmd = {}; 1044 struct scatterlist data_sg; 1045 int vf_id; 1046 int ret; 1047 1048 if (!virtio_dev) 1049 return -ENODEV; 1050 1051 vf_id = pci_iov_vf_id(pdev); 1052 if (vf_id < 0) 1053 return vf_id; 1054 1055 if (obj_type != VIRTIO_RESOURCE_OBJ_DEV_PARTS) 1056 return -EINVAL; 1057 1058 data = kzalloc_obj(*data); 1059 if (!data) 1060 return -ENOMEM; 1061 1062 data->type = cpu_to_le16(obj_type); 1063 data->id = cpu_to_le32(id); 1064 sg_init_one(&data_sg, data, sizeof(*data)); 1065 cmd.opcode = cpu_to_le16(VIRTIO_ADMIN_CMD_RESOURCE_OBJ_DESTROY); 1066 cmd.group_type = cpu_to_le16(VIRTIO_ADMIN_GROUP_TYPE_SRIOV); 1067 cmd.group_member_id = cpu_to_le64(vf_id + 1); 1068 cmd.data_sg = &data_sg; 1069 ret = vp_modern_admin_cmd_exec(virtio_dev, &cmd); 1070 if (!ret) { 1071 vp_dev = to_vp_device(virtio_dev); 1072 ida_free(&vp_dev->admin_vq.dev_parts_ida, id); 1073 } 1074 1075 kfree(data); 1076 return ret; 1077 } 1078 EXPORT_SYMBOL_GPL(virtio_pci_admin_obj_destroy); 1079 1080 /* 1081 * virtio_pci_admin_dev_parts_metadata_get - Gets the metadata of the device parts 1082 * identified by the below attributes. 1083 * @pdev: VF pci_dev 1084 * @obj_type: Object type 1085 * @id: Object id 1086 * @metadata_type: Metadata type 1087 * @out: Upon success holds the output for 'metadata type size' 1088 * 1089 * Note: caller must serialize access for the given device. 1090 * Returns 0 on success, or negative on failure. 1091 */ 1092 int virtio_pci_admin_dev_parts_metadata_get(struct pci_dev *pdev, u16 obj_type, 1093 u32 id, u8 metadata_type, u32 *out) 1094 { 1095 struct virtio_device *virtio_dev = virtio_pci_vf_get_pf_dev(pdev); 1096 struct virtio_admin_cmd_dev_parts_metadata_result *result; 1097 struct virtio_admin_cmd_dev_parts_metadata_data *data; 1098 struct scatterlist data_sg, result_sg; 1099 struct virtio_admin_cmd cmd = {}; 1100 int vf_id; 1101 int ret; 1102 1103 if (!virtio_dev) 1104 return -ENODEV; 1105 1106 if (metadata_type != VIRTIO_ADMIN_CMD_DEV_PARTS_METADATA_TYPE_SIZE) 1107 return -EOPNOTSUPP; 1108 1109 vf_id = pci_iov_vf_id(pdev); 1110 if (vf_id < 0) 1111 return vf_id; 1112 1113 data = kzalloc_obj(*data); 1114 if (!data) 1115 return -ENOMEM; 1116 1117 result = kzalloc_obj(*result); 1118 if (!result) { 1119 ret = -ENOMEM; 1120 goto end; 1121 } 1122 1123 data->hdr.type = cpu_to_le16(obj_type); 1124 data->hdr.id = cpu_to_le32(id); 1125 data->type = metadata_type; 1126 sg_init_one(&data_sg, data, sizeof(*data)); 1127 sg_init_one(&result_sg, result, sizeof(*result)); 1128 cmd.opcode = cpu_to_le16(VIRTIO_ADMIN_CMD_DEV_PARTS_METADATA_GET); 1129 cmd.group_type = cpu_to_le16(VIRTIO_ADMIN_GROUP_TYPE_SRIOV); 1130 cmd.group_member_id = cpu_to_le64(vf_id + 1); 1131 cmd.data_sg = &data_sg; 1132 cmd.result_sg = &result_sg; 1133 ret = vp_modern_admin_cmd_exec(virtio_dev, &cmd); 1134 if (!ret) 1135 *out = le32_to_cpu(result->parts_size.size); 1136 1137 kfree(result); 1138 end: 1139 kfree(data); 1140 return ret; 1141 } 1142 EXPORT_SYMBOL_GPL(virtio_pci_admin_dev_parts_metadata_get); 1143 1144 /* 1145 * virtio_pci_admin_dev_parts_get - Gets the device parts identified by the below attributes. 1146 * @pdev: VF pci_dev 1147 * @obj_type: Object type 1148 * @id: Object id 1149 * @get_type: Get type 1150 * @res_sg: Upon success holds the output result data 1151 * @res_size: Upon success holds the output result size 1152 * 1153 * Note: caller must serialize access for the given device. 1154 * Returns 0 on success, or negative on failure. 1155 */ 1156 int virtio_pci_admin_dev_parts_get(struct pci_dev *pdev, u16 obj_type, u32 id, 1157 u8 get_type, struct scatterlist *res_sg, 1158 u32 *res_size) 1159 { 1160 struct virtio_device *virtio_dev = virtio_pci_vf_get_pf_dev(pdev); 1161 struct virtio_admin_cmd_dev_parts_get_data *data; 1162 struct scatterlist data_sg; 1163 struct virtio_admin_cmd cmd = {}; 1164 int vf_id; 1165 int ret; 1166 1167 if (!virtio_dev) 1168 return -ENODEV; 1169 1170 if (get_type != VIRTIO_ADMIN_CMD_DEV_PARTS_GET_TYPE_ALL) 1171 return -EOPNOTSUPP; 1172 1173 vf_id = pci_iov_vf_id(pdev); 1174 if (vf_id < 0) 1175 return vf_id; 1176 1177 data = kzalloc_obj(*data); 1178 if (!data) 1179 return -ENOMEM; 1180 1181 data->hdr.type = cpu_to_le16(obj_type); 1182 data->hdr.id = cpu_to_le32(id); 1183 data->type = get_type; 1184 sg_init_one(&data_sg, data, sizeof(*data)); 1185 cmd.opcode = cpu_to_le16(VIRTIO_ADMIN_CMD_DEV_PARTS_GET); 1186 cmd.group_type = cpu_to_le16(VIRTIO_ADMIN_GROUP_TYPE_SRIOV); 1187 cmd.group_member_id = cpu_to_le64(vf_id + 1); 1188 cmd.data_sg = &data_sg; 1189 cmd.result_sg = res_sg; 1190 ret = vp_modern_admin_cmd_exec(virtio_dev, &cmd); 1191 if (!ret) 1192 *res_size = cmd.result_sg_size; 1193 1194 kfree(data); 1195 return ret; 1196 } 1197 EXPORT_SYMBOL_GPL(virtio_pci_admin_dev_parts_get); 1198 1199 /* 1200 * virtio_pci_admin_dev_parts_set - Sets the device parts identified by the below attributes. 1201 * @pdev: VF pci_dev 1202 * @data_sg: The device parts data, its layout follows struct virtio_admin_cmd_dev_parts_set_data 1203 * 1204 * Note: caller must serialize access for the given device. 1205 * Returns 0 on success, or negative on failure. 1206 */ 1207 int virtio_pci_admin_dev_parts_set(struct pci_dev *pdev, struct scatterlist *data_sg) 1208 { 1209 struct virtio_device *virtio_dev = virtio_pci_vf_get_pf_dev(pdev); 1210 struct virtio_admin_cmd cmd = {}; 1211 int vf_id; 1212 1213 if (!virtio_dev) 1214 return -ENODEV; 1215 1216 vf_id = pci_iov_vf_id(pdev); 1217 if (vf_id < 0) 1218 return vf_id; 1219 1220 cmd.opcode = cpu_to_le16(VIRTIO_ADMIN_CMD_DEV_PARTS_SET); 1221 cmd.group_type = cpu_to_le16(VIRTIO_ADMIN_GROUP_TYPE_SRIOV); 1222 cmd.group_member_id = cpu_to_le64(vf_id + 1); 1223 cmd.data_sg = data_sg; 1224 return vp_modern_admin_cmd_exec(virtio_dev, &cmd); 1225 } 1226 EXPORT_SYMBOL_GPL(virtio_pci_admin_dev_parts_set); 1227 1228 static const struct virtio_config_ops virtio_pci_config_nodev_ops = { 1229 .get = NULL, 1230 .set = NULL, 1231 .generation = vp_generation, 1232 .get_status = vp_get_status, 1233 .set_status = vp_set_status, 1234 .reset = vp_reset, 1235 .find_vqs = vp_modern_find_vqs, 1236 .del_vqs = vp_modern_del_vqs, 1237 .synchronize_cbs = vp_synchronize_vectors, 1238 .get_extended_features = vp_get_features, 1239 .finalize_features = vp_finalize_features, 1240 .bus_name = vp_bus_name, 1241 .set_vq_affinity = vp_set_vq_affinity, 1242 .get_vq_affinity = vp_get_vq_affinity, 1243 .get_shm_region = vp_get_shm_region, 1244 .disable_vq_and_reset = vp_modern_disable_vq_and_reset, 1245 .enable_vq_after_reset = vp_modern_enable_vq_after_reset, 1246 }; 1247 1248 static const struct virtio_config_ops virtio_pci_config_ops = { 1249 .get = vp_get, 1250 .set = vp_set, 1251 .generation = vp_generation, 1252 .get_status = vp_get_status, 1253 .set_status = vp_set_status, 1254 .reset = vp_reset, 1255 .find_vqs = vp_modern_find_vqs, 1256 .del_vqs = vp_modern_del_vqs, 1257 .synchronize_cbs = vp_synchronize_vectors, 1258 .get_extended_features = vp_get_features, 1259 .finalize_features = vp_finalize_features, 1260 .bus_name = vp_bus_name, 1261 .set_vq_affinity = vp_set_vq_affinity, 1262 .get_vq_affinity = vp_get_vq_affinity, 1263 .get_shm_region = vp_get_shm_region, 1264 .disable_vq_and_reset = vp_modern_disable_vq_and_reset, 1265 .enable_vq_after_reset = vp_modern_enable_vq_after_reset, 1266 }; 1267 1268 /* the PCI probing function */ 1269 int virtio_pci_modern_probe(struct virtio_pci_device *vp_dev) 1270 { 1271 struct virtio_pci_modern_device *mdev = &vp_dev->mdev; 1272 struct pci_dev *pci_dev = vp_dev->pci_dev; 1273 int err; 1274 1275 mdev->pci_dev = pci_dev; 1276 1277 err = vp_modern_probe(mdev); 1278 if (err) 1279 return err; 1280 1281 if (mdev->device) 1282 vp_dev->vdev.config = &virtio_pci_config_ops; 1283 else 1284 vp_dev->vdev.config = &virtio_pci_config_nodev_ops; 1285 1286 vp_dev->config_vector = vp_config_vector; 1287 vp_dev->setup_vq = setup_vq; 1288 vp_dev->del_vq = del_vq; 1289 vp_dev->avq_index = vp_avq_index; 1290 vp_dev->isr = mdev->isr; 1291 vp_dev->vdev.id = mdev->id; 1292 1293 spin_lock_init(&vp_dev->admin_vq.lock); 1294 return 0; 1295 } 1296 1297 void virtio_pci_modern_remove(struct virtio_pci_device *vp_dev) 1298 { 1299 struct virtio_pci_modern_device *mdev = &vp_dev->mdev; 1300 1301 vp_modern_remove(mdev); 1302 } 1303