1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * vDPA bridge driver for modern virtio-pci device 4 * 5 * Copyright (c) 2020, Red Hat Inc. All rights reserved. 6 * Author: Jason Wang <jasowang@redhat.com> 7 * 8 * Based on virtio_pci_modern.c. 9 */ 10 11 #include <linux/interrupt.h> 12 #include <linux/module.h> 13 #include <linux/pci.h> 14 #include <linux/vdpa.h> 15 #include <linux/virtio.h> 16 #include <linux/virtio_config.h> 17 #include <linux/virtio_ring.h> 18 #include <linux/virtio_pci.h> 19 #include <linux/virtio_pci_modern.h> 20 #include <uapi/linux/vdpa.h> 21 22 #define VP_VDPA_QUEUE_MAX 256 23 #define VP_VDPA_DRIVER_NAME "vp_vdpa" 24 #define VP_VDPA_NAME_SIZE 256 25 26 struct vp_vring { 27 void __iomem *notify; 28 char msix_name[VP_VDPA_NAME_SIZE]; 29 struct vdpa_callback cb; 30 resource_size_t notify_pa; 31 int irq; 32 }; 33 34 struct vp_vdpa { 35 struct vdpa_device vdpa; 36 struct virtio_pci_modern_device *mdev; 37 struct vp_vring *vring; 38 struct vdpa_callback config_cb; 39 u64 device_features; 40 char msix_name[VP_VDPA_NAME_SIZE]; 41 int config_irq; 42 int queues; 43 int vectors; 44 }; 45 46 struct vp_vdpa_mgmtdev { 47 struct vdpa_mgmt_dev mgtdev; 48 struct virtio_pci_modern_device *mdev; 49 struct vp_vdpa *vp_vdpa; 50 }; 51 52 static struct vp_vdpa *vdpa_to_vp(struct vdpa_device *vdpa) 53 { 54 return container_of(vdpa, struct vp_vdpa, vdpa); 55 } 56 57 static struct virtio_pci_modern_device *vdpa_to_mdev(struct vdpa_device *vdpa) 58 { 59 struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa); 60 61 return vp_vdpa->mdev; 62 } 63 64 static struct virtio_pci_modern_device *vp_vdpa_to_mdev(struct vp_vdpa *vp_vdpa) 65 { 66 return vp_vdpa->mdev; 67 } 68 69 static u64 vp_vdpa_get_device_features(struct vdpa_device *vdpa) 70 { 71 struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa); 72 73 return vp_vdpa->device_features; 74 } 75 76 static int vp_vdpa_set_driver_features(struct vdpa_device *vdpa, u64 features) 77 { 78 struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); 79 80 vp_modern_set_features(mdev, features); 81 82 return 0; 83 } 84 85 static u64 vp_vdpa_get_driver_features(struct vdpa_device *vdpa) 86 { 87 struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); 88 89 return vp_modern_get_driver_features(mdev); 90 } 91 92 static u8 vp_vdpa_get_status(struct vdpa_device *vdpa) 93 { 94 struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); 95 96 return vp_modern_get_status(mdev); 97 } 98 99 static int vp_vdpa_get_vq_irq(struct vdpa_device *vdpa, u16 idx) 100 { 101 struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa); 102 int irq = vp_vdpa->vring[idx].irq; 103 104 if (irq == VIRTIO_MSI_NO_VECTOR) 105 return -EINVAL; 106 107 return irq; 108 } 109 110 static void vp_vdpa_free_irq(struct vp_vdpa *vp_vdpa) 111 { 112 struct virtio_pci_modern_device *mdev = vp_vdpa_to_mdev(vp_vdpa); 113 struct pci_dev *pdev = mdev->pci_dev; 114 int i; 115 116 for (i = 0; i < vp_vdpa->queues; i++) { 117 if (vp_vdpa->vring[i].irq != VIRTIO_MSI_NO_VECTOR) { 118 vp_modern_queue_vector(mdev, i, VIRTIO_MSI_NO_VECTOR); 119 devm_free_irq(&pdev->dev, vp_vdpa->vring[i].irq, 120 &vp_vdpa->vring[i]); 121 vp_vdpa->vring[i].irq = VIRTIO_MSI_NO_VECTOR; 122 } 123 } 124 125 if (vp_vdpa->config_irq != VIRTIO_MSI_NO_VECTOR) { 126 vp_modern_config_vector(mdev, VIRTIO_MSI_NO_VECTOR); 127 devm_free_irq(&pdev->dev, vp_vdpa->config_irq, vp_vdpa); 128 vp_vdpa->config_irq = VIRTIO_MSI_NO_VECTOR; 129 } 130 131 if (vp_vdpa->vectors) { 132 pci_free_irq_vectors(pdev); 133 vp_vdpa->vectors = 0; 134 } 135 } 136 137 static irqreturn_t vp_vdpa_vq_handler(int irq, void *arg) 138 { 139 struct vp_vring *vring = arg; 140 141 if (vring->cb.callback) 142 return vring->cb.callback(vring->cb.private); 143 144 return IRQ_HANDLED; 145 } 146 147 static irqreturn_t vp_vdpa_config_handler(int irq, void *arg) 148 { 149 struct vp_vdpa *vp_vdpa = arg; 150 151 if (vp_vdpa->config_cb.callback) 152 return vp_vdpa->config_cb.callback(vp_vdpa->config_cb.private); 153 154 return IRQ_HANDLED; 155 } 156 157 static int vp_vdpa_request_irq(struct vp_vdpa *vp_vdpa) 158 { 159 struct virtio_pci_modern_device *mdev = vp_vdpa_to_mdev(vp_vdpa); 160 struct pci_dev *pdev = mdev->pci_dev; 161 int i, ret, irq; 162 int queues = vp_vdpa->queues; 163 int vectors = queues + 1; 164 165 ret = pci_alloc_irq_vectors(pdev, vectors, vectors, PCI_IRQ_MSIX); 166 if (ret != vectors) { 167 dev_err(&pdev->dev, 168 "vp_vdpa: fail to allocate irq vectors want %d but %d\n", 169 vectors, ret); 170 return ret; 171 } 172 173 vp_vdpa->vectors = vectors; 174 175 for (i = 0; i < queues; i++) { 176 snprintf(vp_vdpa->vring[i].msix_name, VP_VDPA_NAME_SIZE, 177 "vp-vdpa[%s]-%d\n", pci_name(pdev), i); 178 irq = pci_irq_vector(pdev, i); 179 ret = devm_request_irq(&pdev->dev, irq, 180 vp_vdpa_vq_handler, 181 0, vp_vdpa->vring[i].msix_name, 182 &vp_vdpa->vring[i]); 183 if (ret) { 184 dev_err(&pdev->dev, 185 "vp_vdpa: fail to request irq for vq %d\n", i); 186 goto err; 187 } 188 vp_modern_queue_vector(mdev, i, i); 189 vp_vdpa->vring[i].irq = irq; 190 } 191 192 snprintf(vp_vdpa->msix_name, VP_VDPA_NAME_SIZE, "vp-vdpa[%s]-config\n", 193 pci_name(pdev)); 194 irq = pci_irq_vector(pdev, queues); 195 ret = devm_request_irq(&pdev->dev, irq, vp_vdpa_config_handler, 0, 196 vp_vdpa->msix_name, vp_vdpa); 197 if (ret) { 198 dev_err(&pdev->dev, 199 "vp_vdpa: fail to request irq for vq %d\n", i); 200 goto err; 201 } 202 vp_modern_config_vector(mdev, queues); 203 vp_vdpa->config_irq = irq; 204 205 return 0; 206 err: 207 vp_vdpa_free_irq(vp_vdpa); 208 return ret; 209 } 210 211 static void vp_vdpa_set_status(struct vdpa_device *vdpa, u8 status) 212 { 213 struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa); 214 struct virtio_pci_modern_device *mdev = vp_vdpa_to_mdev(vp_vdpa); 215 u8 s = vp_vdpa_get_status(vdpa); 216 217 if (status & VIRTIO_CONFIG_S_DRIVER_OK && 218 !(s & VIRTIO_CONFIG_S_DRIVER_OK)) { 219 vp_vdpa_request_irq(vp_vdpa); 220 } 221 222 vp_modern_set_status(mdev, status); 223 } 224 225 static int vp_vdpa_reset(struct vdpa_device *vdpa) 226 { 227 struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa); 228 struct virtio_pci_modern_device *mdev = vp_vdpa_to_mdev(vp_vdpa); 229 u8 s = vp_vdpa_get_status(vdpa); 230 231 vp_modern_set_status(mdev, 0); 232 233 if (s & VIRTIO_CONFIG_S_DRIVER_OK) 234 vp_vdpa_free_irq(vp_vdpa); 235 236 return 0; 237 } 238 239 static u16 vp_vdpa_get_vq_num_max(struct vdpa_device *vdpa) 240 { 241 return VP_VDPA_QUEUE_MAX; 242 } 243 244 static int vp_vdpa_get_vq_state(struct vdpa_device *vdpa, u16 qid, 245 struct vdpa_vq_state *state) 246 { 247 /* Note that this is not supported by virtio specification, so 248 * we return -EOPNOTSUPP here. This means we can't support live 249 * migration, vhost device start/stop. 250 */ 251 return -EOPNOTSUPP; 252 } 253 254 static int vp_vdpa_set_vq_state_split(struct vdpa_device *vdpa, 255 const struct vdpa_vq_state *state) 256 { 257 const struct vdpa_vq_state_split *split = &state->split; 258 259 if (split->avail_index == 0) 260 return 0; 261 262 return -EOPNOTSUPP; 263 } 264 265 static int vp_vdpa_set_vq_state_packed(struct vdpa_device *vdpa, 266 const struct vdpa_vq_state *state) 267 { 268 const struct vdpa_vq_state_packed *packed = &state->packed; 269 270 if (packed->last_avail_counter == 1 && 271 packed->last_avail_idx == 0 && 272 packed->last_used_counter == 1 && 273 packed->last_used_idx == 0) 274 return 0; 275 276 return -EOPNOTSUPP; 277 } 278 279 static int vp_vdpa_set_vq_state(struct vdpa_device *vdpa, u16 qid, 280 const struct vdpa_vq_state *state) 281 { 282 struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); 283 284 /* Note that this is not supported by virtio specification. 285 * But if the state is by chance equal to the device initial 286 * state, we can let it go. 287 */ 288 if ((vp_modern_get_status(mdev) & VIRTIO_CONFIG_S_FEATURES_OK) && 289 !vp_modern_get_queue_enable(mdev, qid)) { 290 if (vp_modern_get_driver_features(mdev) & 291 BIT_ULL(VIRTIO_F_RING_PACKED)) 292 return vp_vdpa_set_vq_state_packed(vdpa, state); 293 else 294 return vp_vdpa_set_vq_state_split(vdpa, state); 295 } 296 297 return -EOPNOTSUPP; 298 } 299 300 static void vp_vdpa_set_vq_cb(struct vdpa_device *vdpa, u16 qid, 301 struct vdpa_callback *cb) 302 { 303 struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa); 304 305 vp_vdpa->vring[qid].cb = *cb; 306 } 307 308 static void vp_vdpa_set_vq_ready(struct vdpa_device *vdpa, 309 u16 qid, bool ready) 310 { 311 struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); 312 313 vp_modern_set_queue_enable(mdev, qid, ready); 314 } 315 316 static bool vp_vdpa_get_vq_ready(struct vdpa_device *vdpa, u16 qid) 317 { 318 struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); 319 320 return vp_modern_get_queue_enable(mdev, qid); 321 } 322 323 static void vp_vdpa_set_vq_num(struct vdpa_device *vdpa, u16 qid, 324 u32 num) 325 { 326 struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); 327 328 vp_modern_set_queue_size(mdev, qid, num); 329 } 330 331 static int vp_vdpa_set_vq_address(struct vdpa_device *vdpa, u16 qid, 332 u64 desc_area, u64 driver_area, 333 u64 device_area) 334 { 335 struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); 336 337 vp_modern_queue_address(mdev, qid, desc_area, 338 driver_area, device_area); 339 340 return 0; 341 } 342 343 static void vp_vdpa_kick_vq(struct vdpa_device *vdpa, u16 qid) 344 { 345 struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa); 346 347 vp_iowrite16(qid, vp_vdpa->vring[qid].notify); 348 } 349 350 static u32 vp_vdpa_get_generation(struct vdpa_device *vdpa) 351 { 352 struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); 353 354 return vp_modern_generation(mdev); 355 } 356 357 static u32 vp_vdpa_get_device_id(struct vdpa_device *vdpa) 358 { 359 struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); 360 361 return mdev->id.device; 362 } 363 364 static u32 vp_vdpa_get_vendor_id(struct vdpa_device *vdpa) 365 { 366 struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); 367 368 return mdev->id.vendor; 369 } 370 371 static u32 vp_vdpa_get_vq_align(struct vdpa_device *vdpa) 372 { 373 return PAGE_SIZE; 374 } 375 376 static size_t vp_vdpa_get_config_size(struct vdpa_device *vdpa) 377 { 378 struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); 379 380 return mdev->device_len; 381 } 382 383 static void vp_vdpa_get_config(struct vdpa_device *vdpa, 384 unsigned int offset, 385 void *buf, unsigned int len) 386 { 387 struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa); 388 struct virtio_pci_modern_device *mdev = vp_vdpa_to_mdev(vp_vdpa); 389 u8 old, new; 390 u8 *p; 391 int i; 392 393 do { 394 old = vp_ioread8(&mdev->common->config_generation); 395 p = buf; 396 for (i = 0; i < len; i++) 397 *p++ = vp_ioread8(mdev->device + offset + i); 398 399 new = vp_ioread8(&mdev->common->config_generation); 400 } while (old != new); 401 } 402 403 static void vp_vdpa_set_config(struct vdpa_device *vdpa, 404 unsigned int offset, const void *buf, 405 unsigned int len) 406 { 407 struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa); 408 struct virtio_pci_modern_device *mdev = vp_vdpa_to_mdev(vp_vdpa); 409 const u8 *p = buf; 410 int i; 411 412 for (i = 0; i < len; i++) 413 vp_iowrite8(*p++, mdev->device + offset + i); 414 } 415 416 static void vp_vdpa_set_config_cb(struct vdpa_device *vdpa, 417 struct vdpa_callback *cb) 418 { 419 struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa); 420 421 vp_vdpa->config_cb = *cb; 422 } 423 424 static struct vdpa_notification_area 425 vp_vdpa_get_vq_notification(struct vdpa_device *vdpa, u16 qid) 426 { 427 struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa); 428 struct virtio_pci_modern_device *mdev = vp_vdpa_to_mdev(vp_vdpa); 429 struct vdpa_notification_area notify; 430 431 notify.addr = vp_vdpa->vring[qid].notify_pa; 432 notify.size = mdev->notify_offset_multiplier; 433 434 return notify; 435 } 436 437 static const struct vdpa_config_ops vp_vdpa_ops = { 438 .get_device_features = vp_vdpa_get_device_features, 439 .set_driver_features = vp_vdpa_set_driver_features, 440 .get_driver_features = vp_vdpa_get_driver_features, 441 .get_status = vp_vdpa_get_status, 442 .set_status = vp_vdpa_set_status, 443 .reset = vp_vdpa_reset, 444 .get_vq_num_max = vp_vdpa_get_vq_num_max, 445 .get_vq_state = vp_vdpa_get_vq_state, 446 .get_vq_notification = vp_vdpa_get_vq_notification, 447 .set_vq_state = vp_vdpa_set_vq_state, 448 .set_vq_cb = vp_vdpa_set_vq_cb, 449 .set_vq_ready = vp_vdpa_set_vq_ready, 450 .get_vq_ready = vp_vdpa_get_vq_ready, 451 .set_vq_num = vp_vdpa_set_vq_num, 452 .set_vq_address = vp_vdpa_set_vq_address, 453 .kick_vq = vp_vdpa_kick_vq, 454 .get_generation = vp_vdpa_get_generation, 455 .get_device_id = vp_vdpa_get_device_id, 456 .get_vendor_id = vp_vdpa_get_vendor_id, 457 .get_vq_align = vp_vdpa_get_vq_align, 458 .get_config_size = vp_vdpa_get_config_size, 459 .get_config = vp_vdpa_get_config, 460 .set_config = vp_vdpa_set_config, 461 .set_config_cb = vp_vdpa_set_config_cb, 462 .get_vq_irq = vp_vdpa_get_vq_irq, 463 }; 464 465 static void vp_vdpa_free_irq_vectors(void *data) 466 { 467 pci_free_irq_vectors(data); 468 } 469 470 static int vp_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, 471 const struct vdpa_dev_set_config *add_config) 472 { 473 struct vp_vdpa_mgmtdev *vp_vdpa_mgtdev = 474 container_of(v_mdev, struct vp_vdpa_mgmtdev, mgtdev); 475 476 struct virtio_pci_modern_device *mdev = vp_vdpa_mgtdev->mdev; 477 struct pci_dev *pdev = mdev->pci_dev; 478 struct device *dev = &pdev->dev; 479 struct vp_vdpa *vp_vdpa = NULL; 480 u64 device_features; 481 int ret, i; 482 483 vp_vdpa = vdpa_alloc_device(struct vp_vdpa, vdpa, 484 dev, &vp_vdpa_ops, 1, 1, name, false); 485 486 if (IS_ERR(vp_vdpa)) { 487 dev_err(dev, "vp_vdpa: Failed to allocate vDPA structure\n"); 488 return PTR_ERR(vp_vdpa); 489 } 490 491 vp_vdpa_mgtdev->vp_vdpa = vp_vdpa; 492 493 vp_vdpa->vdpa.dma_dev = &pdev->dev; 494 vp_vdpa->queues = vp_modern_get_num_queues(mdev); 495 vp_vdpa->mdev = mdev; 496 497 device_features = vp_modern_get_features(mdev); 498 if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) { 499 if (add_config->device_features & ~device_features) { 500 ret = -EINVAL; 501 dev_err(&pdev->dev, "Try to provision features " 502 "that are not supported by the device: " 503 "device_features 0x%llx provisioned 0x%llx\n", 504 device_features, add_config->device_features); 505 goto err; 506 } 507 device_features = add_config->device_features; 508 } 509 vp_vdpa->device_features = device_features; 510 511 ret = devm_add_action_or_reset(dev, vp_vdpa_free_irq_vectors, pdev); 512 if (ret) { 513 dev_err(&pdev->dev, 514 "Failed for adding devres for freeing irq vectors\n"); 515 goto err; 516 } 517 518 vp_vdpa->vring = devm_kcalloc(&pdev->dev, vp_vdpa->queues, 519 sizeof(*vp_vdpa->vring), 520 GFP_KERNEL); 521 if (!vp_vdpa->vring) { 522 ret = -ENOMEM; 523 dev_err(&pdev->dev, "Fail to allocate virtqueues\n"); 524 goto err; 525 } 526 527 for (i = 0; i < vp_vdpa->queues; i++) { 528 vp_vdpa->vring[i].irq = VIRTIO_MSI_NO_VECTOR; 529 vp_vdpa->vring[i].notify = 530 vp_modern_map_vq_notify(mdev, i, 531 &vp_vdpa->vring[i].notify_pa); 532 if (!vp_vdpa->vring[i].notify) { 533 ret = -EINVAL; 534 dev_warn(&pdev->dev, "Fail to map vq notify %d\n", i); 535 goto err; 536 } 537 } 538 vp_vdpa->config_irq = VIRTIO_MSI_NO_VECTOR; 539 540 vp_vdpa->vdpa.mdev = &vp_vdpa_mgtdev->mgtdev; 541 ret = _vdpa_register_device(&vp_vdpa->vdpa, vp_vdpa->queues); 542 if (ret) { 543 dev_err(&pdev->dev, "Failed to register to vdpa bus\n"); 544 goto err; 545 } 546 547 return 0; 548 549 err: 550 put_device(&vp_vdpa->vdpa.dev); 551 return ret; 552 } 553 554 static void vp_vdpa_dev_del(struct vdpa_mgmt_dev *v_mdev, 555 struct vdpa_device *dev) 556 { 557 struct vp_vdpa_mgmtdev *vp_vdpa_mgtdev = 558 container_of(v_mdev, struct vp_vdpa_mgmtdev, mgtdev); 559 560 struct vp_vdpa *vp_vdpa = vp_vdpa_mgtdev->vp_vdpa; 561 562 _vdpa_unregister_device(&vp_vdpa->vdpa); 563 vp_vdpa_mgtdev->vp_vdpa = NULL; 564 } 565 566 static const struct vdpa_mgmtdev_ops vp_vdpa_mdev_ops = { 567 .dev_add = vp_vdpa_dev_add, 568 .dev_del = vp_vdpa_dev_del, 569 }; 570 571 static int vp_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id) 572 { 573 struct vp_vdpa_mgmtdev *vp_vdpa_mgtdev = NULL; 574 struct vdpa_mgmt_dev *mgtdev; 575 struct device *dev = &pdev->dev; 576 struct virtio_pci_modern_device *mdev = NULL; 577 struct virtio_device_id *mdev_id = NULL; 578 int err; 579 580 vp_vdpa_mgtdev = kzalloc(sizeof(*vp_vdpa_mgtdev), GFP_KERNEL); 581 if (!vp_vdpa_mgtdev) 582 return -ENOMEM; 583 584 mgtdev = &vp_vdpa_mgtdev->mgtdev; 585 mgtdev->ops = &vp_vdpa_mdev_ops; 586 mgtdev->device = dev; 587 588 mdev = kzalloc(sizeof(struct virtio_pci_modern_device), GFP_KERNEL); 589 if (!mdev) { 590 err = -ENOMEM; 591 goto mdev_err; 592 } 593 594 mdev_id = kzalloc(sizeof(struct virtio_device_id), GFP_KERNEL); 595 if (!mdev_id) { 596 err = -ENOMEM; 597 goto mdev_id_err; 598 } 599 600 vp_vdpa_mgtdev->mdev = mdev; 601 mdev->pci_dev = pdev; 602 603 err = pcim_enable_device(pdev); 604 if (err) { 605 goto probe_err; 606 } 607 608 err = vp_modern_probe(mdev); 609 if (err) { 610 dev_err(&pdev->dev, "Failed to probe modern PCI device\n"); 611 goto probe_err; 612 } 613 614 mdev_id->device = mdev->id.device; 615 mdev_id->vendor = mdev->id.vendor; 616 mgtdev->id_table = mdev_id; 617 mgtdev->max_supported_vqs = vp_modern_get_num_queues(mdev); 618 mgtdev->supported_features = vp_modern_get_features(mdev); 619 mgtdev->config_attr_mask = (1 << VDPA_ATTR_DEV_FEATURES); 620 pci_set_master(pdev); 621 pci_set_drvdata(pdev, vp_vdpa_mgtdev); 622 623 err = vdpa_mgmtdev_register(mgtdev); 624 if (err) { 625 dev_err(&pdev->dev, "Failed to register vdpa mgmtdev device\n"); 626 goto register_err; 627 } 628 629 return 0; 630 631 register_err: 632 vp_modern_remove(vp_vdpa_mgtdev->mdev); 633 probe_err: 634 kfree(mdev_id); 635 mdev_id_err: 636 kfree(mdev); 637 mdev_err: 638 kfree(vp_vdpa_mgtdev); 639 return err; 640 } 641 642 static void vp_vdpa_remove(struct pci_dev *pdev) 643 { 644 struct vp_vdpa_mgmtdev *vp_vdpa_mgtdev = pci_get_drvdata(pdev); 645 struct virtio_pci_modern_device *mdev = NULL; 646 647 mdev = vp_vdpa_mgtdev->mdev; 648 vdpa_mgmtdev_unregister(&vp_vdpa_mgtdev->mgtdev); 649 vp_modern_remove(mdev); 650 kfree(vp_vdpa_mgtdev->mgtdev.id_table); 651 kfree(mdev); 652 kfree(vp_vdpa_mgtdev); 653 } 654 655 static struct pci_driver vp_vdpa_driver = { 656 .name = "vp-vdpa", 657 .id_table = NULL, /* only dynamic ids */ 658 .probe = vp_vdpa_probe, 659 .remove = vp_vdpa_remove, 660 }; 661 662 module_pci_driver(vp_vdpa_driver); 663 664 MODULE_AUTHOR("Jason Wang <jasowang@redhat.com>"); 665 MODULE_DESCRIPTION("vp-vdpa"); 666 MODULE_LICENSE("GPL"); 667 MODULE_VERSION("1"); 668