1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VIRTIO based driver for vDPA device 4 * 5 * Copyright (c) 2020, Red Hat. All rights reserved. 6 * Author: Jason Wang <jasowang@redhat.com> 7 * 8 */ 9 10 #include <linux/init.h> 11 #include <linux/module.h> 12 #include <linux/device.h> 13 #include <linux/kernel.h> 14 #include <linux/slab.h> 15 #include <linux/uuid.h> 16 #include <linux/group_cpus.h> 17 #include <linux/virtio.h> 18 #include <linux/vdpa.h> 19 #include <linux/virtio_config.h> 20 #include <linux/virtio_ring.h> 21 22 #define MOD_VERSION "0.1" 23 #define MOD_AUTHOR "Jason Wang <jasowang@redhat.com>" 24 #define MOD_DESC "vDPA bus driver for virtio devices" 25 #define MOD_LICENSE "GPL v2" 26 27 struct virtio_vdpa_device { 28 struct virtio_device vdev; 29 struct vdpa_device *vdpa; 30 u64 features; 31 32 /* The lock to protect virtqueue list */ 33 spinlock_t lock; 34 /* List of virtio_vdpa_vq_info */ 35 struct list_head virtqueues; 36 }; 37 38 struct virtio_vdpa_vq_info { 39 /* the actual virtqueue */ 40 struct virtqueue *vq; 41 42 /* the list node for the virtqueues list */ 43 struct list_head node; 44 }; 45 46 static inline struct virtio_vdpa_device * 47 to_virtio_vdpa_device(struct virtio_device *dev) 48 { 49 return container_of(dev, struct virtio_vdpa_device, vdev); 50 } 51 52 static struct vdpa_device *vd_get_vdpa(struct virtio_device *vdev) 53 { 54 return to_virtio_vdpa_device(vdev)->vdpa; 55 } 56 57 static void virtio_vdpa_get(struct virtio_device *vdev, unsigned int offset, 58 void *buf, unsigned int len) 59 { 60 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 61 62 vdpa_get_config(vdpa, offset, buf, len); 63 } 64 65 static void virtio_vdpa_set(struct virtio_device *vdev, unsigned int offset, 66 const void *buf, unsigned int len) 67 { 68 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 69 70 vdpa_set_config(vdpa, offset, buf, len); 71 } 72 73 static u32 virtio_vdpa_generation(struct virtio_device *vdev) 74 { 75 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 76 const struct vdpa_config_ops *ops = vdpa->config; 77 78 if (ops->get_generation) 79 return ops->get_generation(vdpa); 80 81 return 0; 82 } 83 84 static u8 virtio_vdpa_get_status(struct virtio_device *vdev) 85 { 86 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 87 const struct vdpa_config_ops *ops = vdpa->config; 88 89 return ops->get_status(vdpa); 90 } 91 92 static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status) 93 { 94 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 95 96 return vdpa_set_status(vdpa, status); 97 } 98 99 static void virtio_vdpa_reset(struct virtio_device *vdev) 100 { 101 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 102 103 vdpa_reset(vdpa, 0); 104 } 105 106 static bool virtio_vdpa_notify(struct virtqueue *vq) 107 { 108 struct vdpa_device *vdpa = vd_get_vdpa(vq->vdev); 109 const struct vdpa_config_ops *ops = vdpa->config; 110 111 ops->kick_vq(vdpa, vq->index); 112 113 return true; 114 } 115 116 static bool virtio_vdpa_notify_with_data(struct virtqueue *vq) 117 { 118 struct vdpa_device *vdpa = vd_get_vdpa(vq->vdev); 119 const struct vdpa_config_ops *ops = vdpa->config; 120 u32 data = vring_notification_data(vq); 121 122 ops->kick_vq_with_data(vdpa, data); 123 124 return true; 125 } 126 127 static irqreturn_t virtio_vdpa_config_cb(void *private) 128 { 129 struct virtio_vdpa_device *vd_dev = private; 130 131 virtio_config_changed(&vd_dev->vdev); 132 133 return IRQ_HANDLED; 134 } 135 136 static irqreturn_t virtio_vdpa_virtqueue_cb(void *private) 137 { 138 struct virtio_vdpa_vq_info *info = private; 139 140 return vring_interrupt(0, info->vq); 141 } 142 143 static struct virtqueue * 144 virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, 145 void (*callback)(struct virtqueue *vq), 146 const char *name, bool ctx) 147 { 148 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev); 149 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 150 struct device *dma_dev; 151 const struct vdpa_config_ops *ops = vdpa->config; 152 struct virtio_vdpa_vq_info *info; 153 bool (*notify)(struct virtqueue *vq) = virtio_vdpa_notify; 154 struct vdpa_callback cb; 155 struct virtqueue *vq; 156 u64 desc_addr, driver_addr, device_addr; 157 /* Assume split virtqueue, switch to packed if necessary */ 158 struct vdpa_vq_state state = {0}; 159 unsigned long flags; 160 u32 align, max_num, min_num = 1; 161 bool may_reduce_num = true; 162 int err; 163 164 if (!name) 165 return NULL; 166 167 if (index >= vdpa->nvqs) 168 return ERR_PTR(-ENOENT); 169 170 /* We cannot accept VIRTIO_F_NOTIFICATION_DATA without kick_vq_with_data */ 171 if (__virtio_test_bit(vdev, VIRTIO_F_NOTIFICATION_DATA)) { 172 if (ops->kick_vq_with_data) 173 notify = virtio_vdpa_notify_with_data; 174 else 175 __virtio_clear_bit(vdev, VIRTIO_F_NOTIFICATION_DATA); 176 } 177 178 /* Queue shouldn't already be set up. */ 179 if (ops->get_vq_ready(vdpa, index)) 180 return ERR_PTR(-ENOENT); 181 182 /* Allocate and fill out our active queue description */ 183 info = kmalloc(sizeof(*info), GFP_KERNEL); 184 if (!info) 185 return ERR_PTR(-ENOMEM); 186 if (ops->get_vq_size) 187 max_num = ops->get_vq_size(vdpa, index); 188 else 189 max_num = ops->get_vq_num_max(vdpa); 190 191 if (max_num == 0) { 192 err = -ENOENT; 193 goto error_new_virtqueue; 194 } 195 196 if (ops->get_vq_num_min) 197 min_num = ops->get_vq_num_min(vdpa); 198 199 may_reduce_num = (max_num == min_num) ? false : true; 200 201 /* Create the vring */ 202 align = ops->get_vq_align(vdpa); 203 204 if (ops->get_vq_dma_dev) 205 dma_dev = ops->get_vq_dma_dev(vdpa, index); 206 else 207 dma_dev = vdpa_get_dma_dev(vdpa); 208 vq = vring_create_virtqueue_dma(index, max_num, align, vdev, 209 true, may_reduce_num, ctx, 210 notify, callback, name, dma_dev); 211 if (!vq) { 212 err = -ENOMEM; 213 goto error_new_virtqueue; 214 } 215 216 vq->num_max = max_num; 217 218 /* Setup virtqueue callback */ 219 cb.callback = callback ? virtio_vdpa_virtqueue_cb : NULL; 220 cb.private = info; 221 cb.trigger = NULL; 222 ops->set_vq_cb(vdpa, index, &cb); 223 ops->set_vq_num(vdpa, index, virtqueue_get_vring_size(vq)); 224 225 desc_addr = virtqueue_get_desc_addr(vq); 226 driver_addr = virtqueue_get_avail_addr(vq); 227 device_addr = virtqueue_get_used_addr(vq); 228 229 if (ops->set_vq_address(vdpa, index, 230 desc_addr, driver_addr, 231 device_addr)) { 232 err = -EINVAL; 233 goto err_vq; 234 } 235 236 /* reset virtqueue state index */ 237 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 238 struct vdpa_vq_state_packed *s = &state.packed; 239 240 s->last_avail_counter = 1; 241 s->last_avail_idx = 0; 242 s->last_used_counter = 1; 243 s->last_used_idx = 0; 244 } 245 err = ops->set_vq_state(vdpa, index, &state); 246 if (err) 247 goto err_vq; 248 249 ops->set_vq_ready(vdpa, index, 1); 250 251 vq->priv = info; 252 info->vq = vq; 253 254 spin_lock_irqsave(&vd_dev->lock, flags); 255 list_add(&info->node, &vd_dev->virtqueues); 256 spin_unlock_irqrestore(&vd_dev->lock, flags); 257 258 return vq; 259 260 err_vq: 261 vring_del_virtqueue(vq); 262 error_new_virtqueue: 263 ops->set_vq_ready(vdpa, index, 0); 264 /* VDPA driver should make sure vq is stopeed here */ 265 WARN_ON(ops->get_vq_ready(vdpa, index)); 266 kfree(info); 267 return ERR_PTR(err); 268 } 269 270 static void virtio_vdpa_del_vq(struct virtqueue *vq) 271 { 272 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev); 273 struct vdpa_device *vdpa = vd_dev->vdpa; 274 const struct vdpa_config_ops *ops = vdpa->config; 275 struct virtio_vdpa_vq_info *info = vq->priv; 276 unsigned int index = vq->index; 277 unsigned long flags; 278 279 spin_lock_irqsave(&vd_dev->lock, flags); 280 list_del(&info->node); 281 spin_unlock_irqrestore(&vd_dev->lock, flags); 282 283 /* Select and deactivate the queue (best effort) */ 284 ops->set_vq_ready(vdpa, index, 0); 285 286 vring_del_virtqueue(vq); 287 288 kfree(info); 289 } 290 291 static void virtio_vdpa_del_vqs(struct virtio_device *vdev) 292 { 293 struct virtqueue *vq, *n; 294 295 list_for_each_entry_safe(vq, n, &vdev->vqs, list) 296 virtio_vdpa_del_vq(vq); 297 } 298 299 static void default_calc_sets(struct irq_affinity *affd, unsigned int affvecs) 300 { 301 affd->nr_sets = 1; 302 affd->set_size[0] = affvecs; 303 } 304 305 static struct cpumask * 306 create_affinity_masks(unsigned int nvecs, struct irq_affinity *affd) 307 { 308 unsigned int affvecs = 0, curvec, usedvecs, i; 309 struct cpumask *masks = NULL; 310 311 if (nvecs > affd->pre_vectors + affd->post_vectors) 312 affvecs = nvecs - affd->pre_vectors - affd->post_vectors; 313 314 if (!affd->calc_sets) 315 affd->calc_sets = default_calc_sets; 316 317 affd->calc_sets(affd, affvecs); 318 319 if (!affvecs) 320 return NULL; 321 322 masks = kcalloc(nvecs, sizeof(*masks), GFP_KERNEL); 323 if (!masks) 324 return NULL; 325 326 /* Fill out vectors at the beginning that don't need affinity */ 327 for (curvec = 0; curvec < affd->pre_vectors; curvec++) 328 cpumask_setall(&masks[curvec]); 329 330 for (i = 0, usedvecs = 0; i < affd->nr_sets; i++) { 331 unsigned int this_vecs = affd->set_size[i]; 332 int j; 333 struct cpumask *result = group_cpus_evenly(this_vecs); 334 335 if (!result) { 336 kfree(masks); 337 return NULL; 338 } 339 340 for (j = 0; j < this_vecs; j++) 341 cpumask_copy(&masks[curvec + j], &result[j]); 342 kfree(result); 343 344 curvec += this_vecs; 345 usedvecs += this_vecs; 346 } 347 348 /* Fill out vectors at the end that don't need affinity */ 349 if (usedvecs >= affvecs) 350 curvec = affd->pre_vectors + affvecs; 351 else 352 curvec = affd->pre_vectors + usedvecs; 353 for (; curvec < nvecs; curvec++) 354 cpumask_setall(&masks[curvec]); 355 356 return masks; 357 } 358 359 static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs, 360 struct virtqueue *vqs[], 361 struct virtqueue_info vqs_info[], 362 struct irq_affinity *desc) 363 { 364 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev); 365 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 366 const struct vdpa_config_ops *ops = vdpa->config; 367 struct irq_affinity default_affd = { 0 }; 368 struct cpumask *masks; 369 struct vdpa_callback cb; 370 bool has_affinity = desc && ops->set_vq_affinity; 371 int i, err, queue_idx = 0; 372 373 if (has_affinity) { 374 masks = create_affinity_masks(nvqs, desc ? desc : &default_affd); 375 if (!masks) 376 return -ENOMEM; 377 } 378 379 for (i = 0; i < nvqs; ++i) { 380 struct virtqueue_info *vqi = &vqs_info[i]; 381 382 if (!vqi->name) { 383 vqs[i] = NULL; 384 continue; 385 } 386 387 vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++, vqi->callback, 388 vqi->name, vqi->ctx); 389 if (IS_ERR(vqs[i])) { 390 err = PTR_ERR(vqs[i]); 391 goto err_setup_vq; 392 } 393 394 if (has_affinity) 395 ops->set_vq_affinity(vdpa, i, &masks[i]); 396 } 397 398 cb.callback = virtio_vdpa_config_cb; 399 cb.private = vd_dev; 400 ops->set_config_cb(vdpa, &cb); 401 if (has_affinity) 402 kfree(masks); 403 404 return 0; 405 406 err_setup_vq: 407 virtio_vdpa_del_vqs(vdev); 408 if (has_affinity) 409 kfree(masks); 410 return err; 411 } 412 413 static u64 virtio_vdpa_get_features(struct virtio_device *vdev) 414 { 415 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 416 const struct vdpa_config_ops *ops = vdpa->config; 417 418 return ops->get_device_features(vdpa); 419 } 420 421 static int virtio_vdpa_finalize_features(struct virtio_device *vdev) 422 { 423 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 424 425 /* Give virtio_ring a chance to accept features. */ 426 vring_transport_features(vdev); 427 428 return vdpa_set_features(vdpa, vdev->features); 429 } 430 431 static const char *virtio_vdpa_bus_name(struct virtio_device *vdev) 432 { 433 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev); 434 struct vdpa_device *vdpa = vd_dev->vdpa; 435 436 return dev_name(&vdpa->dev); 437 } 438 439 static int virtio_vdpa_set_vq_affinity(struct virtqueue *vq, 440 const struct cpumask *cpu_mask) 441 { 442 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev); 443 struct vdpa_device *vdpa = vd_dev->vdpa; 444 const struct vdpa_config_ops *ops = vdpa->config; 445 unsigned int index = vq->index; 446 447 if (ops->set_vq_affinity) 448 return ops->set_vq_affinity(vdpa, index, cpu_mask); 449 450 return 0; 451 } 452 453 static const struct cpumask * 454 virtio_vdpa_get_vq_affinity(struct virtio_device *vdev, int index) 455 { 456 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 457 const struct vdpa_config_ops *ops = vdpa->config; 458 459 if (ops->get_vq_affinity) 460 return ops->get_vq_affinity(vdpa, index); 461 462 return NULL; 463 } 464 465 static const struct virtio_config_ops virtio_vdpa_config_ops = { 466 .get = virtio_vdpa_get, 467 .set = virtio_vdpa_set, 468 .generation = virtio_vdpa_generation, 469 .get_status = virtio_vdpa_get_status, 470 .set_status = virtio_vdpa_set_status, 471 .reset = virtio_vdpa_reset, 472 .find_vqs = virtio_vdpa_find_vqs, 473 .del_vqs = virtio_vdpa_del_vqs, 474 .get_features = virtio_vdpa_get_features, 475 .finalize_features = virtio_vdpa_finalize_features, 476 .bus_name = virtio_vdpa_bus_name, 477 .set_vq_affinity = virtio_vdpa_set_vq_affinity, 478 .get_vq_affinity = virtio_vdpa_get_vq_affinity, 479 }; 480 481 static void virtio_vdpa_release_dev(struct device *_d) 482 { 483 struct virtio_device *vdev = 484 container_of(_d, struct virtio_device, dev); 485 struct virtio_vdpa_device *vd_dev = 486 container_of(vdev, struct virtio_vdpa_device, vdev); 487 488 kfree(vd_dev); 489 } 490 491 static int virtio_vdpa_probe(struct vdpa_device *vdpa) 492 { 493 const struct vdpa_config_ops *ops = vdpa->config; 494 struct virtio_vdpa_device *vd_dev, *reg_dev = NULL; 495 int ret = -EINVAL; 496 497 vd_dev = kzalloc(sizeof(*vd_dev), GFP_KERNEL); 498 if (!vd_dev) 499 return -ENOMEM; 500 501 vd_dev->vdev.dev.parent = vdpa_get_dma_dev(vdpa); 502 vd_dev->vdev.dev.release = virtio_vdpa_release_dev; 503 vd_dev->vdev.config = &virtio_vdpa_config_ops; 504 vd_dev->vdpa = vdpa; 505 INIT_LIST_HEAD(&vd_dev->virtqueues); 506 spin_lock_init(&vd_dev->lock); 507 508 vd_dev->vdev.id.device = ops->get_device_id(vdpa); 509 if (vd_dev->vdev.id.device == 0) 510 goto err; 511 512 vd_dev->vdev.id.vendor = ops->get_vendor_id(vdpa); 513 ret = register_virtio_device(&vd_dev->vdev); 514 reg_dev = vd_dev; 515 if (ret) 516 goto err; 517 518 vdpa_set_drvdata(vdpa, vd_dev); 519 520 return 0; 521 522 err: 523 if (reg_dev) 524 put_device(&vd_dev->vdev.dev); 525 else 526 kfree(vd_dev); 527 return ret; 528 } 529 530 static void virtio_vdpa_remove(struct vdpa_device *vdpa) 531 { 532 struct virtio_vdpa_device *vd_dev = vdpa_get_drvdata(vdpa); 533 534 unregister_virtio_device(&vd_dev->vdev); 535 } 536 537 static struct vdpa_driver virtio_vdpa_driver = { 538 .driver = { 539 .name = "virtio_vdpa", 540 }, 541 .probe = virtio_vdpa_probe, 542 .remove = virtio_vdpa_remove, 543 }; 544 545 module_vdpa_driver(virtio_vdpa_driver); 546 547 MODULE_VERSION(MOD_VERSION); 548 MODULE_LICENSE(MOD_LICENSE); 549 MODULE_AUTHOR(MOD_AUTHOR); 550 MODULE_DESCRIPTION(MOD_DESC); 551