1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VIRTIO based driver for vDPA device 4 * 5 * Copyright (c) 2020, Red Hat. All rights reserved. 6 * Author: Jason Wang <jasowang@redhat.com> 7 * 8 */ 9 10 #include <linux/init.h> 11 #include <linux/module.h> 12 #include <linux/device.h> 13 #include <linux/kernel.h> 14 #include <linux/slab.h> 15 #include <linux/uuid.h> 16 #include <linux/group_cpus.h> 17 #include <linux/virtio.h> 18 #include <linux/vdpa.h> 19 #include <linux/virtio_config.h> 20 #include <linux/virtio_ring.h> 21 22 #define MOD_VERSION "0.1" 23 #define MOD_AUTHOR "Jason Wang <jasowang@redhat.com>" 24 #define MOD_DESC "vDPA bus driver for virtio devices" 25 #define MOD_LICENSE "GPL v2" 26 27 struct virtio_vdpa_device { 28 struct virtio_device vdev; 29 struct vdpa_device *vdpa; 30 u64 features; 31 32 /* The lock to protect virtqueue list */ 33 spinlock_t lock; 34 /* List of virtio_vdpa_vq_info */ 35 struct list_head virtqueues; 36 }; 37 38 struct virtio_vdpa_vq_info { 39 /* the actual virtqueue */ 40 struct virtqueue *vq; 41 42 /* the list node for the virtqueues list */ 43 struct list_head node; 44 }; 45 46 static inline struct virtio_vdpa_device * 47 to_virtio_vdpa_device(struct virtio_device *dev) 48 { 49 return container_of(dev, struct virtio_vdpa_device, vdev); 50 } 51 52 static struct vdpa_device *vd_get_vdpa(struct virtio_device *vdev) 53 { 54 return to_virtio_vdpa_device(vdev)->vdpa; 55 } 56 57 static void virtio_vdpa_get(struct virtio_device *vdev, unsigned int offset, 58 void *buf, unsigned int len) 59 { 60 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 61 62 vdpa_get_config(vdpa, offset, buf, len); 63 } 64 65 static void virtio_vdpa_set(struct virtio_device *vdev, unsigned int offset, 66 const void *buf, unsigned int len) 67 { 68 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 69 70 vdpa_set_config(vdpa, offset, buf, len); 71 } 72 73 static u32 virtio_vdpa_generation(struct virtio_device *vdev) 74 { 75 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 76 const struct vdpa_config_ops *ops = vdpa->config; 77 78 if (ops->get_generation) 79 return ops->get_generation(vdpa); 80 81 return 0; 82 } 83 84 static u8 virtio_vdpa_get_status(struct virtio_device *vdev) 85 { 86 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 87 const struct vdpa_config_ops *ops = vdpa->config; 88 89 return ops->get_status(vdpa); 90 } 91 92 static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status) 93 { 94 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 95 96 return vdpa_set_status(vdpa, status); 97 } 98 99 static void virtio_vdpa_reset(struct virtio_device *vdev) 100 { 101 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 102 103 vdpa_reset(vdpa); 104 } 105 106 static bool virtio_vdpa_notify(struct virtqueue *vq) 107 { 108 struct vdpa_device *vdpa = vd_get_vdpa(vq->vdev); 109 const struct vdpa_config_ops *ops = vdpa->config; 110 111 ops->kick_vq(vdpa, vq->index); 112 113 return true; 114 } 115 116 static irqreturn_t virtio_vdpa_config_cb(void *private) 117 { 118 struct virtio_vdpa_device *vd_dev = private; 119 120 virtio_config_changed(&vd_dev->vdev); 121 122 return IRQ_HANDLED; 123 } 124 125 static irqreturn_t virtio_vdpa_virtqueue_cb(void *private) 126 { 127 struct virtio_vdpa_vq_info *info = private; 128 129 return vring_interrupt(0, info->vq); 130 } 131 132 static struct virtqueue * 133 virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, 134 void (*callback)(struct virtqueue *vq), 135 const char *name, bool ctx) 136 { 137 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev); 138 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 139 struct device *dma_dev; 140 const struct vdpa_config_ops *ops = vdpa->config; 141 struct virtio_vdpa_vq_info *info; 142 struct vdpa_callback cb; 143 struct virtqueue *vq; 144 u64 desc_addr, driver_addr, device_addr; 145 /* Assume split virtqueue, switch to packed if necessary */ 146 struct vdpa_vq_state state = {0}; 147 unsigned long flags; 148 u32 align, max_num, min_num = 1; 149 bool may_reduce_num = true; 150 int err; 151 152 if (!name) 153 return NULL; 154 155 if (index >= vdpa->nvqs) 156 return ERR_PTR(-ENOENT); 157 158 /* Queue shouldn't already be set up. */ 159 if (ops->get_vq_ready(vdpa, index)) 160 return ERR_PTR(-ENOENT); 161 162 /* Allocate and fill out our active queue description */ 163 info = kmalloc(sizeof(*info), GFP_KERNEL); 164 if (!info) 165 return ERR_PTR(-ENOMEM); 166 167 max_num = ops->get_vq_num_max(vdpa); 168 if (max_num == 0) { 169 err = -ENOENT; 170 goto error_new_virtqueue; 171 } 172 173 if (ops->get_vq_num_min) 174 min_num = ops->get_vq_num_min(vdpa); 175 176 may_reduce_num = (max_num == min_num) ? false : true; 177 178 /* Create the vring */ 179 align = ops->get_vq_align(vdpa); 180 181 if (ops->get_vq_dma_dev) 182 dma_dev = ops->get_vq_dma_dev(vdpa, index); 183 else 184 dma_dev = vdpa_get_dma_dev(vdpa); 185 vq = vring_create_virtqueue_dma(index, max_num, align, vdev, 186 true, may_reduce_num, ctx, 187 virtio_vdpa_notify, callback, 188 name, dma_dev); 189 if (!vq) { 190 err = -ENOMEM; 191 goto error_new_virtqueue; 192 } 193 194 vq->num_max = max_num; 195 196 /* Setup virtqueue callback */ 197 cb.callback = callback ? virtio_vdpa_virtqueue_cb : NULL; 198 cb.private = info; 199 cb.trigger = NULL; 200 ops->set_vq_cb(vdpa, index, &cb); 201 ops->set_vq_num(vdpa, index, virtqueue_get_vring_size(vq)); 202 203 desc_addr = virtqueue_get_desc_addr(vq); 204 driver_addr = virtqueue_get_avail_addr(vq); 205 device_addr = virtqueue_get_used_addr(vq); 206 207 if (ops->set_vq_address(vdpa, index, 208 desc_addr, driver_addr, 209 device_addr)) { 210 err = -EINVAL; 211 goto err_vq; 212 } 213 214 /* reset virtqueue state index */ 215 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 216 struct vdpa_vq_state_packed *s = &state.packed; 217 218 s->last_avail_counter = 1; 219 s->last_avail_idx = 0; 220 s->last_used_counter = 1; 221 s->last_used_idx = 0; 222 } 223 err = ops->set_vq_state(vdpa, index, &state); 224 if (err) 225 goto err_vq; 226 227 ops->set_vq_ready(vdpa, index, 1); 228 229 vq->priv = info; 230 info->vq = vq; 231 232 spin_lock_irqsave(&vd_dev->lock, flags); 233 list_add(&info->node, &vd_dev->virtqueues); 234 spin_unlock_irqrestore(&vd_dev->lock, flags); 235 236 return vq; 237 238 err_vq: 239 vring_del_virtqueue(vq); 240 error_new_virtqueue: 241 ops->set_vq_ready(vdpa, index, 0); 242 /* VDPA driver should make sure vq is stopeed here */ 243 WARN_ON(ops->get_vq_ready(vdpa, index)); 244 kfree(info); 245 return ERR_PTR(err); 246 } 247 248 static void virtio_vdpa_del_vq(struct virtqueue *vq) 249 { 250 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev); 251 struct vdpa_device *vdpa = vd_dev->vdpa; 252 const struct vdpa_config_ops *ops = vdpa->config; 253 struct virtio_vdpa_vq_info *info = vq->priv; 254 unsigned int index = vq->index; 255 unsigned long flags; 256 257 spin_lock_irqsave(&vd_dev->lock, flags); 258 list_del(&info->node); 259 spin_unlock_irqrestore(&vd_dev->lock, flags); 260 261 /* Select and deactivate the queue (best effort) */ 262 ops->set_vq_ready(vdpa, index, 0); 263 264 vring_del_virtqueue(vq); 265 266 kfree(info); 267 } 268 269 static void virtio_vdpa_del_vqs(struct virtio_device *vdev) 270 { 271 struct virtqueue *vq, *n; 272 273 list_for_each_entry_safe(vq, n, &vdev->vqs, list) 274 virtio_vdpa_del_vq(vq); 275 } 276 277 static void default_calc_sets(struct irq_affinity *affd, unsigned int affvecs) 278 { 279 affd->nr_sets = 1; 280 affd->set_size[0] = affvecs; 281 } 282 283 static struct cpumask * 284 create_affinity_masks(unsigned int nvecs, struct irq_affinity *affd) 285 { 286 unsigned int affvecs = 0, curvec, usedvecs, i; 287 struct cpumask *masks = NULL; 288 289 if (nvecs > affd->pre_vectors + affd->post_vectors) 290 affvecs = nvecs - affd->pre_vectors - affd->post_vectors; 291 292 if (!affd->calc_sets) 293 affd->calc_sets = default_calc_sets; 294 295 affd->calc_sets(affd, affvecs); 296 297 if (!affvecs) 298 return NULL; 299 300 masks = kcalloc(nvecs, sizeof(*masks), GFP_KERNEL); 301 if (!masks) 302 return NULL; 303 304 /* Fill out vectors at the beginning that don't need affinity */ 305 for (curvec = 0; curvec < affd->pre_vectors; curvec++) 306 cpumask_setall(&masks[curvec]); 307 308 for (i = 0, usedvecs = 0; i < affd->nr_sets; i++) { 309 unsigned int this_vecs = affd->set_size[i]; 310 int j; 311 struct cpumask *result = group_cpus_evenly(this_vecs); 312 313 if (!result) { 314 kfree(masks); 315 return NULL; 316 } 317 318 for (j = 0; j < this_vecs; j++) 319 cpumask_copy(&masks[curvec + j], &result[j]); 320 kfree(result); 321 322 curvec += this_vecs; 323 usedvecs += this_vecs; 324 } 325 326 /* Fill out vectors at the end that don't need affinity */ 327 if (usedvecs >= affvecs) 328 curvec = affd->pre_vectors + affvecs; 329 else 330 curvec = affd->pre_vectors + usedvecs; 331 for (; curvec < nvecs; curvec++) 332 cpumask_setall(&masks[curvec]); 333 334 return masks; 335 } 336 337 static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs, 338 struct virtqueue *vqs[], 339 vq_callback_t *callbacks[], 340 const char * const names[], 341 const bool *ctx, 342 struct irq_affinity *desc) 343 { 344 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev); 345 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 346 const struct vdpa_config_ops *ops = vdpa->config; 347 struct irq_affinity default_affd = { 0 }; 348 struct cpumask *masks; 349 struct vdpa_callback cb; 350 int i, err, queue_idx = 0; 351 352 masks = create_affinity_masks(nvqs, desc ? desc : &default_affd); 353 if (!masks) 354 return -ENOMEM; 355 356 for (i = 0; i < nvqs; ++i) { 357 if (!names[i]) { 358 vqs[i] = NULL; 359 continue; 360 } 361 362 vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++, 363 callbacks[i], names[i], ctx ? 364 ctx[i] : false); 365 if (IS_ERR(vqs[i])) { 366 err = PTR_ERR(vqs[i]); 367 goto err_setup_vq; 368 } 369 ops->set_vq_affinity(vdpa, i, &masks[i]); 370 } 371 372 cb.callback = virtio_vdpa_config_cb; 373 cb.private = vd_dev; 374 ops->set_config_cb(vdpa, &cb); 375 376 return 0; 377 378 err_setup_vq: 379 virtio_vdpa_del_vqs(vdev); 380 return err; 381 } 382 383 static u64 virtio_vdpa_get_features(struct virtio_device *vdev) 384 { 385 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 386 const struct vdpa_config_ops *ops = vdpa->config; 387 388 return ops->get_device_features(vdpa); 389 } 390 391 static int virtio_vdpa_finalize_features(struct virtio_device *vdev) 392 { 393 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 394 395 /* Give virtio_ring a chance to accept features. */ 396 vring_transport_features(vdev); 397 398 return vdpa_set_features(vdpa, vdev->features); 399 } 400 401 static const char *virtio_vdpa_bus_name(struct virtio_device *vdev) 402 { 403 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev); 404 struct vdpa_device *vdpa = vd_dev->vdpa; 405 406 return dev_name(&vdpa->dev); 407 } 408 409 static int virtio_vdpa_set_vq_affinity(struct virtqueue *vq, 410 const struct cpumask *cpu_mask) 411 { 412 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev); 413 struct vdpa_device *vdpa = vd_dev->vdpa; 414 const struct vdpa_config_ops *ops = vdpa->config; 415 unsigned int index = vq->index; 416 417 if (ops->set_vq_affinity) 418 return ops->set_vq_affinity(vdpa, index, cpu_mask); 419 420 return 0; 421 } 422 423 static const struct cpumask * 424 virtio_vdpa_get_vq_affinity(struct virtio_device *vdev, int index) 425 { 426 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 427 const struct vdpa_config_ops *ops = vdpa->config; 428 429 if (ops->get_vq_affinity) 430 return ops->get_vq_affinity(vdpa, index); 431 432 return NULL; 433 } 434 435 static const struct virtio_config_ops virtio_vdpa_config_ops = { 436 .get = virtio_vdpa_get, 437 .set = virtio_vdpa_set, 438 .generation = virtio_vdpa_generation, 439 .get_status = virtio_vdpa_get_status, 440 .set_status = virtio_vdpa_set_status, 441 .reset = virtio_vdpa_reset, 442 .find_vqs = virtio_vdpa_find_vqs, 443 .del_vqs = virtio_vdpa_del_vqs, 444 .get_features = virtio_vdpa_get_features, 445 .finalize_features = virtio_vdpa_finalize_features, 446 .bus_name = virtio_vdpa_bus_name, 447 .set_vq_affinity = virtio_vdpa_set_vq_affinity, 448 .get_vq_affinity = virtio_vdpa_get_vq_affinity, 449 }; 450 451 static void virtio_vdpa_release_dev(struct device *_d) 452 { 453 struct virtio_device *vdev = 454 container_of(_d, struct virtio_device, dev); 455 struct virtio_vdpa_device *vd_dev = 456 container_of(vdev, struct virtio_vdpa_device, vdev); 457 458 kfree(vd_dev); 459 } 460 461 static int virtio_vdpa_probe(struct vdpa_device *vdpa) 462 { 463 const struct vdpa_config_ops *ops = vdpa->config; 464 struct virtio_vdpa_device *vd_dev, *reg_dev = NULL; 465 int ret = -EINVAL; 466 467 vd_dev = kzalloc(sizeof(*vd_dev), GFP_KERNEL); 468 if (!vd_dev) 469 return -ENOMEM; 470 471 vd_dev->vdev.dev.parent = vdpa_get_dma_dev(vdpa); 472 vd_dev->vdev.dev.release = virtio_vdpa_release_dev; 473 vd_dev->vdev.config = &virtio_vdpa_config_ops; 474 vd_dev->vdpa = vdpa; 475 INIT_LIST_HEAD(&vd_dev->virtqueues); 476 spin_lock_init(&vd_dev->lock); 477 478 vd_dev->vdev.id.device = ops->get_device_id(vdpa); 479 if (vd_dev->vdev.id.device == 0) 480 goto err; 481 482 vd_dev->vdev.id.vendor = ops->get_vendor_id(vdpa); 483 ret = register_virtio_device(&vd_dev->vdev); 484 reg_dev = vd_dev; 485 if (ret) 486 goto err; 487 488 vdpa_set_drvdata(vdpa, vd_dev); 489 490 return 0; 491 492 err: 493 if (reg_dev) 494 put_device(&vd_dev->vdev.dev); 495 else 496 kfree(vd_dev); 497 return ret; 498 } 499 500 static void virtio_vdpa_remove(struct vdpa_device *vdpa) 501 { 502 struct virtio_vdpa_device *vd_dev = vdpa_get_drvdata(vdpa); 503 504 unregister_virtio_device(&vd_dev->vdev); 505 } 506 507 static struct vdpa_driver virtio_vdpa_driver = { 508 .driver = { 509 .name = "virtio_vdpa", 510 }, 511 .probe = virtio_vdpa_probe, 512 .remove = virtio_vdpa_remove, 513 }; 514 515 module_vdpa_driver(virtio_vdpa_driver); 516 517 MODULE_VERSION(MOD_VERSION); 518 MODULE_LICENSE(MOD_LICENSE); 519 MODULE_AUTHOR(MOD_AUTHOR); 520 MODULE_DESCRIPTION(MOD_DESC); 521