1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VIRTIO based driver for vDPA device 4 * 5 * Copyright (c) 2020, Red Hat. All rights reserved. 6 * Author: Jason Wang <jasowang@redhat.com> 7 * 8 */ 9 10 #include <linux/init.h> 11 #include <linux/module.h> 12 #include <linux/device.h> 13 #include <linux/kernel.h> 14 #include <linux/slab.h> 15 #include <linux/uuid.h> 16 #include <linux/group_cpus.h> 17 #include <linux/virtio.h> 18 #include <linux/vdpa.h> 19 #include <linux/virtio_config.h> 20 #include <linux/virtio_ring.h> 21 22 #define MOD_VERSION "0.1" 23 #define MOD_AUTHOR "Jason Wang <jasowang@redhat.com>" 24 #define MOD_DESC "vDPA bus driver for virtio devices" 25 #define MOD_LICENSE "GPL v2" 26 27 struct virtio_vdpa_device { 28 struct virtio_device vdev; 29 struct vdpa_device *vdpa; 30 u64 features; 31 }; 32 33 static inline struct virtio_vdpa_device * 34 to_virtio_vdpa_device(struct virtio_device *dev) 35 { 36 return container_of(dev, struct virtio_vdpa_device, vdev); 37 } 38 39 static struct vdpa_device *vd_get_vdpa(struct virtio_device *vdev) 40 { 41 return to_virtio_vdpa_device(vdev)->vdpa; 42 } 43 44 static void virtio_vdpa_get(struct virtio_device *vdev, unsigned int offset, 45 void *buf, unsigned int len) 46 { 47 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 48 49 vdpa_get_config(vdpa, offset, buf, len); 50 } 51 52 static void virtio_vdpa_set(struct virtio_device *vdev, unsigned int offset, 53 const void *buf, unsigned int len) 54 { 55 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 56 57 vdpa_set_config(vdpa, offset, buf, len); 58 } 59 60 static u32 virtio_vdpa_generation(struct virtio_device *vdev) 61 { 62 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 63 const struct vdpa_config_ops *ops = vdpa->config; 64 65 if (ops->get_generation) 66 return ops->get_generation(vdpa); 67 68 return 0; 69 } 70 71 static u8 virtio_vdpa_get_status(struct virtio_device *vdev) 72 { 73 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 74 const struct vdpa_config_ops *ops = vdpa->config; 75 76 return ops->get_status(vdpa); 77 } 78 79 static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status) 80 { 81 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 82 83 return vdpa_set_status(vdpa, status); 84 } 85 86 static void virtio_vdpa_reset(struct virtio_device *vdev) 87 { 88 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 89 90 vdpa_reset(vdpa, 0); 91 } 92 93 static bool virtio_vdpa_notify(struct virtqueue *vq) 94 { 95 struct vdpa_device *vdpa = vd_get_vdpa(vq->vdev); 96 const struct vdpa_config_ops *ops = vdpa->config; 97 98 ops->kick_vq(vdpa, vq->index); 99 100 return true; 101 } 102 103 static bool virtio_vdpa_notify_with_data(struct virtqueue *vq) 104 { 105 struct vdpa_device *vdpa = vd_get_vdpa(vq->vdev); 106 const struct vdpa_config_ops *ops = vdpa->config; 107 u32 data = vring_notification_data(vq); 108 109 ops->kick_vq_with_data(vdpa, data); 110 111 return true; 112 } 113 114 static irqreturn_t virtio_vdpa_config_cb(void *private) 115 { 116 struct virtio_vdpa_device *vd_dev = private; 117 118 virtio_config_changed(&vd_dev->vdev); 119 120 return IRQ_HANDLED; 121 } 122 123 static irqreturn_t virtio_vdpa_virtqueue_cb(void *private) 124 { 125 struct virtqueue *vq = private; 126 127 return vring_interrupt(0, vq); 128 } 129 130 static struct virtqueue * 131 virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, 132 void (*callback)(struct virtqueue *vq), 133 const char *name, bool ctx) 134 { 135 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 136 const struct vdpa_config_ops *ops = vdpa->config; 137 bool (*notify)(struct virtqueue *vq) = virtio_vdpa_notify; 138 struct vdpa_callback cb; 139 struct virtqueue *vq; 140 u64 desc_addr, driver_addr, device_addr; 141 union virtio_map map = {0}; 142 /* Assume split virtqueue, switch to packed if necessary */ 143 struct vdpa_vq_state state = {0}; 144 u32 align, max_num, min_num = 1; 145 bool may_reduce_num = true; 146 int err; 147 148 if (!name) 149 return NULL; 150 151 if (index >= vdpa->nvqs) 152 return ERR_PTR(-ENOENT); 153 154 /* We cannot accept VIRTIO_F_NOTIFICATION_DATA without kick_vq_with_data */ 155 if (__virtio_test_bit(vdev, VIRTIO_F_NOTIFICATION_DATA)) { 156 if (ops->kick_vq_with_data) 157 notify = virtio_vdpa_notify_with_data; 158 else 159 __virtio_clear_bit(vdev, VIRTIO_F_NOTIFICATION_DATA); 160 } 161 162 /* Queue shouldn't already be set up. */ 163 if (ops->get_vq_ready(vdpa, index)) 164 return ERR_PTR(-ENOENT); 165 166 if (ops->get_vq_size) 167 max_num = ops->get_vq_size(vdpa, index); 168 else 169 max_num = ops->get_vq_num_max(vdpa); 170 171 if (max_num == 0) { 172 err = -ENOENT; 173 goto error_new_virtqueue; 174 } 175 176 if (ops->get_vq_num_min) 177 min_num = ops->get_vq_num_min(vdpa); 178 179 may_reduce_num = (max_num != min_num); 180 181 /* Create the vring */ 182 align = ops->get_vq_align(vdpa); 183 184 if (ops->get_vq_map) 185 map = ops->get_vq_map(vdpa, index); 186 else 187 map = vdpa_get_map(vdpa); 188 189 vq = vring_create_virtqueue_map(index, max_num, align, vdev, 190 true, may_reduce_num, ctx, 191 notify, callback, name, map); 192 if (!vq) { 193 err = -ENOMEM; 194 goto error_new_virtqueue; 195 } 196 197 if (index == 0) 198 vdev->vmap = map; 199 200 vq->num_max = max_num; 201 202 /* Setup virtqueue callback */ 203 cb.callback = callback ? virtio_vdpa_virtqueue_cb : NULL; 204 cb.private = vq; 205 cb.trigger = NULL; 206 ops->set_vq_cb(vdpa, index, &cb); 207 ops->set_vq_num(vdpa, index, virtqueue_get_vring_size(vq)); 208 209 desc_addr = virtqueue_get_desc_addr(vq); 210 driver_addr = virtqueue_get_avail_addr(vq); 211 device_addr = virtqueue_get_used_addr(vq); 212 213 if (ops->set_vq_address(vdpa, index, 214 desc_addr, driver_addr, 215 device_addr)) { 216 err = -EINVAL; 217 goto err_vq; 218 } 219 220 /* reset virtqueue state index */ 221 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 222 struct vdpa_vq_state_packed *s = &state.packed; 223 224 s->last_avail_counter = 1; 225 s->last_avail_idx = 0; 226 s->last_used_counter = 1; 227 s->last_used_idx = 0; 228 } 229 err = ops->set_vq_state(vdpa, index, &state); 230 if (err) 231 goto err_vq; 232 233 ops->set_vq_ready(vdpa, index, 1); 234 235 return vq; 236 237 err_vq: 238 vring_del_virtqueue(vq); 239 error_new_virtqueue: 240 ops->set_vq_ready(vdpa, index, 0); 241 /* VDPA driver should make sure vq is stopeed here */ 242 WARN_ON(ops->get_vq_ready(vdpa, index)); 243 return ERR_PTR(err); 244 } 245 246 static void virtio_vdpa_del_vq(struct virtqueue *vq) 247 { 248 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev); 249 struct vdpa_device *vdpa = vd_dev->vdpa; 250 const struct vdpa_config_ops *ops = vdpa->config; 251 unsigned int index = vq->index; 252 253 /* Select and deactivate the queue (best effort) */ 254 ops->set_vq_ready(vdpa, index, 0); 255 256 vring_del_virtqueue(vq); 257 } 258 259 static void virtio_vdpa_del_vqs(struct virtio_device *vdev) 260 { 261 struct virtqueue *vq, *n; 262 263 list_for_each_entry_safe(vq, n, &vdev->vqs, list) 264 virtio_vdpa_del_vq(vq); 265 } 266 267 static void default_calc_sets(struct irq_affinity *affd, unsigned int affvecs) 268 { 269 affd->nr_sets = 1; 270 affd->set_size[0] = affvecs; 271 } 272 273 static struct cpumask * 274 create_affinity_masks(unsigned int nvecs, struct irq_affinity *affd) 275 { 276 unsigned int affvecs = 0, curvec, usedvecs, i; 277 struct cpumask *masks = NULL; 278 279 if (nvecs > affd->pre_vectors + affd->post_vectors) 280 affvecs = nvecs - affd->pre_vectors - affd->post_vectors; 281 282 if (!affd->calc_sets) 283 affd->calc_sets = default_calc_sets; 284 285 affd->calc_sets(affd, affvecs); 286 287 if (!affvecs) 288 return NULL; 289 290 masks = kcalloc(nvecs, sizeof(*masks), GFP_KERNEL); 291 if (!masks) 292 return NULL; 293 294 /* Fill out vectors at the beginning that don't need affinity */ 295 for (curvec = 0; curvec < affd->pre_vectors; curvec++) 296 cpumask_setall(&masks[curvec]); 297 298 for (i = 0, usedvecs = 0; i < affd->nr_sets; i++) { 299 unsigned int this_vecs = affd->set_size[i]; 300 unsigned int nr_masks; 301 int j; 302 struct cpumask *result = group_cpus_evenly(this_vecs, &nr_masks); 303 304 if (!result) { 305 kfree(masks); 306 return NULL; 307 } 308 309 for (j = 0; j < nr_masks; j++) 310 cpumask_copy(&masks[curvec + j], &result[j]); 311 kfree(result); 312 313 curvec += nr_masks; 314 usedvecs += nr_masks; 315 } 316 317 /* Fill out vectors at the end that don't need affinity */ 318 if (usedvecs >= affvecs) 319 curvec = affd->pre_vectors + affvecs; 320 else 321 curvec = affd->pre_vectors + usedvecs; 322 for (; curvec < nvecs; curvec++) 323 cpumask_setall(&masks[curvec]); 324 325 return masks; 326 } 327 328 static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs, 329 struct virtqueue *vqs[], 330 struct virtqueue_info vqs_info[], 331 struct irq_affinity *desc) 332 { 333 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev); 334 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 335 const struct vdpa_config_ops *ops = vdpa->config; 336 struct cpumask *masks; 337 struct vdpa_callback cb; 338 bool has_affinity = desc && ops->set_vq_affinity; 339 int i, err, queue_idx = 0; 340 341 if (has_affinity) { 342 masks = create_affinity_masks(nvqs, desc); 343 if (!masks) 344 return -ENOMEM; 345 } 346 347 for (i = 0; i < nvqs; ++i) { 348 struct virtqueue_info *vqi = &vqs_info[i]; 349 350 if (!vqi->name) { 351 vqs[i] = NULL; 352 continue; 353 } 354 355 vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++, vqi->callback, 356 vqi->name, vqi->ctx); 357 if (IS_ERR(vqs[i])) { 358 err = PTR_ERR(vqs[i]); 359 goto err_setup_vq; 360 } 361 362 if (has_affinity) 363 ops->set_vq_affinity(vdpa, i, &masks[i]); 364 } 365 366 cb.callback = virtio_vdpa_config_cb; 367 cb.private = vd_dev; 368 ops->set_config_cb(vdpa, &cb); 369 if (has_affinity) 370 kfree(masks); 371 372 return 0; 373 374 err_setup_vq: 375 virtio_vdpa_del_vqs(vdev); 376 if (has_affinity) 377 kfree(masks); 378 return err; 379 } 380 381 static u64 virtio_vdpa_get_features(struct virtio_device *vdev) 382 { 383 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 384 const struct vdpa_config_ops *ops = vdpa->config; 385 386 return ops->get_device_features(vdpa); 387 } 388 389 static int virtio_vdpa_finalize_features(struct virtio_device *vdev) 390 { 391 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 392 393 /* Give virtio_ring a chance to accept features. */ 394 vring_transport_features(vdev); 395 396 return vdpa_set_features(vdpa, vdev->features); 397 } 398 399 static const char *virtio_vdpa_bus_name(struct virtio_device *vdev) 400 { 401 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev); 402 struct vdpa_device *vdpa = vd_dev->vdpa; 403 404 return dev_name(&vdpa->dev); 405 } 406 407 static int virtio_vdpa_set_vq_affinity(struct virtqueue *vq, 408 const struct cpumask *cpu_mask) 409 { 410 struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev); 411 struct vdpa_device *vdpa = vd_dev->vdpa; 412 const struct vdpa_config_ops *ops = vdpa->config; 413 unsigned int index = vq->index; 414 415 if (ops->set_vq_affinity) 416 return ops->set_vq_affinity(vdpa, index, cpu_mask); 417 418 return 0; 419 } 420 421 static const struct cpumask * 422 virtio_vdpa_get_vq_affinity(struct virtio_device *vdev, int index) 423 { 424 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 425 const struct vdpa_config_ops *ops = vdpa->config; 426 427 if (ops->get_vq_affinity) 428 return ops->get_vq_affinity(vdpa, index); 429 430 return NULL; 431 } 432 433 static const struct virtio_config_ops virtio_vdpa_config_ops = { 434 .get = virtio_vdpa_get, 435 .set = virtio_vdpa_set, 436 .generation = virtio_vdpa_generation, 437 .get_status = virtio_vdpa_get_status, 438 .set_status = virtio_vdpa_set_status, 439 .reset = virtio_vdpa_reset, 440 .find_vqs = virtio_vdpa_find_vqs, 441 .del_vqs = virtio_vdpa_del_vqs, 442 .get_features = virtio_vdpa_get_features, 443 .finalize_features = virtio_vdpa_finalize_features, 444 .bus_name = virtio_vdpa_bus_name, 445 .set_vq_affinity = virtio_vdpa_set_vq_affinity, 446 .get_vq_affinity = virtio_vdpa_get_vq_affinity, 447 }; 448 449 static void virtio_vdpa_release_dev(struct device *_d) 450 { 451 struct virtio_device *vdev = 452 container_of(_d, struct virtio_device, dev); 453 struct virtio_vdpa_device *vd_dev = 454 container_of(vdev, struct virtio_vdpa_device, vdev); 455 456 kfree(vd_dev); 457 } 458 459 static int virtio_vdpa_probe(struct vdpa_device *vdpa) 460 { 461 const struct vdpa_config_ops *ops = vdpa->config; 462 struct virtio_vdpa_device *vd_dev, *reg_dev = NULL; 463 int ret = -EINVAL; 464 465 vd_dev = kzalloc(sizeof(*vd_dev), GFP_KERNEL); 466 if (!vd_dev) 467 return -ENOMEM; 468 469 vd_dev->vdev.dev.parent = vdpa->map ? &vdpa->dev : 470 vdpa_get_map(vdpa).dma_dev; 471 vd_dev->vdev.dev.release = virtio_vdpa_release_dev; 472 vd_dev->vdev.config = &virtio_vdpa_config_ops; 473 vd_dev->vdev.map = vdpa->map; 474 vd_dev->vdpa = vdpa; 475 476 vd_dev->vdev.id.device = ops->get_device_id(vdpa); 477 if (vd_dev->vdev.id.device == 0) 478 goto err; 479 480 vd_dev->vdev.id.vendor = ops->get_vendor_id(vdpa); 481 ret = register_virtio_device(&vd_dev->vdev); 482 reg_dev = vd_dev; 483 if (ret) 484 goto err; 485 486 vdpa_set_drvdata(vdpa, vd_dev); 487 488 return 0; 489 490 err: 491 if (reg_dev) 492 put_device(&vd_dev->vdev.dev); 493 else 494 kfree(vd_dev); 495 return ret; 496 } 497 498 static void virtio_vdpa_remove(struct vdpa_device *vdpa) 499 { 500 struct virtio_vdpa_device *vd_dev = vdpa_get_drvdata(vdpa); 501 502 unregister_virtio_device(&vd_dev->vdev); 503 } 504 505 static struct vdpa_driver virtio_vdpa_driver = { 506 .driver = { 507 .name = "virtio_vdpa", 508 }, 509 .probe = virtio_vdpa_probe, 510 .remove = virtio_vdpa_remove, 511 }; 512 513 module_vdpa_driver(virtio_vdpa_driver); 514 515 MODULE_VERSION(MOD_VERSION); 516 MODULE_LICENSE(MOD_LICENSE); 517 MODULE_AUTHOR(MOD_AUTHOR); 518 MODULE_DESCRIPTION(MOD_DESC); 519