1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VDPA device simulator core. 4 * 5 * Copyright (c) 2020, Red Hat Inc. All rights reserved. 6 * Author: Jason Wang <jasowang@redhat.com> 7 * 8 */ 9 10 #include <linux/init.h> 11 #include <linux/module.h> 12 #include <linux/device.h> 13 #include <linux/kernel.h> 14 #include <linux/kthread.h> 15 #include <linux/slab.h> 16 #include <linux/dma-map-ops.h> 17 #include <linux/vringh.h> 18 #include <linux/vdpa.h> 19 #include <linux/vhost_iotlb.h> 20 #include <uapi/linux/vdpa.h> 21 #include <uapi/linux/vhost_types.h> 22 23 #include "vdpa_sim.h" 24 25 #define DRV_VERSION "0.1" 26 #define DRV_AUTHOR "Jason Wang <jasowang@redhat.com>" 27 #define DRV_DESC "vDPA Device Simulator core" 28 #define DRV_LICENSE "GPL v2" 29 30 static int batch_mapping = 1; 31 module_param(batch_mapping, int, 0444); 32 MODULE_PARM_DESC(batch_mapping, "Batched mapping 1 -Enable; 0 - Disable"); 33 34 static int max_iotlb_entries = 2048; 35 module_param(max_iotlb_entries, int, 0444); 36 MODULE_PARM_DESC(max_iotlb_entries, 37 "Maximum number of iotlb entries for each address space. (default: 2048)"); 38 39 static bool use_va = true; 40 module_param(use_va, bool, 0444); 41 MODULE_PARM_DESC(use_va, "Enable/disable the device's ability to use VA"); 42 43 #define VDPASIM_QUEUE_ALIGN PAGE_SIZE 44 #define VDPASIM_QUEUE_MAX 256 45 #define VDPASIM_VENDOR_ID 0 46 47 struct vdpasim_mm_work { 48 struct kthread_work work; 49 struct vdpasim *vdpasim; 50 struct mm_struct *mm_to_bind; 51 int ret; 52 }; 53 54 static void vdpasim_mm_work_fn(struct kthread_work *work) 55 { 56 struct vdpasim_mm_work *mm_work = 57 container_of(work, struct vdpasim_mm_work, work); 58 struct vdpasim *vdpasim = mm_work->vdpasim; 59 60 mm_work->ret = 0; 61 62 //TODO: should we attach the cgroup of the mm owner? 63 vdpasim->mm_bound = mm_work->mm_to_bind; 64 } 65 66 static void vdpasim_worker_change_mm_sync(struct vdpasim *vdpasim, 67 struct vdpasim_mm_work *mm_work) 68 { 69 struct kthread_work *work = &mm_work->work; 70 71 kthread_init_work(work, vdpasim_mm_work_fn); 72 kthread_queue_work(vdpasim->worker, work); 73 74 kthread_flush_work(work); 75 } 76 77 static struct vdpasim *vdpa_to_sim(struct vdpa_device *vdpa) 78 { 79 return container_of(vdpa, struct vdpasim, vdpa); 80 } 81 82 static void vdpasim_vq_notify(struct vringh *vring) 83 { 84 struct vdpasim_virtqueue *vq = 85 container_of(vring, struct vdpasim_virtqueue, vring); 86 87 if (!vq->cb) 88 return; 89 90 vq->cb(vq->private); 91 } 92 93 static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx) 94 { 95 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 96 uint16_t last_avail_idx = vq->vring.last_avail_idx; 97 struct vring_desc *desc = (struct vring_desc *) 98 (uintptr_t)vq->desc_addr; 99 struct vring_avail *avail = (struct vring_avail *) 100 (uintptr_t)vq->driver_addr; 101 struct vring_used *used = (struct vring_used *) 102 (uintptr_t)vq->device_addr; 103 104 if (use_va && vdpasim->mm_bound) { 105 vringh_init_iotlb_va(&vq->vring, vdpasim->features, vq->num, 106 true, desc, avail, used); 107 } else { 108 vringh_init_iotlb(&vq->vring, vdpasim->features, vq->num, 109 true, desc, avail, used); 110 } 111 112 vq->vring.last_avail_idx = last_avail_idx; 113 114 /* 115 * Since vdpa_sim does not support receive inflight descriptors as a 116 * destination of a migration, let's set both avail_idx and used_idx 117 * the same at vq start. This is how vhost-user works in a 118 * VHOST_SET_VRING_BASE call. 119 * 120 * Although the simple fix is to set last_used_idx at 121 * vdpasim_set_vq_state, it would be reset at vdpasim_queue_ready. 122 */ 123 vq->vring.last_used_idx = last_avail_idx; 124 vq->vring.notify = vdpasim_vq_notify; 125 } 126 127 static void vdpasim_vq_reset(struct vdpasim *vdpasim, 128 struct vdpasim_virtqueue *vq) 129 { 130 vq->ready = false; 131 vq->desc_addr = 0; 132 vq->driver_addr = 0; 133 vq->device_addr = 0; 134 vq->cb = NULL; 135 vq->private = NULL; 136 vringh_init_iotlb(&vq->vring, vdpasim->dev_attr.supported_features, 137 VDPASIM_QUEUE_MAX, false, NULL, NULL, NULL); 138 139 vq->vring.notify = NULL; 140 } 141 142 static void vdpasim_do_reset(struct vdpasim *vdpasim, u32 flags) 143 { 144 int i; 145 146 spin_lock(&vdpasim->iommu_lock); 147 148 for (i = 0; i < vdpasim->dev_attr.nvqs; i++) { 149 vdpasim_vq_reset(vdpasim, &vdpasim->vqs[i]); 150 vringh_set_iotlb(&vdpasim->vqs[i].vring, &vdpasim->iommu[0], 151 &vdpasim->iommu_lock); 152 } 153 154 if (flags & VDPA_RESET_F_CLEAN_MAP) { 155 for (i = 0; i < vdpasim->dev_attr.nas; i++) { 156 vhost_iotlb_reset(&vdpasim->iommu[i]); 157 vhost_iotlb_add_range(&vdpasim->iommu[i], 0, ULONG_MAX, 158 0, VHOST_MAP_RW); 159 vdpasim->iommu_pt[i] = true; 160 } 161 } 162 163 vdpasim->running = false; 164 vdpasim->pending_kick = false; 165 spin_unlock(&vdpasim->iommu_lock); 166 167 vdpasim->features = 0; 168 vdpasim->status = 0; 169 ++vdpasim->generation; 170 } 171 172 static const struct vdpa_config_ops vdpasim_config_ops; 173 static const struct vdpa_config_ops vdpasim_batch_config_ops; 174 175 static void vdpasim_work_fn(struct kthread_work *work) 176 { 177 struct vdpasim *vdpasim = container_of(work, struct vdpasim, work); 178 struct mm_struct *mm = vdpasim->mm_bound; 179 180 if (use_va && mm) { 181 if (!mmget_not_zero(mm)) 182 return; 183 kthread_use_mm(mm); 184 } 185 186 vdpasim->dev_attr.work_fn(vdpasim); 187 188 if (use_va && mm) { 189 kthread_unuse_mm(mm); 190 mmput(mm); 191 } 192 } 193 194 struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr, 195 const struct vdpa_dev_set_config *config) 196 { 197 const struct vdpa_config_ops *ops; 198 struct vdpa_device *vdpa; 199 struct vdpasim *vdpasim; 200 struct device *dev; 201 int i, ret = -ENOMEM; 202 203 if (!dev_attr->alloc_size) 204 return ERR_PTR(-EINVAL); 205 if (max_iotlb_entries < 2) 206 return ERR_PTR(-EINVAL); 207 208 if (config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) { 209 if (config->device_features & 210 ~dev_attr->supported_features) 211 return ERR_PTR(-EINVAL); 212 dev_attr->supported_features = 213 config->device_features; 214 } 215 216 if (batch_mapping) 217 ops = &vdpasim_batch_config_ops; 218 else 219 ops = &vdpasim_config_ops; 220 221 vdpa = __vdpa_alloc_device(NULL, ops, NULL, 222 dev_attr->ngroups, dev_attr->nas, 223 dev_attr->alloc_size, 224 dev_attr->name, use_va); 225 if (IS_ERR(vdpa)) { 226 ret = PTR_ERR(vdpa); 227 goto err_alloc; 228 } 229 230 vdpasim = vdpa_to_sim(vdpa); 231 vdpasim->dev_attr = *dev_attr; 232 dev = &vdpasim->vdpa.dev; 233 234 kthread_init_work(&vdpasim->work, vdpasim_work_fn); 235 vdpasim->worker = kthread_run_worker(0, "vDPA sim worker: %s", 236 dev_attr->name); 237 if (IS_ERR(vdpasim->worker)) { 238 ret = PTR_ERR(vdpasim->worker); 239 vdpasim->worker = NULL; 240 goto err_iommu; 241 } 242 243 mutex_init(&vdpasim->mutex); 244 spin_lock_init(&vdpasim->iommu_lock); 245 246 dev->dma_mask = &dev->coherent_dma_mask; 247 if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64))) 248 goto err_iommu; 249 vdpasim->vdpa.mdev = dev_attr->mgmt_dev; 250 251 vdpasim->config = kzalloc(dev_attr->config_size, GFP_KERNEL); 252 if (!vdpasim->config) 253 goto err_iommu; 254 255 vdpasim->vqs = kzalloc_objs(struct vdpasim_virtqueue, dev_attr->nvqs); 256 if (!vdpasim->vqs) 257 goto err_iommu; 258 259 vdpasim->iommu = kmalloc_objs(*vdpasim->iommu, vdpasim->dev_attr.nas); 260 if (!vdpasim->iommu) 261 goto err_iommu; 262 263 vdpasim->iommu_pt = kmalloc_objs(*vdpasim->iommu_pt, 264 vdpasim->dev_attr.nas); 265 if (!vdpasim->iommu_pt) 266 goto err_iommu; 267 268 for (i = 0; i < vdpasim->dev_attr.nas; i++) { 269 vhost_iotlb_init(&vdpasim->iommu[i], max_iotlb_entries, 0); 270 ret = vhost_iotlb_add_range(&vdpasim->iommu[i], 0, ULONG_MAX, 271 0, VHOST_MAP_RW); 272 if (ret) 273 goto err_iommu; 274 vdpasim->iommu_pt[i] = true; 275 } 276 277 for (i = 0; i < dev_attr->nvqs; i++) 278 vringh_set_iotlb(&vdpasim->vqs[i].vring, &vdpasim->iommu[0], 279 &vdpasim->iommu_lock); 280 281 vdpasim->vdpa.vmap.dma_dev = dev; 282 283 return vdpasim; 284 285 err_iommu: 286 put_device(dev); 287 err_alloc: 288 return ERR_PTR(ret); 289 } 290 EXPORT_SYMBOL_GPL(vdpasim_create); 291 292 void vdpasim_schedule_work(struct vdpasim *vdpasim) 293 { 294 kthread_queue_work(vdpasim->worker, &vdpasim->work); 295 } 296 EXPORT_SYMBOL_GPL(vdpasim_schedule_work); 297 298 static int vdpasim_set_vq_address(struct vdpa_device *vdpa, u16 idx, 299 u64 desc_area, u64 driver_area, 300 u64 device_area) 301 { 302 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 303 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 304 305 vq->desc_addr = desc_area; 306 vq->driver_addr = driver_area; 307 vq->device_addr = device_area; 308 309 return 0; 310 } 311 312 static void vdpasim_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num) 313 { 314 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 315 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 316 317 vq->num = num; 318 } 319 320 static u16 vdpasim_get_vq_size(struct vdpa_device *vdpa, u16 idx) 321 { 322 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 323 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 324 325 if (vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK) 326 return vq->num; 327 else 328 return VDPASIM_QUEUE_MAX; 329 } 330 331 static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx) 332 { 333 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 334 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 335 336 if (!vdpasim->running && 337 (vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK)) { 338 vdpasim->pending_kick = true; 339 return; 340 } 341 342 if (vq->ready) 343 vdpasim_schedule_work(vdpasim); 344 } 345 346 static void vdpasim_set_vq_cb(struct vdpa_device *vdpa, u16 idx, 347 struct vdpa_callback *cb) 348 { 349 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 350 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 351 352 vq->cb = cb->callback; 353 vq->private = cb->private; 354 } 355 356 static void vdpasim_set_vq_ready(struct vdpa_device *vdpa, u16 idx, bool ready) 357 { 358 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 359 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 360 bool old_ready; 361 362 mutex_lock(&vdpasim->mutex); 363 old_ready = vq->ready; 364 vq->ready = ready; 365 if (vq->ready && !old_ready) { 366 vdpasim_queue_ready(vdpasim, idx); 367 } 368 mutex_unlock(&vdpasim->mutex); 369 } 370 371 static bool vdpasim_get_vq_ready(struct vdpa_device *vdpa, u16 idx) 372 { 373 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 374 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 375 376 return vq->ready; 377 } 378 379 static int vdpasim_set_vq_state(struct vdpa_device *vdpa, u16 idx, 380 const struct vdpa_vq_state *state) 381 { 382 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 383 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 384 struct vringh *vrh = &vq->vring; 385 386 mutex_lock(&vdpasim->mutex); 387 vrh->last_avail_idx = state->split.avail_index; 388 mutex_unlock(&vdpasim->mutex); 389 390 return 0; 391 } 392 393 static int vdpasim_get_vq_state(struct vdpa_device *vdpa, u16 idx, 394 struct vdpa_vq_state *state) 395 { 396 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 397 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 398 struct vringh *vrh = &vq->vring; 399 400 state->split.avail_index = vrh->last_avail_idx; 401 return 0; 402 } 403 404 static int vdpasim_get_vq_stats(struct vdpa_device *vdpa, u16 idx, 405 struct sk_buff *msg, 406 struct netlink_ext_ack *extack) 407 { 408 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 409 410 if (vdpasim->dev_attr.get_stats) 411 return vdpasim->dev_attr.get_stats(vdpasim, idx, 412 msg, extack); 413 return -EOPNOTSUPP; 414 } 415 416 static u32 vdpasim_get_vq_align(struct vdpa_device *vdpa) 417 { 418 return VDPASIM_QUEUE_ALIGN; 419 } 420 421 static u32 vdpasim_get_vq_group(struct vdpa_device *vdpa, u16 idx) 422 { 423 /* RX and TX belongs to group 0, CVQ belongs to group 1 */ 424 if (idx == 2) 425 return 1; 426 else 427 return 0; 428 } 429 430 static u64 vdpasim_get_device_features(struct vdpa_device *vdpa) 431 { 432 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 433 434 return vdpasim->dev_attr.supported_features; 435 } 436 437 static u64 vdpasim_get_backend_features(const struct vdpa_device *vdpa) 438 { 439 return BIT_ULL(VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK); 440 } 441 442 static int vdpasim_set_driver_features(struct vdpa_device *vdpa, u64 features) 443 { 444 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 445 446 /* DMA mapping must be done by driver */ 447 if (!(features & (1ULL << VIRTIO_F_ACCESS_PLATFORM))) 448 return -EINVAL; 449 450 vdpasim->features = features & vdpasim->dev_attr.supported_features; 451 452 return 0; 453 } 454 455 static u64 vdpasim_get_driver_features(struct vdpa_device *vdpa) 456 { 457 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 458 459 return vdpasim->features; 460 } 461 462 static void vdpasim_set_config_cb(struct vdpa_device *vdpa, 463 struct vdpa_callback *cb) 464 { 465 /* We don't support config interrupt */ 466 } 467 468 static u16 vdpasim_get_vq_num_max(struct vdpa_device *vdpa) 469 { 470 return VDPASIM_QUEUE_MAX; 471 } 472 473 static u32 vdpasim_get_device_id(struct vdpa_device *vdpa) 474 { 475 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 476 477 return vdpasim->dev_attr.id; 478 } 479 480 static u32 vdpasim_get_vendor_id(struct vdpa_device *vdpa) 481 { 482 return VDPASIM_VENDOR_ID; 483 } 484 485 static u8 vdpasim_get_status(struct vdpa_device *vdpa) 486 { 487 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 488 u8 status; 489 490 mutex_lock(&vdpasim->mutex); 491 status = vdpasim->status; 492 mutex_unlock(&vdpasim->mutex); 493 494 return status; 495 } 496 497 static void vdpasim_set_status(struct vdpa_device *vdpa, u8 status) 498 { 499 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 500 501 mutex_lock(&vdpasim->mutex); 502 vdpasim->status = status; 503 vdpasim->running = (status & VIRTIO_CONFIG_S_DRIVER_OK) != 0; 504 mutex_unlock(&vdpasim->mutex); 505 } 506 507 static int vdpasim_compat_reset(struct vdpa_device *vdpa, u32 flags) 508 { 509 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 510 511 mutex_lock(&vdpasim->mutex); 512 vdpasim->status = 0; 513 vdpasim_do_reset(vdpasim, flags); 514 mutex_unlock(&vdpasim->mutex); 515 516 return 0; 517 } 518 519 static int vdpasim_reset(struct vdpa_device *vdpa) 520 { 521 return vdpasim_compat_reset(vdpa, 0); 522 } 523 524 static int vdpasim_suspend(struct vdpa_device *vdpa) 525 { 526 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 527 528 mutex_lock(&vdpasim->mutex); 529 vdpasim->running = false; 530 mutex_unlock(&vdpasim->mutex); 531 532 return 0; 533 } 534 535 static int vdpasim_resume(struct vdpa_device *vdpa) 536 { 537 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 538 int i; 539 540 mutex_lock(&vdpasim->mutex); 541 vdpasim->running = true; 542 543 if (vdpasim->pending_kick) { 544 /* Process pending descriptors */ 545 for (i = 0; i < vdpasim->dev_attr.nvqs; ++i) 546 vdpasim_kick_vq(vdpa, i); 547 548 vdpasim->pending_kick = false; 549 } 550 551 mutex_unlock(&vdpasim->mutex); 552 553 return 0; 554 } 555 556 static size_t vdpasim_get_config_size(struct vdpa_device *vdpa) 557 { 558 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 559 560 return vdpasim->dev_attr.config_size; 561 } 562 563 static void vdpasim_get_config(struct vdpa_device *vdpa, unsigned int offset, 564 void *buf, unsigned int len) 565 { 566 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 567 568 if (offset + len > vdpasim->dev_attr.config_size) 569 return; 570 571 if (vdpasim->dev_attr.get_config) 572 vdpasim->dev_attr.get_config(vdpasim, vdpasim->config); 573 574 memcpy(buf, vdpasim->config + offset, len); 575 } 576 577 static void vdpasim_set_config(struct vdpa_device *vdpa, unsigned int offset, 578 const void *buf, unsigned int len) 579 { 580 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 581 582 if (offset + len > vdpasim->dev_attr.config_size) 583 return; 584 585 memcpy(vdpasim->config + offset, buf, len); 586 587 if (vdpasim->dev_attr.set_config) 588 vdpasim->dev_attr.set_config(vdpasim, vdpasim->config); 589 } 590 591 static u32 vdpasim_get_generation(struct vdpa_device *vdpa) 592 { 593 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 594 595 return vdpasim->generation; 596 } 597 598 static struct vdpa_iova_range vdpasim_get_iova_range(struct vdpa_device *vdpa) 599 { 600 struct vdpa_iova_range range = { 601 .first = 0ULL, 602 .last = ULLONG_MAX, 603 }; 604 605 return range; 606 } 607 608 static int vdpasim_set_group_asid(struct vdpa_device *vdpa, unsigned int group, 609 unsigned int asid) 610 { 611 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 612 struct vhost_iotlb *iommu; 613 int i; 614 615 iommu = &vdpasim->iommu[asid]; 616 617 mutex_lock(&vdpasim->mutex); 618 619 for (i = 0; i < vdpasim->dev_attr.nvqs; i++) 620 if (vdpasim_get_vq_group(vdpa, i) == group) 621 vringh_set_iotlb(&vdpasim->vqs[i].vring, iommu, 622 &vdpasim->iommu_lock); 623 624 mutex_unlock(&vdpasim->mutex); 625 626 return 0; 627 } 628 629 static int vdpasim_set_map(struct vdpa_device *vdpa, unsigned int asid, 630 struct vhost_iotlb *iotlb) 631 { 632 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 633 struct vhost_iotlb_map *map; 634 struct vhost_iotlb *iommu; 635 u64 start = 0ULL, last = 0ULL - 1; 636 int ret; 637 638 if (asid >= vdpasim->dev_attr.nas) 639 return -EINVAL; 640 641 spin_lock(&vdpasim->iommu_lock); 642 643 iommu = &vdpasim->iommu[asid]; 644 vhost_iotlb_reset(iommu); 645 vdpasim->iommu_pt[asid] = false; 646 647 for (map = vhost_iotlb_itree_first(iotlb, start, last); map; 648 map = vhost_iotlb_itree_next(map, start, last)) { 649 ret = vhost_iotlb_add_range(iommu, map->start, 650 map->last, map->addr, map->perm); 651 if (ret) 652 goto err; 653 } 654 spin_unlock(&vdpasim->iommu_lock); 655 return 0; 656 657 err: 658 vhost_iotlb_reset(iommu); 659 spin_unlock(&vdpasim->iommu_lock); 660 return ret; 661 } 662 663 static int vdpasim_reset_map(struct vdpa_device *vdpa, unsigned int asid) 664 { 665 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 666 667 if (asid >= vdpasim->dev_attr.nas) 668 return -EINVAL; 669 670 spin_lock(&vdpasim->iommu_lock); 671 if (vdpasim->iommu_pt[asid]) 672 goto out; 673 vhost_iotlb_reset(&vdpasim->iommu[asid]); 674 vhost_iotlb_add_range(&vdpasim->iommu[asid], 0, ULONG_MAX, 675 0, VHOST_MAP_RW); 676 vdpasim->iommu_pt[asid] = true; 677 out: 678 spin_unlock(&vdpasim->iommu_lock); 679 return 0; 680 } 681 682 static int vdpasim_bind_mm(struct vdpa_device *vdpa, struct mm_struct *mm) 683 { 684 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 685 struct vdpasim_mm_work mm_work; 686 687 mm_work.vdpasim = vdpasim; 688 mm_work.mm_to_bind = mm; 689 690 vdpasim_worker_change_mm_sync(vdpasim, &mm_work); 691 692 return mm_work.ret; 693 } 694 695 static void vdpasim_unbind_mm(struct vdpa_device *vdpa) 696 { 697 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 698 struct vdpasim_mm_work mm_work; 699 700 mm_work.vdpasim = vdpasim; 701 mm_work.mm_to_bind = NULL; 702 703 vdpasim_worker_change_mm_sync(vdpasim, &mm_work); 704 } 705 706 static int vdpasim_dma_map(struct vdpa_device *vdpa, unsigned int asid, 707 u64 iova, u64 size, 708 u64 pa, u32 perm, void *opaque) 709 { 710 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 711 int ret; 712 713 if (asid >= vdpasim->dev_attr.nas) 714 return -EINVAL; 715 716 spin_lock(&vdpasim->iommu_lock); 717 if (vdpasim->iommu_pt[asid]) { 718 vhost_iotlb_reset(&vdpasim->iommu[asid]); 719 vdpasim->iommu_pt[asid] = false; 720 } 721 ret = vhost_iotlb_add_range_ctx(&vdpasim->iommu[asid], iova, 722 iova + size - 1, pa, perm, opaque); 723 spin_unlock(&vdpasim->iommu_lock); 724 725 return ret; 726 } 727 728 static int vdpasim_dma_unmap(struct vdpa_device *vdpa, unsigned int asid, 729 u64 iova, u64 size) 730 { 731 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 732 733 if (asid >= vdpasim->dev_attr.nas) 734 return -EINVAL; 735 736 spin_lock(&vdpasim->iommu_lock); 737 if (vdpasim->iommu_pt[asid]) { 738 vhost_iotlb_reset(&vdpasim->iommu[asid]); 739 vdpasim->iommu_pt[asid] = false; 740 } 741 vhost_iotlb_del_range(&vdpasim->iommu[asid], iova, iova + size - 1); 742 spin_unlock(&vdpasim->iommu_lock); 743 744 return 0; 745 } 746 747 static void vdpasim_free(struct vdpa_device *vdpa) 748 { 749 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 750 int i; 751 752 if (vdpasim->worker) { 753 kthread_cancel_work_sync(&vdpasim->work); 754 kthread_destroy_worker(vdpasim->worker); 755 } 756 757 if (vdpasim->vqs) { 758 for (i = 0; i < vdpasim->dev_attr.nvqs; i++) { 759 vringh_kiov_cleanup(&vdpasim->vqs[i].out_iov); 760 vringh_kiov_cleanup(&vdpasim->vqs[i].in_iov); 761 } 762 } 763 764 vdpasim->dev_attr.free(vdpasim); 765 766 if (vdpasim->iommu) { 767 for (i = 0; i < vdpasim->dev_attr.nas; i++) 768 vhost_iotlb_reset(&vdpasim->iommu[i]); 769 } 770 kfree(vdpasim->iommu); 771 kfree(vdpasim->iommu_pt); 772 kfree(vdpasim->vqs); 773 kfree(vdpasim->config); 774 } 775 776 static const struct vdpa_config_ops vdpasim_config_ops = { 777 .set_vq_address = vdpasim_set_vq_address, 778 .set_vq_num = vdpasim_set_vq_num, 779 .kick_vq = vdpasim_kick_vq, 780 .set_vq_cb = vdpasim_set_vq_cb, 781 .set_vq_ready = vdpasim_set_vq_ready, 782 .get_vq_ready = vdpasim_get_vq_ready, 783 .set_vq_state = vdpasim_set_vq_state, 784 .get_vendor_vq_stats = vdpasim_get_vq_stats, 785 .get_vq_state = vdpasim_get_vq_state, 786 .get_vq_align = vdpasim_get_vq_align, 787 .get_vq_group = vdpasim_get_vq_group, 788 .get_device_features = vdpasim_get_device_features, 789 .get_backend_features = vdpasim_get_backend_features, 790 .set_driver_features = vdpasim_set_driver_features, 791 .get_driver_features = vdpasim_get_driver_features, 792 .set_config_cb = vdpasim_set_config_cb, 793 .get_vq_num_max = vdpasim_get_vq_num_max, 794 .get_vq_size = vdpasim_get_vq_size, 795 .get_device_id = vdpasim_get_device_id, 796 .get_vendor_id = vdpasim_get_vendor_id, 797 .get_status = vdpasim_get_status, 798 .set_status = vdpasim_set_status, 799 .reset = vdpasim_reset, 800 .compat_reset = vdpasim_compat_reset, 801 .suspend = vdpasim_suspend, 802 .resume = vdpasim_resume, 803 .get_config_size = vdpasim_get_config_size, 804 .get_config = vdpasim_get_config, 805 .set_config = vdpasim_set_config, 806 .get_generation = vdpasim_get_generation, 807 .get_iova_range = vdpasim_get_iova_range, 808 .set_group_asid = vdpasim_set_group_asid, 809 .dma_map = vdpasim_dma_map, 810 .dma_unmap = vdpasim_dma_unmap, 811 .reset_map = vdpasim_reset_map, 812 .bind_mm = vdpasim_bind_mm, 813 .unbind_mm = vdpasim_unbind_mm, 814 .free = vdpasim_free, 815 }; 816 817 static const struct vdpa_config_ops vdpasim_batch_config_ops = { 818 .set_vq_address = vdpasim_set_vq_address, 819 .set_vq_num = vdpasim_set_vq_num, 820 .kick_vq = vdpasim_kick_vq, 821 .set_vq_cb = vdpasim_set_vq_cb, 822 .set_vq_ready = vdpasim_set_vq_ready, 823 .get_vq_ready = vdpasim_get_vq_ready, 824 .set_vq_state = vdpasim_set_vq_state, 825 .get_vendor_vq_stats = vdpasim_get_vq_stats, 826 .get_vq_state = vdpasim_get_vq_state, 827 .get_vq_align = vdpasim_get_vq_align, 828 .get_vq_group = vdpasim_get_vq_group, 829 .get_device_features = vdpasim_get_device_features, 830 .get_backend_features = vdpasim_get_backend_features, 831 .set_driver_features = vdpasim_set_driver_features, 832 .get_driver_features = vdpasim_get_driver_features, 833 .set_config_cb = vdpasim_set_config_cb, 834 .get_vq_num_max = vdpasim_get_vq_num_max, 835 .get_device_id = vdpasim_get_device_id, 836 .get_vendor_id = vdpasim_get_vendor_id, 837 .get_status = vdpasim_get_status, 838 .set_status = vdpasim_set_status, 839 .reset = vdpasim_reset, 840 .compat_reset = vdpasim_compat_reset, 841 .suspend = vdpasim_suspend, 842 .resume = vdpasim_resume, 843 .get_config_size = vdpasim_get_config_size, 844 .get_config = vdpasim_get_config, 845 .set_config = vdpasim_set_config, 846 .get_generation = vdpasim_get_generation, 847 .get_iova_range = vdpasim_get_iova_range, 848 .set_group_asid = vdpasim_set_group_asid, 849 .set_map = vdpasim_set_map, 850 .reset_map = vdpasim_reset_map, 851 .bind_mm = vdpasim_bind_mm, 852 .unbind_mm = vdpasim_unbind_mm, 853 .free = vdpasim_free, 854 }; 855 856 MODULE_VERSION(DRV_VERSION); 857 MODULE_LICENSE(DRV_LICENSE); 858 MODULE_AUTHOR(DRV_AUTHOR); 859 MODULE_DESCRIPTION(DRV_DESC); 860