1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VDPA device simulator core. 4 * 5 * Copyright (c) 2020, Red Hat Inc. All rights reserved. 6 * Author: Jason Wang <jasowang@redhat.com> 7 * 8 */ 9 10 #include <linux/init.h> 11 #include <linux/module.h> 12 #include <linux/device.h> 13 #include <linux/kernel.h> 14 #include <linux/kthread.h> 15 #include <linux/slab.h> 16 #include <linux/dma-map-ops.h> 17 #include <linux/vringh.h> 18 #include <linux/vdpa.h> 19 #include <linux/vhost_iotlb.h> 20 #include <uapi/linux/vdpa.h> 21 #include <uapi/linux/vhost_types.h> 22 23 #include "vdpa_sim.h" 24 25 #define DRV_VERSION "0.1" 26 #define DRV_AUTHOR "Jason Wang <jasowang@redhat.com>" 27 #define DRV_DESC "vDPA Device Simulator core" 28 #define DRV_LICENSE "GPL v2" 29 30 static int batch_mapping = 1; 31 module_param(batch_mapping, int, 0444); 32 MODULE_PARM_DESC(batch_mapping, "Batched mapping 1 -Enable; 0 - Disable"); 33 34 static int max_iotlb_entries = 2048; 35 module_param(max_iotlb_entries, int, 0444); 36 MODULE_PARM_DESC(max_iotlb_entries, 37 "Maximum number of iotlb entries for each address space. 0 means unlimited. (default: 2048)"); 38 39 static bool use_va = true; 40 module_param(use_va, bool, 0444); 41 MODULE_PARM_DESC(use_va, "Enable/disable the device's ability to use VA"); 42 43 #define VDPASIM_QUEUE_ALIGN PAGE_SIZE 44 #define VDPASIM_QUEUE_MAX 256 45 #define VDPASIM_VENDOR_ID 0 46 47 struct vdpasim_mm_work { 48 struct kthread_work work; 49 struct vdpasim *vdpasim; 50 struct mm_struct *mm_to_bind; 51 int ret; 52 }; 53 54 static void vdpasim_mm_work_fn(struct kthread_work *work) 55 { 56 struct vdpasim_mm_work *mm_work = 57 container_of(work, struct vdpasim_mm_work, work); 58 struct vdpasim *vdpasim = mm_work->vdpasim; 59 60 mm_work->ret = 0; 61 62 //TODO: should we attach the cgroup of the mm owner? 63 vdpasim->mm_bound = mm_work->mm_to_bind; 64 } 65 66 static void vdpasim_worker_change_mm_sync(struct vdpasim *vdpasim, 67 struct vdpasim_mm_work *mm_work) 68 { 69 struct kthread_work *work = &mm_work->work; 70 71 kthread_init_work(work, vdpasim_mm_work_fn); 72 kthread_queue_work(vdpasim->worker, work); 73 74 kthread_flush_work(work); 75 } 76 77 static struct vdpasim *vdpa_to_sim(struct vdpa_device *vdpa) 78 { 79 return container_of(vdpa, struct vdpasim, vdpa); 80 } 81 82 static void vdpasim_vq_notify(struct vringh *vring) 83 { 84 struct vdpasim_virtqueue *vq = 85 container_of(vring, struct vdpasim_virtqueue, vring); 86 87 if (!vq->cb) 88 return; 89 90 vq->cb(vq->private); 91 } 92 93 static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx) 94 { 95 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 96 uint16_t last_avail_idx = vq->vring.last_avail_idx; 97 struct vring_desc *desc = (struct vring_desc *) 98 (uintptr_t)vq->desc_addr; 99 struct vring_avail *avail = (struct vring_avail *) 100 (uintptr_t)vq->driver_addr; 101 struct vring_used *used = (struct vring_used *) 102 (uintptr_t)vq->device_addr; 103 104 if (use_va && vdpasim->mm_bound) { 105 vringh_init_iotlb_va(&vq->vring, vdpasim->features, vq->num, 106 true, desc, avail, used); 107 } else { 108 vringh_init_iotlb(&vq->vring, vdpasim->features, vq->num, 109 true, desc, avail, used); 110 } 111 112 vq->vring.last_avail_idx = last_avail_idx; 113 114 /* 115 * Since vdpa_sim does not support receive inflight descriptors as a 116 * destination of a migration, let's set both avail_idx and used_idx 117 * the same at vq start. This is how vhost-user works in a 118 * VHOST_SET_VRING_BASE call. 119 * 120 * Although the simple fix is to set last_used_idx at 121 * vdpasim_set_vq_state, it would be reset at vdpasim_queue_ready. 122 */ 123 vq->vring.last_used_idx = last_avail_idx; 124 vq->vring.notify = vdpasim_vq_notify; 125 } 126 127 static void vdpasim_vq_reset(struct vdpasim *vdpasim, 128 struct vdpasim_virtqueue *vq) 129 { 130 vq->ready = false; 131 vq->desc_addr = 0; 132 vq->driver_addr = 0; 133 vq->device_addr = 0; 134 vq->cb = NULL; 135 vq->private = NULL; 136 vringh_init_iotlb(&vq->vring, vdpasim->dev_attr.supported_features, 137 VDPASIM_QUEUE_MAX, false, NULL, NULL, NULL); 138 139 vq->vring.notify = NULL; 140 } 141 142 static void vdpasim_do_reset(struct vdpasim *vdpasim, u32 flags) 143 { 144 int i; 145 146 spin_lock(&vdpasim->iommu_lock); 147 148 for (i = 0; i < vdpasim->dev_attr.nvqs; i++) { 149 vdpasim_vq_reset(vdpasim, &vdpasim->vqs[i]); 150 vringh_set_iotlb(&vdpasim->vqs[i].vring, &vdpasim->iommu[0], 151 &vdpasim->iommu_lock); 152 } 153 154 if (flags & VDPA_RESET_F_CLEAN_MAP) { 155 for (i = 0; i < vdpasim->dev_attr.nas; i++) { 156 vhost_iotlb_reset(&vdpasim->iommu[i]); 157 vhost_iotlb_add_range(&vdpasim->iommu[i], 0, ULONG_MAX, 158 0, VHOST_MAP_RW); 159 vdpasim->iommu_pt[i] = true; 160 } 161 } 162 163 vdpasim->running = false; 164 spin_unlock(&vdpasim->iommu_lock); 165 166 vdpasim->features = 0; 167 vdpasim->status = 0; 168 ++vdpasim->generation; 169 } 170 171 static const struct vdpa_config_ops vdpasim_config_ops; 172 static const struct vdpa_config_ops vdpasim_batch_config_ops; 173 174 static void vdpasim_work_fn(struct kthread_work *work) 175 { 176 struct vdpasim *vdpasim = container_of(work, struct vdpasim, work); 177 struct mm_struct *mm = vdpasim->mm_bound; 178 179 if (use_va && mm) { 180 if (!mmget_not_zero(mm)) 181 return; 182 kthread_use_mm(mm); 183 } 184 185 vdpasim->dev_attr.work_fn(vdpasim); 186 187 if (use_va && mm) { 188 kthread_unuse_mm(mm); 189 mmput(mm); 190 } 191 } 192 193 struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr, 194 const struct vdpa_dev_set_config *config) 195 { 196 const struct vdpa_config_ops *ops; 197 struct vdpa_device *vdpa; 198 struct vdpasim *vdpasim; 199 struct device *dev; 200 int i, ret = -ENOMEM; 201 202 if (!dev_attr->alloc_size) 203 return ERR_PTR(-EINVAL); 204 205 if (config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) { 206 if (config->device_features & 207 ~dev_attr->supported_features) 208 return ERR_PTR(-EINVAL); 209 dev_attr->supported_features = 210 config->device_features; 211 } 212 213 if (batch_mapping) 214 ops = &vdpasim_batch_config_ops; 215 else 216 ops = &vdpasim_config_ops; 217 218 vdpa = __vdpa_alloc_device(NULL, ops, 219 dev_attr->ngroups, dev_attr->nas, 220 dev_attr->alloc_size, 221 dev_attr->name, use_va); 222 if (IS_ERR(vdpa)) { 223 ret = PTR_ERR(vdpa); 224 goto err_alloc; 225 } 226 227 vdpasim = vdpa_to_sim(vdpa); 228 vdpasim->dev_attr = *dev_attr; 229 dev = &vdpasim->vdpa.dev; 230 231 kthread_init_work(&vdpasim->work, vdpasim_work_fn); 232 vdpasim->worker = kthread_create_worker(0, "vDPA sim worker: %s", 233 dev_attr->name); 234 if (IS_ERR(vdpasim->worker)) 235 goto err_iommu; 236 237 mutex_init(&vdpasim->mutex); 238 spin_lock_init(&vdpasim->iommu_lock); 239 240 dev->dma_mask = &dev->coherent_dma_mask; 241 if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64))) 242 goto err_iommu; 243 vdpasim->vdpa.mdev = dev_attr->mgmt_dev; 244 245 vdpasim->config = kzalloc(dev_attr->config_size, GFP_KERNEL); 246 if (!vdpasim->config) 247 goto err_iommu; 248 249 vdpasim->vqs = kcalloc(dev_attr->nvqs, sizeof(struct vdpasim_virtqueue), 250 GFP_KERNEL); 251 if (!vdpasim->vqs) 252 goto err_iommu; 253 254 vdpasim->iommu = kmalloc_array(vdpasim->dev_attr.nas, 255 sizeof(*vdpasim->iommu), GFP_KERNEL); 256 if (!vdpasim->iommu) 257 goto err_iommu; 258 259 vdpasim->iommu_pt = kmalloc_array(vdpasim->dev_attr.nas, 260 sizeof(*vdpasim->iommu_pt), GFP_KERNEL); 261 if (!vdpasim->iommu_pt) 262 goto err_iommu; 263 264 for (i = 0; i < vdpasim->dev_attr.nas; i++) { 265 vhost_iotlb_init(&vdpasim->iommu[i], max_iotlb_entries, 0); 266 vhost_iotlb_add_range(&vdpasim->iommu[i], 0, ULONG_MAX, 0, 267 VHOST_MAP_RW); 268 vdpasim->iommu_pt[i] = true; 269 } 270 271 for (i = 0; i < dev_attr->nvqs; i++) 272 vringh_set_iotlb(&vdpasim->vqs[i].vring, &vdpasim->iommu[0], 273 &vdpasim->iommu_lock); 274 275 vdpasim->vdpa.dma_dev = dev; 276 277 return vdpasim; 278 279 err_iommu: 280 put_device(dev); 281 err_alloc: 282 return ERR_PTR(ret); 283 } 284 EXPORT_SYMBOL_GPL(vdpasim_create); 285 286 void vdpasim_schedule_work(struct vdpasim *vdpasim) 287 { 288 kthread_queue_work(vdpasim->worker, &vdpasim->work); 289 } 290 EXPORT_SYMBOL_GPL(vdpasim_schedule_work); 291 292 static int vdpasim_set_vq_address(struct vdpa_device *vdpa, u16 idx, 293 u64 desc_area, u64 driver_area, 294 u64 device_area) 295 { 296 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 297 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 298 299 vq->desc_addr = desc_area; 300 vq->driver_addr = driver_area; 301 vq->device_addr = device_area; 302 303 return 0; 304 } 305 306 static void vdpasim_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num) 307 { 308 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 309 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 310 311 vq->num = num; 312 } 313 314 static u16 vdpasim_get_vq_size(struct vdpa_device *vdpa, u16 idx) 315 { 316 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 317 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 318 319 if (vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK) 320 return vq->num; 321 else 322 return VDPASIM_QUEUE_MAX; 323 } 324 325 static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx) 326 { 327 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 328 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 329 330 if (!vdpasim->running && 331 (vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK)) { 332 vdpasim->pending_kick = true; 333 return; 334 } 335 336 if (vq->ready) 337 vdpasim_schedule_work(vdpasim); 338 } 339 340 static void vdpasim_set_vq_cb(struct vdpa_device *vdpa, u16 idx, 341 struct vdpa_callback *cb) 342 { 343 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 344 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 345 346 vq->cb = cb->callback; 347 vq->private = cb->private; 348 } 349 350 static void vdpasim_set_vq_ready(struct vdpa_device *vdpa, u16 idx, bool ready) 351 { 352 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 353 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 354 bool old_ready; 355 356 mutex_lock(&vdpasim->mutex); 357 old_ready = vq->ready; 358 vq->ready = ready; 359 if (vq->ready && !old_ready) { 360 vdpasim_queue_ready(vdpasim, idx); 361 } 362 mutex_unlock(&vdpasim->mutex); 363 } 364 365 static bool vdpasim_get_vq_ready(struct vdpa_device *vdpa, u16 idx) 366 { 367 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 368 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 369 370 return vq->ready; 371 } 372 373 static int vdpasim_set_vq_state(struct vdpa_device *vdpa, u16 idx, 374 const struct vdpa_vq_state *state) 375 { 376 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 377 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 378 struct vringh *vrh = &vq->vring; 379 380 mutex_lock(&vdpasim->mutex); 381 vrh->last_avail_idx = state->split.avail_index; 382 mutex_unlock(&vdpasim->mutex); 383 384 return 0; 385 } 386 387 static int vdpasim_get_vq_state(struct vdpa_device *vdpa, u16 idx, 388 struct vdpa_vq_state *state) 389 { 390 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 391 struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; 392 struct vringh *vrh = &vq->vring; 393 394 state->split.avail_index = vrh->last_avail_idx; 395 return 0; 396 } 397 398 static int vdpasim_get_vq_stats(struct vdpa_device *vdpa, u16 idx, 399 struct sk_buff *msg, 400 struct netlink_ext_ack *extack) 401 { 402 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 403 404 if (vdpasim->dev_attr.get_stats) 405 return vdpasim->dev_attr.get_stats(vdpasim, idx, 406 msg, extack); 407 return -EOPNOTSUPP; 408 } 409 410 static u32 vdpasim_get_vq_align(struct vdpa_device *vdpa) 411 { 412 return VDPASIM_QUEUE_ALIGN; 413 } 414 415 static u32 vdpasim_get_vq_group(struct vdpa_device *vdpa, u16 idx) 416 { 417 /* RX and TX belongs to group 0, CVQ belongs to group 1 */ 418 if (idx == 2) 419 return 1; 420 else 421 return 0; 422 } 423 424 static u64 vdpasim_get_device_features(struct vdpa_device *vdpa) 425 { 426 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 427 428 return vdpasim->dev_attr.supported_features; 429 } 430 431 static u64 vdpasim_get_backend_features(const struct vdpa_device *vdpa) 432 { 433 return BIT_ULL(VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK); 434 } 435 436 static int vdpasim_set_driver_features(struct vdpa_device *vdpa, u64 features) 437 { 438 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 439 440 /* DMA mapping must be done by driver */ 441 if (!(features & (1ULL << VIRTIO_F_ACCESS_PLATFORM))) 442 return -EINVAL; 443 444 vdpasim->features = features & vdpasim->dev_attr.supported_features; 445 446 return 0; 447 } 448 449 static u64 vdpasim_get_driver_features(struct vdpa_device *vdpa) 450 { 451 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 452 453 return vdpasim->features; 454 } 455 456 static void vdpasim_set_config_cb(struct vdpa_device *vdpa, 457 struct vdpa_callback *cb) 458 { 459 /* We don't support config interrupt */ 460 } 461 462 static u16 vdpasim_get_vq_num_max(struct vdpa_device *vdpa) 463 { 464 return VDPASIM_QUEUE_MAX; 465 } 466 467 static u32 vdpasim_get_device_id(struct vdpa_device *vdpa) 468 { 469 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 470 471 return vdpasim->dev_attr.id; 472 } 473 474 static u32 vdpasim_get_vendor_id(struct vdpa_device *vdpa) 475 { 476 return VDPASIM_VENDOR_ID; 477 } 478 479 static u8 vdpasim_get_status(struct vdpa_device *vdpa) 480 { 481 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 482 u8 status; 483 484 mutex_lock(&vdpasim->mutex); 485 status = vdpasim->status; 486 mutex_unlock(&vdpasim->mutex); 487 488 return status; 489 } 490 491 static void vdpasim_set_status(struct vdpa_device *vdpa, u8 status) 492 { 493 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 494 495 mutex_lock(&vdpasim->mutex); 496 vdpasim->status = status; 497 vdpasim->running = (status & VIRTIO_CONFIG_S_DRIVER_OK) != 0; 498 mutex_unlock(&vdpasim->mutex); 499 } 500 501 static int vdpasim_compat_reset(struct vdpa_device *vdpa, u32 flags) 502 { 503 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 504 505 mutex_lock(&vdpasim->mutex); 506 vdpasim->status = 0; 507 vdpasim_do_reset(vdpasim, flags); 508 mutex_unlock(&vdpasim->mutex); 509 510 return 0; 511 } 512 513 static int vdpasim_reset(struct vdpa_device *vdpa) 514 { 515 return vdpasim_compat_reset(vdpa, 0); 516 } 517 518 static int vdpasim_suspend(struct vdpa_device *vdpa) 519 { 520 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 521 522 mutex_lock(&vdpasim->mutex); 523 vdpasim->running = false; 524 mutex_unlock(&vdpasim->mutex); 525 526 return 0; 527 } 528 529 static int vdpasim_resume(struct vdpa_device *vdpa) 530 { 531 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 532 int i; 533 534 mutex_lock(&vdpasim->mutex); 535 vdpasim->running = true; 536 537 if (vdpasim->pending_kick) { 538 /* Process pending descriptors */ 539 for (i = 0; i < vdpasim->dev_attr.nvqs; ++i) 540 vdpasim_kick_vq(vdpa, i); 541 542 vdpasim->pending_kick = false; 543 } 544 545 mutex_unlock(&vdpasim->mutex); 546 547 return 0; 548 } 549 550 static size_t vdpasim_get_config_size(struct vdpa_device *vdpa) 551 { 552 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 553 554 return vdpasim->dev_attr.config_size; 555 } 556 557 static void vdpasim_get_config(struct vdpa_device *vdpa, unsigned int offset, 558 void *buf, unsigned int len) 559 { 560 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 561 562 if (offset + len > vdpasim->dev_attr.config_size) 563 return; 564 565 if (vdpasim->dev_attr.get_config) 566 vdpasim->dev_attr.get_config(vdpasim, vdpasim->config); 567 568 memcpy(buf, vdpasim->config + offset, len); 569 } 570 571 static void vdpasim_set_config(struct vdpa_device *vdpa, unsigned int offset, 572 const void *buf, unsigned int len) 573 { 574 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 575 576 if (offset + len > vdpasim->dev_attr.config_size) 577 return; 578 579 memcpy(vdpasim->config + offset, buf, len); 580 581 if (vdpasim->dev_attr.set_config) 582 vdpasim->dev_attr.set_config(vdpasim, vdpasim->config); 583 } 584 585 static u32 vdpasim_get_generation(struct vdpa_device *vdpa) 586 { 587 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 588 589 return vdpasim->generation; 590 } 591 592 static struct vdpa_iova_range vdpasim_get_iova_range(struct vdpa_device *vdpa) 593 { 594 struct vdpa_iova_range range = { 595 .first = 0ULL, 596 .last = ULLONG_MAX, 597 }; 598 599 return range; 600 } 601 602 static int vdpasim_set_group_asid(struct vdpa_device *vdpa, unsigned int group, 603 unsigned int asid) 604 { 605 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 606 struct vhost_iotlb *iommu; 607 int i; 608 609 if (group > vdpasim->dev_attr.ngroups) 610 return -EINVAL; 611 612 if (asid >= vdpasim->dev_attr.nas) 613 return -EINVAL; 614 615 iommu = &vdpasim->iommu[asid]; 616 617 mutex_lock(&vdpasim->mutex); 618 619 for (i = 0; i < vdpasim->dev_attr.nvqs; i++) 620 if (vdpasim_get_vq_group(vdpa, i) == group) 621 vringh_set_iotlb(&vdpasim->vqs[i].vring, iommu, 622 &vdpasim->iommu_lock); 623 624 mutex_unlock(&vdpasim->mutex); 625 626 return 0; 627 } 628 629 static int vdpasim_set_map(struct vdpa_device *vdpa, unsigned int asid, 630 struct vhost_iotlb *iotlb) 631 { 632 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 633 struct vhost_iotlb_map *map; 634 struct vhost_iotlb *iommu; 635 u64 start = 0ULL, last = 0ULL - 1; 636 int ret; 637 638 if (asid >= vdpasim->dev_attr.nas) 639 return -EINVAL; 640 641 spin_lock(&vdpasim->iommu_lock); 642 643 iommu = &vdpasim->iommu[asid]; 644 vhost_iotlb_reset(iommu); 645 vdpasim->iommu_pt[asid] = false; 646 647 for (map = vhost_iotlb_itree_first(iotlb, start, last); map; 648 map = vhost_iotlb_itree_next(map, start, last)) { 649 ret = vhost_iotlb_add_range(iommu, map->start, 650 map->last, map->addr, map->perm); 651 if (ret) 652 goto err; 653 } 654 spin_unlock(&vdpasim->iommu_lock); 655 return 0; 656 657 err: 658 vhost_iotlb_reset(iommu); 659 spin_unlock(&vdpasim->iommu_lock); 660 return ret; 661 } 662 663 static int vdpasim_reset_map(struct vdpa_device *vdpa, unsigned int asid) 664 { 665 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 666 667 if (asid >= vdpasim->dev_attr.nas) 668 return -EINVAL; 669 670 spin_lock(&vdpasim->iommu_lock); 671 if (vdpasim->iommu_pt[asid]) 672 goto out; 673 vhost_iotlb_reset(&vdpasim->iommu[asid]); 674 vhost_iotlb_add_range(&vdpasim->iommu[asid], 0, ULONG_MAX, 675 0, VHOST_MAP_RW); 676 vdpasim->iommu_pt[asid] = true; 677 out: 678 spin_unlock(&vdpasim->iommu_lock); 679 return 0; 680 } 681 682 static int vdpasim_bind_mm(struct vdpa_device *vdpa, struct mm_struct *mm) 683 { 684 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 685 struct vdpasim_mm_work mm_work; 686 687 mm_work.vdpasim = vdpasim; 688 mm_work.mm_to_bind = mm; 689 690 vdpasim_worker_change_mm_sync(vdpasim, &mm_work); 691 692 return mm_work.ret; 693 } 694 695 static void vdpasim_unbind_mm(struct vdpa_device *vdpa) 696 { 697 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 698 struct vdpasim_mm_work mm_work; 699 700 mm_work.vdpasim = vdpasim; 701 mm_work.mm_to_bind = NULL; 702 703 vdpasim_worker_change_mm_sync(vdpasim, &mm_work); 704 } 705 706 static int vdpasim_dma_map(struct vdpa_device *vdpa, unsigned int asid, 707 u64 iova, u64 size, 708 u64 pa, u32 perm, void *opaque) 709 { 710 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 711 int ret; 712 713 if (asid >= vdpasim->dev_attr.nas) 714 return -EINVAL; 715 716 spin_lock(&vdpasim->iommu_lock); 717 if (vdpasim->iommu_pt[asid]) { 718 vhost_iotlb_reset(&vdpasim->iommu[asid]); 719 vdpasim->iommu_pt[asid] = false; 720 } 721 ret = vhost_iotlb_add_range_ctx(&vdpasim->iommu[asid], iova, 722 iova + size - 1, pa, perm, opaque); 723 spin_unlock(&vdpasim->iommu_lock); 724 725 return ret; 726 } 727 728 static int vdpasim_dma_unmap(struct vdpa_device *vdpa, unsigned int asid, 729 u64 iova, u64 size) 730 { 731 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 732 733 if (asid >= vdpasim->dev_attr.nas) 734 return -EINVAL; 735 736 if (vdpasim->iommu_pt[asid]) { 737 vhost_iotlb_reset(&vdpasim->iommu[asid]); 738 vdpasim->iommu_pt[asid] = false; 739 } 740 741 spin_lock(&vdpasim->iommu_lock); 742 vhost_iotlb_del_range(&vdpasim->iommu[asid], iova, iova + size - 1); 743 spin_unlock(&vdpasim->iommu_lock); 744 745 return 0; 746 } 747 748 static void vdpasim_free(struct vdpa_device *vdpa) 749 { 750 struct vdpasim *vdpasim = vdpa_to_sim(vdpa); 751 int i; 752 753 kthread_cancel_work_sync(&vdpasim->work); 754 kthread_destroy_worker(vdpasim->worker); 755 756 for (i = 0; i < vdpasim->dev_attr.nvqs; i++) { 757 vringh_kiov_cleanup(&vdpasim->vqs[i].out_iov); 758 vringh_kiov_cleanup(&vdpasim->vqs[i].in_iov); 759 } 760 761 vdpasim->dev_attr.free(vdpasim); 762 763 for (i = 0; i < vdpasim->dev_attr.nas; i++) 764 vhost_iotlb_reset(&vdpasim->iommu[i]); 765 kfree(vdpasim->iommu); 766 kfree(vdpasim->iommu_pt); 767 kfree(vdpasim->vqs); 768 kfree(vdpasim->config); 769 } 770 771 static const struct vdpa_config_ops vdpasim_config_ops = { 772 .set_vq_address = vdpasim_set_vq_address, 773 .set_vq_num = vdpasim_set_vq_num, 774 .kick_vq = vdpasim_kick_vq, 775 .set_vq_cb = vdpasim_set_vq_cb, 776 .set_vq_ready = vdpasim_set_vq_ready, 777 .get_vq_ready = vdpasim_get_vq_ready, 778 .set_vq_state = vdpasim_set_vq_state, 779 .get_vendor_vq_stats = vdpasim_get_vq_stats, 780 .get_vq_state = vdpasim_get_vq_state, 781 .get_vq_align = vdpasim_get_vq_align, 782 .get_vq_group = vdpasim_get_vq_group, 783 .get_device_features = vdpasim_get_device_features, 784 .get_backend_features = vdpasim_get_backend_features, 785 .set_driver_features = vdpasim_set_driver_features, 786 .get_driver_features = vdpasim_get_driver_features, 787 .set_config_cb = vdpasim_set_config_cb, 788 .get_vq_num_max = vdpasim_get_vq_num_max, 789 .get_vq_size = vdpasim_get_vq_size, 790 .get_device_id = vdpasim_get_device_id, 791 .get_vendor_id = vdpasim_get_vendor_id, 792 .get_status = vdpasim_get_status, 793 .set_status = vdpasim_set_status, 794 .reset = vdpasim_reset, 795 .compat_reset = vdpasim_compat_reset, 796 .suspend = vdpasim_suspend, 797 .resume = vdpasim_resume, 798 .get_config_size = vdpasim_get_config_size, 799 .get_config = vdpasim_get_config, 800 .set_config = vdpasim_set_config, 801 .get_generation = vdpasim_get_generation, 802 .get_iova_range = vdpasim_get_iova_range, 803 .set_group_asid = vdpasim_set_group_asid, 804 .dma_map = vdpasim_dma_map, 805 .dma_unmap = vdpasim_dma_unmap, 806 .reset_map = vdpasim_reset_map, 807 .bind_mm = vdpasim_bind_mm, 808 .unbind_mm = vdpasim_unbind_mm, 809 .free = vdpasim_free, 810 }; 811 812 static const struct vdpa_config_ops vdpasim_batch_config_ops = { 813 .set_vq_address = vdpasim_set_vq_address, 814 .set_vq_num = vdpasim_set_vq_num, 815 .kick_vq = vdpasim_kick_vq, 816 .set_vq_cb = vdpasim_set_vq_cb, 817 .set_vq_ready = vdpasim_set_vq_ready, 818 .get_vq_ready = vdpasim_get_vq_ready, 819 .set_vq_state = vdpasim_set_vq_state, 820 .get_vendor_vq_stats = vdpasim_get_vq_stats, 821 .get_vq_state = vdpasim_get_vq_state, 822 .get_vq_align = vdpasim_get_vq_align, 823 .get_vq_group = vdpasim_get_vq_group, 824 .get_device_features = vdpasim_get_device_features, 825 .get_backend_features = vdpasim_get_backend_features, 826 .set_driver_features = vdpasim_set_driver_features, 827 .get_driver_features = vdpasim_get_driver_features, 828 .set_config_cb = vdpasim_set_config_cb, 829 .get_vq_num_max = vdpasim_get_vq_num_max, 830 .get_device_id = vdpasim_get_device_id, 831 .get_vendor_id = vdpasim_get_vendor_id, 832 .get_status = vdpasim_get_status, 833 .set_status = vdpasim_set_status, 834 .reset = vdpasim_reset, 835 .compat_reset = vdpasim_compat_reset, 836 .suspend = vdpasim_suspend, 837 .resume = vdpasim_resume, 838 .get_config_size = vdpasim_get_config_size, 839 .get_config = vdpasim_get_config, 840 .set_config = vdpasim_set_config, 841 .get_generation = vdpasim_get_generation, 842 .get_iova_range = vdpasim_get_iova_range, 843 .set_group_asid = vdpasim_set_group_asid, 844 .set_map = vdpasim_set_map, 845 .reset_map = vdpasim_reset_map, 846 .bind_mm = vdpasim_bind_mm, 847 .unbind_mm = vdpasim_unbind_mm, 848 .free = vdpasim_free, 849 }; 850 851 MODULE_VERSION(DRV_VERSION); 852 MODULE_LICENSE(DRV_LICENSE); 853 MODULE_AUTHOR(DRV_AUTHOR); 854 MODULE_DESCRIPTION(DRV_DESC); 855