1 /* 2 * Copyright 2018 Red Hat Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 #include "nouveau_svm.h" 23 #include "nouveau_drv.h" 24 #include "nouveau_chan.h" 25 #include "nouveau_dmem.h" 26 27 #include <nvif/notify.h> 28 #include <nvif/object.h> 29 #include <nvif/vmm.h> 30 31 #include <nvif/class.h> 32 #include <nvif/clb069.h> 33 #include <nvif/ifc00d.h> 34 35 #include <linux/sched/mm.h> 36 #include <linux/sort.h> 37 #include <linux/hmm.h> 38 #include <linux/memremap.h> 39 #include <linux/rmap.h> 40 41 struct nouveau_svm { 42 struct nouveau_drm *drm; 43 struct mutex mutex; 44 struct list_head inst; 45 46 struct nouveau_svm_fault_buffer { 47 int id; 48 struct nvif_object object; 49 u32 entries; 50 u32 getaddr; 51 u32 putaddr; 52 u32 get; 53 u32 put; 54 struct nvif_notify notify; 55 56 struct nouveau_svm_fault { 57 u64 inst; 58 u64 addr; 59 u64 time; 60 u32 engine; 61 u8 gpc; 62 u8 hub; 63 u8 access; 64 u8 client; 65 u8 fault; 66 struct nouveau_svmm *svmm; 67 } **fault; 68 int fault_nr; 69 } buffer[1]; 70 }; 71 72 #define FAULT_ACCESS_READ 0 73 #define FAULT_ACCESS_WRITE 1 74 #define FAULT_ACCESS_ATOMIC 2 75 #define FAULT_ACCESS_PREFETCH 3 76 77 #define SVM_DBG(s,f,a...) NV_DEBUG((s)->drm, "svm: "f"\n", ##a) 78 #define SVM_ERR(s,f,a...) NV_WARN((s)->drm, "svm: "f"\n", ##a) 79 80 struct nouveau_pfnmap_args { 81 struct nvif_ioctl_v0 i; 82 struct nvif_ioctl_mthd_v0 m; 83 struct nvif_vmm_pfnmap_v0 p; 84 }; 85 86 struct nouveau_ivmm { 87 struct nouveau_svmm *svmm; 88 u64 inst; 89 struct list_head head; 90 }; 91 92 static struct nouveau_ivmm * 93 nouveau_ivmm_find(struct nouveau_svm *svm, u64 inst) 94 { 95 struct nouveau_ivmm *ivmm; 96 list_for_each_entry(ivmm, &svm->inst, head) { 97 if (ivmm->inst == inst) 98 return ivmm; 99 } 100 return NULL; 101 } 102 103 #define SVMM_DBG(s,f,a...) \ 104 NV_DEBUG((s)->vmm->cli->drm, "svm-%p: "f"\n", (s), ##a) 105 #define SVMM_ERR(s,f,a...) \ 106 NV_WARN((s)->vmm->cli->drm, "svm-%p: "f"\n", (s), ##a) 107 108 int 109 nouveau_svmm_bind(struct drm_device *dev, void *data, 110 struct drm_file *file_priv) 111 { 112 struct nouveau_cli *cli = nouveau_cli(file_priv); 113 struct drm_nouveau_svm_bind *args = data; 114 unsigned target, cmd, priority; 115 unsigned long addr, end; 116 struct mm_struct *mm; 117 118 args->va_start &= PAGE_MASK; 119 args->va_end = ALIGN(args->va_end, PAGE_SIZE); 120 121 /* Sanity check arguments */ 122 if (args->reserved0 || args->reserved1) 123 return -EINVAL; 124 if (args->header & (~NOUVEAU_SVM_BIND_VALID_MASK)) 125 return -EINVAL; 126 if (args->va_start >= args->va_end) 127 return -EINVAL; 128 129 cmd = args->header >> NOUVEAU_SVM_BIND_COMMAND_SHIFT; 130 cmd &= NOUVEAU_SVM_BIND_COMMAND_MASK; 131 switch (cmd) { 132 case NOUVEAU_SVM_BIND_COMMAND__MIGRATE: 133 break; 134 default: 135 return -EINVAL; 136 } 137 138 priority = args->header >> NOUVEAU_SVM_BIND_PRIORITY_SHIFT; 139 priority &= NOUVEAU_SVM_BIND_PRIORITY_MASK; 140 141 /* FIXME support CPU target ie all target value < GPU_VRAM */ 142 target = args->header >> NOUVEAU_SVM_BIND_TARGET_SHIFT; 143 target &= NOUVEAU_SVM_BIND_TARGET_MASK; 144 switch (target) { 145 case NOUVEAU_SVM_BIND_TARGET__GPU_VRAM: 146 break; 147 default: 148 return -EINVAL; 149 } 150 151 /* 152 * FIXME: For now refuse non 0 stride, we need to change the migrate 153 * kernel function to handle stride to avoid to create a mess within 154 * each device driver. 155 */ 156 if (args->stride) 157 return -EINVAL; 158 159 /* 160 * Ok we are ask to do something sane, for now we only support migrate 161 * commands but we will add things like memory policy (what to do on 162 * page fault) and maybe some other commands. 163 */ 164 165 mm = get_task_mm(current); 166 if (!mm) { 167 return -EINVAL; 168 } 169 mmap_read_lock(mm); 170 171 if (!cli->svm.svmm) { 172 mmap_read_unlock(mm); 173 mmput(mm); 174 return -EINVAL; 175 } 176 177 for (addr = args->va_start, end = args->va_end; addr < end;) { 178 struct vm_area_struct *vma; 179 unsigned long next; 180 181 vma = find_vma_intersection(mm, addr, end); 182 if (!vma) 183 break; 184 185 addr = max(addr, vma->vm_start); 186 next = min(vma->vm_end, end); 187 /* This is a best effort so we ignore errors */ 188 nouveau_dmem_migrate_vma(cli->drm, cli->svm.svmm, vma, addr, 189 next); 190 addr = next; 191 } 192 193 /* 194 * FIXME Return the number of page we have migrated, again we need to 195 * update the migrate API to return that information so that we can 196 * report it to user space. 197 */ 198 args->result = 0; 199 200 mmap_read_unlock(mm); 201 mmput(mm); 202 203 return 0; 204 } 205 206 /* Unlink channel instance from SVMM. */ 207 void 208 nouveau_svmm_part(struct nouveau_svmm *svmm, u64 inst) 209 { 210 struct nouveau_ivmm *ivmm; 211 if (svmm) { 212 mutex_lock(&svmm->vmm->cli->drm->svm->mutex); 213 ivmm = nouveau_ivmm_find(svmm->vmm->cli->drm->svm, inst); 214 if (ivmm) { 215 list_del(&ivmm->head); 216 kfree(ivmm); 217 } 218 mutex_unlock(&svmm->vmm->cli->drm->svm->mutex); 219 } 220 } 221 222 /* Link channel instance to SVMM. */ 223 int 224 nouveau_svmm_join(struct nouveau_svmm *svmm, u64 inst) 225 { 226 struct nouveau_ivmm *ivmm; 227 if (svmm) { 228 if (!(ivmm = kmalloc(sizeof(*ivmm), GFP_KERNEL))) 229 return -ENOMEM; 230 ivmm->svmm = svmm; 231 ivmm->inst = inst; 232 233 mutex_lock(&svmm->vmm->cli->drm->svm->mutex); 234 list_add(&ivmm->head, &svmm->vmm->cli->drm->svm->inst); 235 mutex_unlock(&svmm->vmm->cli->drm->svm->mutex); 236 } 237 return 0; 238 } 239 240 /* Invalidate SVMM address-range on GPU. */ 241 void 242 nouveau_svmm_invalidate(struct nouveau_svmm *svmm, u64 start, u64 limit) 243 { 244 if (limit > start) { 245 nvif_object_mthd(&svmm->vmm->vmm.object, NVIF_VMM_V0_PFNCLR, 246 &(struct nvif_vmm_pfnclr_v0) { 247 .addr = start, 248 .size = limit - start, 249 }, sizeof(struct nvif_vmm_pfnclr_v0)); 250 } 251 } 252 253 static int 254 nouveau_svmm_invalidate_range_start(struct mmu_notifier *mn, 255 const struct mmu_notifier_range *update) 256 { 257 struct nouveau_svmm *svmm = 258 container_of(mn, struct nouveau_svmm, notifier); 259 unsigned long start = update->start; 260 unsigned long limit = update->end; 261 262 if (!mmu_notifier_range_blockable(update)) 263 return -EAGAIN; 264 265 SVMM_DBG(svmm, "invalidate %016lx-%016lx", start, limit); 266 267 mutex_lock(&svmm->mutex); 268 if (unlikely(!svmm->vmm)) 269 goto out; 270 271 /* 272 * Ignore invalidation callbacks for device private pages since 273 * the invalidation is handled as part of the migration process. 274 */ 275 if (update->event == MMU_NOTIFY_MIGRATE && 276 update->owner == svmm->vmm->cli->drm->dev) 277 goto out; 278 279 if (limit > svmm->unmanaged.start && start < svmm->unmanaged.limit) { 280 if (start < svmm->unmanaged.start) { 281 nouveau_svmm_invalidate(svmm, start, 282 svmm->unmanaged.limit); 283 } 284 start = svmm->unmanaged.limit; 285 } 286 287 nouveau_svmm_invalidate(svmm, start, limit); 288 289 out: 290 mutex_unlock(&svmm->mutex); 291 return 0; 292 } 293 294 static void nouveau_svmm_free_notifier(struct mmu_notifier *mn) 295 { 296 kfree(container_of(mn, struct nouveau_svmm, notifier)); 297 } 298 299 static const struct mmu_notifier_ops nouveau_mn_ops = { 300 .invalidate_range_start = nouveau_svmm_invalidate_range_start, 301 .free_notifier = nouveau_svmm_free_notifier, 302 }; 303 304 void 305 nouveau_svmm_fini(struct nouveau_svmm **psvmm) 306 { 307 struct nouveau_svmm *svmm = *psvmm; 308 if (svmm) { 309 mutex_lock(&svmm->mutex); 310 svmm->vmm = NULL; 311 mutex_unlock(&svmm->mutex); 312 mmu_notifier_put(&svmm->notifier); 313 *psvmm = NULL; 314 } 315 } 316 317 int 318 nouveau_svmm_init(struct drm_device *dev, void *data, 319 struct drm_file *file_priv) 320 { 321 struct nouveau_cli *cli = nouveau_cli(file_priv); 322 struct nouveau_svmm *svmm; 323 struct drm_nouveau_svm_init *args = data; 324 int ret; 325 326 /* We need to fail if svm is disabled */ 327 if (!cli->drm->svm) 328 return -ENOSYS; 329 330 /* Allocate tracking for SVM-enabled VMM. */ 331 if (!(svmm = kzalloc(sizeof(*svmm), GFP_KERNEL))) 332 return -ENOMEM; 333 svmm->vmm = &cli->svm; 334 svmm->unmanaged.start = args->unmanaged_addr; 335 svmm->unmanaged.limit = args->unmanaged_addr + args->unmanaged_size; 336 mutex_init(&svmm->mutex); 337 338 /* Check that SVM isn't already enabled for the client. */ 339 mutex_lock(&cli->mutex); 340 if (cli->svm.cli) { 341 ret = -EBUSY; 342 goto out_free; 343 } 344 345 /* Allocate a new GPU VMM that can support SVM (managed by the 346 * client, with replayable faults enabled). 347 * 348 * All future channel/memory allocations will make use of this 349 * VMM instead of the standard one. 350 */ 351 ret = nvif_vmm_ctor(&cli->mmu, "svmVmm", 352 cli->vmm.vmm.object.oclass, true, 353 args->unmanaged_addr, args->unmanaged_size, 354 &(struct gp100_vmm_v0) { 355 .fault_replay = true, 356 }, sizeof(struct gp100_vmm_v0), &cli->svm.vmm); 357 if (ret) 358 goto out_free; 359 360 mmap_write_lock(current->mm); 361 svmm->notifier.ops = &nouveau_mn_ops; 362 ret = __mmu_notifier_register(&svmm->notifier, current->mm); 363 if (ret) 364 goto out_mm_unlock; 365 /* Note, ownership of svmm transfers to mmu_notifier */ 366 367 cli->svm.svmm = svmm; 368 cli->svm.cli = cli; 369 mmap_write_unlock(current->mm); 370 mutex_unlock(&cli->mutex); 371 return 0; 372 373 out_mm_unlock: 374 mmap_write_unlock(current->mm); 375 out_free: 376 mutex_unlock(&cli->mutex); 377 kfree(svmm); 378 return ret; 379 } 380 381 /* Issue fault replay for GPU to retry accesses that faulted previously. */ 382 static void 383 nouveau_svm_fault_replay(struct nouveau_svm *svm) 384 { 385 SVM_DBG(svm, "replay"); 386 WARN_ON(nvif_object_mthd(&svm->drm->client.vmm.vmm.object, 387 GP100_VMM_VN_FAULT_REPLAY, 388 &(struct gp100_vmm_fault_replay_vn) {}, 389 sizeof(struct gp100_vmm_fault_replay_vn))); 390 } 391 392 /* Cancel a replayable fault that could not be handled. 393 * 394 * Cancelling the fault will trigger recovery to reset the engine 395 * and kill the offending channel (ie. GPU SIGSEGV). 396 */ 397 static void 398 nouveau_svm_fault_cancel(struct nouveau_svm *svm, 399 u64 inst, u8 hub, u8 gpc, u8 client) 400 { 401 SVM_DBG(svm, "cancel %016llx %d %02x %02x", inst, hub, gpc, client); 402 WARN_ON(nvif_object_mthd(&svm->drm->client.vmm.vmm.object, 403 GP100_VMM_VN_FAULT_CANCEL, 404 &(struct gp100_vmm_fault_cancel_v0) { 405 .hub = hub, 406 .gpc = gpc, 407 .client = client, 408 .inst = inst, 409 }, sizeof(struct gp100_vmm_fault_cancel_v0))); 410 } 411 412 static void 413 nouveau_svm_fault_cancel_fault(struct nouveau_svm *svm, 414 struct nouveau_svm_fault *fault) 415 { 416 nouveau_svm_fault_cancel(svm, fault->inst, 417 fault->hub, 418 fault->gpc, 419 fault->client); 420 } 421 422 static int 423 nouveau_svm_fault_priority(u8 fault) 424 { 425 switch (fault) { 426 case FAULT_ACCESS_PREFETCH: 427 return 0; 428 case FAULT_ACCESS_READ: 429 return 1; 430 case FAULT_ACCESS_WRITE: 431 return 2; 432 case FAULT_ACCESS_ATOMIC: 433 return 3; 434 default: 435 WARN_ON_ONCE(1); 436 return -1; 437 } 438 } 439 440 static int 441 nouveau_svm_fault_cmp(const void *a, const void *b) 442 { 443 const struct nouveau_svm_fault *fa = *(struct nouveau_svm_fault **)a; 444 const struct nouveau_svm_fault *fb = *(struct nouveau_svm_fault **)b; 445 int ret; 446 if ((ret = (s64)fa->inst - fb->inst)) 447 return ret; 448 if ((ret = (s64)fa->addr - fb->addr)) 449 return ret; 450 return nouveau_svm_fault_priority(fa->access) - 451 nouveau_svm_fault_priority(fb->access); 452 } 453 454 static void 455 nouveau_svm_fault_cache(struct nouveau_svm *svm, 456 struct nouveau_svm_fault_buffer *buffer, u32 offset) 457 { 458 struct nvif_object *memory = &buffer->object; 459 const u32 instlo = nvif_rd32(memory, offset + 0x00); 460 const u32 insthi = nvif_rd32(memory, offset + 0x04); 461 const u32 addrlo = nvif_rd32(memory, offset + 0x08); 462 const u32 addrhi = nvif_rd32(memory, offset + 0x0c); 463 const u32 timelo = nvif_rd32(memory, offset + 0x10); 464 const u32 timehi = nvif_rd32(memory, offset + 0x14); 465 const u32 engine = nvif_rd32(memory, offset + 0x18); 466 const u32 info = nvif_rd32(memory, offset + 0x1c); 467 const u64 inst = (u64)insthi << 32 | instlo; 468 const u8 gpc = (info & 0x1f000000) >> 24; 469 const u8 hub = (info & 0x00100000) >> 20; 470 const u8 client = (info & 0x00007f00) >> 8; 471 struct nouveau_svm_fault *fault; 472 473 //XXX: i think we're supposed to spin waiting */ 474 if (WARN_ON(!(info & 0x80000000))) 475 return; 476 477 nvif_mask(memory, offset + 0x1c, 0x80000000, 0x00000000); 478 479 if (!buffer->fault[buffer->fault_nr]) { 480 fault = kmalloc(sizeof(*fault), GFP_KERNEL); 481 if (WARN_ON(!fault)) { 482 nouveau_svm_fault_cancel(svm, inst, hub, gpc, client); 483 return; 484 } 485 buffer->fault[buffer->fault_nr] = fault; 486 } 487 488 fault = buffer->fault[buffer->fault_nr++]; 489 fault->inst = inst; 490 fault->addr = (u64)addrhi << 32 | addrlo; 491 fault->time = (u64)timehi << 32 | timelo; 492 fault->engine = engine; 493 fault->gpc = gpc; 494 fault->hub = hub; 495 fault->access = (info & 0x000f0000) >> 16; 496 fault->client = client; 497 fault->fault = (info & 0x0000001f); 498 499 SVM_DBG(svm, "fault %016llx %016llx %02x", 500 fault->inst, fault->addr, fault->access); 501 } 502 503 struct svm_notifier { 504 struct mmu_interval_notifier notifier; 505 struct nouveau_svmm *svmm; 506 }; 507 508 static bool nouveau_svm_range_invalidate(struct mmu_interval_notifier *mni, 509 const struct mmu_notifier_range *range, 510 unsigned long cur_seq) 511 { 512 struct svm_notifier *sn = 513 container_of(mni, struct svm_notifier, notifier); 514 515 if (range->event == MMU_NOTIFY_EXCLUSIVE && 516 range->owner == sn->svmm->vmm->cli->drm->dev) 517 return true; 518 519 /* 520 * serializes the update to mni->invalidate_seq done by caller and 521 * prevents invalidation of the PTE from progressing while HW is being 522 * programmed. This is very hacky and only works because the normal 523 * notifier that does invalidation is always called after the range 524 * notifier. 525 */ 526 if (mmu_notifier_range_blockable(range)) 527 mutex_lock(&sn->svmm->mutex); 528 else if (!mutex_trylock(&sn->svmm->mutex)) 529 return false; 530 mmu_interval_set_seq(mni, cur_seq); 531 mutex_unlock(&sn->svmm->mutex); 532 return true; 533 } 534 535 static const struct mmu_interval_notifier_ops nouveau_svm_mni_ops = { 536 .invalidate = nouveau_svm_range_invalidate, 537 }; 538 539 static void nouveau_hmm_convert_pfn(struct nouveau_drm *drm, 540 struct hmm_range *range, 541 struct nouveau_pfnmap_args *args) 542 { 543 struct page *page; 544 545 /* 546 * The address prepared here is passed through nvif_object_ioctl() 547 * to an eventual DMA map in something like gp100_vmm_pgt_pfn() 548 * 549 * This is all just encoding the internal hmm representation into a 550 * different nouveau internal representation. 551 */ 552 if (!(range->hmm_pfns[0] & HMM_PFN_VALID)) { 553 args->p.phys[0] = 0; 554 return; 555 } 556 557 page = hmm_pfn_to_page(range->hmm_pfns[0]); 558 /* 559 * Only map compound pages to the GPU if the CPU is also mapping the 560 * page as a compound page. Otherwise, the PTE protections might not be 561 * consistent (e.g., CPU only maps part of a compound page). 562 * Note that the underlying page might still be larger than the 563 * CPU mapping (e.g., a PUD sized compound page partially mapped with 564 * a PMD sized page table entry). 565 */ 566 if (hmm_pfn_to_map_order(range->hmm_pfns[0])) { 567 unsigned long addr = args->p.addr; 568 569 args->p.page = hmm_pfn_to_map_order(range->hmm_pfns[0]) + 570 PAGE_SHIFT; 571 args->p.size = 1UL << args->p.page; 572 args->p.addr &= ~(args->p.size - 1); 573 page -= (addr - args->p.addr) >> PAGE_SHIFT; 574 } 575 if (is_device_private_page(page)) 576 args->p.phys[0] = nouveau_dmem_page_addr(page) | 577 NVIF_VMM_PFNMAP_V0_V | 578 NVIF_VMM_PFNMAP_V0_VRAM; 579 else 580 args->p.phys[0] = page_to_phys(page) | 581 NVIF_VMM_PFNMAP_V0_V | 582 NVIF_VMM_PFNMAP_V0_HOST; 583 if (range->hmm_pfns[0] & HMM_PFN_WRITE) 584 args->p.phys[0] |= NVIF_VMM_PFNMAP_V0_W; 585 } 586 587 static int nouveau_atomic_range_fault(struct nouveau_svmm *svmm, 588 struct nouveau_drm *drm, 589 struct nouveau_pfnmap_args *args, u32 size, 590 struct svm_notifier *notifier) 591 { 592 unsigned long timeout = 593 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 594 struct mm_struct *mm = svmm->notifier.mm; 595 struct page *page; 596 unsigned long start = args->p.addr; 597 unsigned long notifier_seq; 598 int ret = 0; 599 600 ret = mmu_interval_notifier_insert(¬ifier->notifier, mm, 601 args->p.addr, args->p.size, 602 &nouveau_svm_mni_ops); 603 if (ret) 604 return ret; 605 606 while (true) { 607 if (time_after(jiffies, timeout)) { 608 ret = -EBUSY; 609 goto out; 610 } 611 612 notifier_seq = mmu_interval_read_begin(¬ifier->notifier); 613 mmap_read_lock(mm); 614 ret = make_device_exclusive_range(mm, start, start + PAGE_SIZE, 615 &page, drm->dev); 616 mmap_read_unlock(mm); 617 if (ret <= 0 || !page) { 618 ret = -EINVAL; 619 goto out; 620 } 621 622 mutex_lock(&svmm->mutex); 623 if (!mmu_interval_read_retry(¬ifier->notifier, 624 notifier_seq)) 625 break; 626 mutex_unlock(&svmm->mutex); 627 } 628 629 /* Map the page on the GPU. */ 630 args->p.page = 12; 631 args->p.size = PAGE_SIZE; 632 args->p.addr = start; 633 args->p.phys[0] = page_to_phys(page) | 634 NVIF_VMM_PFNMAP_V0_V | 635 NVIF_VMM_PFNMAP_V0_W | 636 NVIF_VMM_PFNMAP_V0_A | 637 NVIF_VMM_PFNMAP_V0_HOST; 638 639 ret = nvif_object_ioctl(&svmm->vmm->vmm.object, args, size, NULL); 640 mutex_unlock(&svmm->mutex); 641 642 unlock_page(page); 643 put_page(page); 644 645 out: 646 mmu_interval_notifier_remove(¬ifier->notifier); 647 return ret; 648 } 649 650 static int nouveau_range_fault(struct nouveau_svmm *svmm, 651 struct nouveau_drm *drm, 652 struct nouveau_pfnmap_args *args, u32 size, 653 unsigned long hmm_flags, 654 struct svm_notifier *notifier) 655 { 656 unsigned long timeout = 657 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 658 /* Have HMM fault pages within the fault window to the GPU. */ 659 unsigned long hmm_pfns[1]; 660 struct hmm_range range = { 661 .notifier = ¬ifier->notifier, 662 .default_flags = hmm_flags, 663 .hmm_pfns = hmm_pfns, 664 .dev_private_owner = drm->dev, 665 }; 666 struct mm_struct *mm = svmm->notifier.mm; 667 int ret; 668 669 ret = mmu_interval_notifier_insert(¬ifier->notifier, mm, 670 args->p.addr, args->p.size, 671 &nouveau_svm_mni_ops); 672 if (ret) 673 return ret; 674 675 range.start = notifier->notifier.interval_tree.start; 676 range.end = notifier->notifier.interval_tree.last + 1; 677 678 while (true) { 679 if (time_after(jiffies, timeout)) { 680 ret = -EBUSY; 681 goto out; 682 } 683 684 range.notifier_seq = mmu_interval_read_begin(range.notifier); 685 mmap_read_lock(mm); 686 ret = hmm_range_fault(&range); 687 mmap_read_unlock(mm); 688 if (ret) { 689 if (ret == -EBUSY) 690 continue; 691 goto out; 692 } 693 694 mutex_lock(&svmm->mutex); 695 if (mmu_interval_read_retry(range.notifier, 696 range.notifier_seq)) { 697 mutex_unlock(&svmm->mutex); 698 continue; 699 } 700 break; 701 } 702 703 nouveau_hmm_convert_pfn(drm, &range, args); 704 705 ret = nvif_object_ioctl(&svmm->vmm->vmm.object, args, size, NULL); 706 mutex_unlock(&svmm->mutex); 707 708 out: 709 mmu_interval_notifier_remove(¬ifier->notifier); 710 711 return ret; 712 } 713 714 static int 715 nouveau_svm_fault(struct nvif_notify *notify) 716 { 717 struct nouveau_svm_fault_buffer *buffer = 718 container_of(notify, typeof(*buffer), notify); 719 struct nouveau_svm *svm = 720 container_of(buffer, typeof(*svm), buffer[buffer->id]); 721 struct nvif_object *device = &svm->drm->client.device.object; 722 struct nouveau_svmm *svmm; 723 struct { 724 struct nouveau_pfnmap_args i; 725 u64 phys[1]; 726 } args; 727 unsigned long hmm_flags; 728 u64 inst, start, limit; 729 int fi, fn; 730 int replay = 0, atomic = 0, ret; 731 732 /* Parse available fault buffer entries into a cache, and update 733 * the GET pointer so HW can reuse the entries. 734 */ 735 SVM_DBG(svm, "fault handler"); 736 if (buffer->get == buffer->put) { 737 buffer->put = nvif_rd32(device, buffer->putaddr); 738 buffer->get = nvif_rd32(device, buffer->getaddr); 739 if (buffer->get == buffer->put) 740 return NVIF_NOTIFY_KEEP; 741 } 742 buffer->fault_nr = 0; 743 744 SVM_DBG(svm, "get %08x put %08x", buffer->get, buffer->put); 745 while (buffer->get != buffer->put) { 746 nouveau_svm_fault_cache(svm, buffer, buffer->get * 0x20); 747 if (++buffer->get == buffer->entries) 748 buffer->get = 0; 749 } 750 nvif_wr32(device, buffer->getaddr, buffer->get); 751 SVM_DBG(svm, "%d fault(s) pending", buffer->fault_nr); 752 753 /* Sort parsed faults by instance pointer to prevent unnecessary 754 * instance to SVMM translations, followed by address and access 755 * type to reduce the amount of work when handling the faults. 756 */ 757 sort(buffer->fault, buffer->fault_nr, sizeof(*buffer->fault), 758 nouveau_svm_fault_cmp, NULL); 759 760 /* Lookup SVMM structure for each unique instance pointer. */ 761 mutex_lock(&svm->mutex); 762 for (fi = 0, svmm = NULL; fi < buffer->fault_nr; fi++) { 763 if (!svmm || buffer->fault[fi]->inst != inst) { 764 struct nouveau_ivmm *ivmm = 765 nouveau_ivmm_find(svm, buffer->fault[fi]->inst); 766 svmm = ivmm ? ivmm->svmm : NULL; 767 inst = buffer->fault[fi]->inst; 768 SVM_DBG(svm, "inst %016llx -> svm-%p", inst, svmm); 769 } 770 buffer->fault[fi]->svmm = svmm; 771 } 772 mutex_unlock(&svm->mutex); 773 774 /* Process list of faults. */ 775 args.i.i.version = 0; 776 args.i.i.type = NVIF_IOCTL_V0_MTHD; 777 args.i.m.version = 0; 778 args.i.m.method = NVIF_VMM_V0_PFNMAP; 779 args.i.p.version = 0; 780 781 for (fi = 0; fn = fi + 1, fi < buffer->fault_nr; fi = fn) { 782 struct svm_notifier notifier; 783 struct mm_struct *mm; 784 785 /* Cancel any faults from non-SVM channels. */ 786 if (!(svmm = buffer->fault[fi]->svmm)) { 787 nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]); 788 continue; 789 } 790 SVMM_DBG(svmm, "addr %016llx", buffer->fault[fi]->addr); 791 792 /* We try and group handling of faults within a small 793 * window into a single update. 794 */ 795 start = buffer->fault[fi]->addr; 796 limit = start + PAGE_SIZE; 797 if (start < svmm->unmanaged.limit) 798 limit = min_t(u64, limit, svmm->unmanaged.start); 799 800 /* 801 * Prepare the GPU-side update of all pages within the 802 * fault window, determining required pages and access 803 * permissions based on pending faults. 804 */ 805 args.i.p.addr = start; 806 args.i.p.page = PAGE_SHIFT; 807 args.i.p.size = PAGE_SIZE; 808 /* 809 * Determine required permissions based on GPU fault 810 * access flags. 811 */ 812 switch (buffer->fault[fi]->access) { 813 case 0: /* READ. */ 814 hmm_flags = HMM_PFN_REQ_FAULT; 815 break; 816 case 2: /* ATOMIC. */ 817 atomic = true; 818 break; 819 case 3: /* PREFETCH. */ 820 hmm_flags = 0; 821 break; 822 default: 823 hmm_flags = HMM_PFN_REQ_FAULT | HMM_PFN_REQ_WRITE; 824 break; 825 } 826 827 mm = svmm->notifier.mm; 828 if (!mmget_not_zero(mm)) { 829 nouveau_svm_fault_cancel_fault(svm, buffer->fault[fi]); 830 continue; 831 } 832 833 notifier.svmm = svmm; 834 if (atomic) 835 ret = nouveau_atomic_range_fault(svmm, svm->drm, 836 &args.i, sizeof(args), 837 ¬ifier); 838 else 839 ret = nouveau_range_fault(svmm, svm->drm, &args.i, 840 sizeof(args), hmm_flags, 841 ¬ifier); 842 mmput(mm); 843 844 limit = args.i.p.addr + args.i.p.size; 845 for (fn = fi; ++fn < buffer->fault_nr; ) { 846 /* It's okay to skip over duplicate addresses from the 847 * same SVMM as faults are ordered by access type such 848 * that only the first one needs to be handled. 849 * 850 * ie. WRITE faults appear first, thus any handling of 851 * pending READ faults will already be satisfied. 852 * But if a large page is mapped, make sure subsequent 853 * fault addresses have sufficient access permission. 854 */ 855 if (buffer->fault[fn]->svmm != svmm || 856 buffer->fault[fn]->addr >= limit || 857 (buffer->fault[fi]->access == FAULT_ACCESS_READ && 858 !(args.phys[0] & NVIF_VMM_PFNMAP_V0_V)) || 859 (buffer->fault[fi]->access != FAULT_ACCESS_READ && 860 buffer->fault[fi]->access != FAULT_ACCESS_PREFETCH && 861 !(args.phys[0] & NVIF_VMM_PFNMAP_V0_W)) || 862 (buffer->fault[fi]->access != FAULT_ACCESS_READ && 863 buffer->fault[fi]->access != FAULT_ACCESS_WRITE && 864 buffer->fault[fi]->access != FAULT_ACCESS_PREFETCH && 865 !(args.phys[0] & NVIF_VMM_PFNMAP_V0_A))) 866 break; 867 } 868 869 /* If handling failed completely, cancel all faults. */ 870 if (ret) { 871 while (fi < fn) { 872 struct nouveau_svm_fault *fault = 873 buffer->fault[fi++]; 874 875 nouveau_svm_fault_cancel_fault(svm, fault); 876 } 877 } else 878 replay++; 879 } 880 881 /* Issue fault replay to the GPU. */ 882 if (replay) 883 nouveau_svm_fault_replay(svm); 884 return NVIF_NOTIFY_KEEP; 885 } 886 887 static struct nouveau_pfnmap_args * 888 nouveau_pfns_to_args(void *pfns) 889 { 890 return container_of(pfns, struct nouveau_pfnmap_args, p.phys); 891 } 892 893 u64 * 894 nouveau_pfns_alloc(unsigned long npages) 895 { 896 struct nouveau_pfnmap_args *args; 897 898 args = kzalloc(struct_size(args, p.phys, npages), GFP_KERNEL); 899 if (!args) 900 return NULL; 901 902 args->i.type = NVIF_IOCTL_V0_MTHD; 903 args->m.method = NVIF_VMM_V0_PFNMAP; 904 args->p.page = PAGE_SHIFT; 905 906 return args->p.phys; 907 } 908 909 void 910 nouveau_pfns_free(u64 *pfns) 911 { 912 struct nouveau_pfnmap_args *args = nouveau_pfns_to_args(pfns); 913 914 kfree(args); 915 } 916 917 void 918 nouveau_pfns_map(struct nouveau_svmm *svmm, struct mm_struct *mm, 919 unsigned long addr, u64 *pfns, unsigned long npages) 920 { 921 struct nouveau_pfnmap_args *args = nouveau_pfns_to_args(pfns); 922 int ret; 923 924 args->p.addr = addr; 925 args->p.size = npages << PAGE_SHIFT; 926 927 mutex_lock(&svmm->mutex); 928 929 ret = nvif_object_ioctl(&svmm->vmm->vmm.object, args, 930 struct_size(args, p.phys, npages), NULL); 931 932 mutex_unlock(&svmm->mutex); 933 } 934 935 static void 936 nouveau_svm_fault_buffer_fini(struct nouveau_svm *svm, int id) 937 { 938 struct nouveau_svm_fault_buffer *buffer = &svm->buffer[id]; 939 nvif_notify_put(&buffer->notify); 940 } 941 942 static int 943 nouveau_svm_fault_buffer_init(struct nouveau_svm *svm, int id) 944 { 945 struct nouveau_svm_fault_buffer *buffer = &svm->buffer[id]; 946 struct nvif_object *device = &svm->drm->client.device.object; 947 buffer->get = nvif_rd32(device, buffer->getaddr); 948 buffer->put = nvif_rd32(device, buffer->putaddr); 949 SVM_DBG(svm, "get %08x put %08x (init)", buffer->get, buffer->put); 950 return nvif_notify_get(&buffer->notify); 951 } 952 953 static void 954 nouveau_svm_fault_buffer_dtor(struct nouveau_svm *svm, int id) 955 { 956 struct nouveau_svm_fault_buffer *buffer = &svm->buffer[id]; 957 int i; 958 959 if (buffer->fault) { 960 for (i = 0; buffer->fault[i] && i < buffer->entries; i++) 961 kfree(buffer->fault[i]); 962 kvfree(buffer->fault); 963 } 964 965 nouveau_svm_fault_buffer_fini(svm, id); 966 967 nvif_notify_dtor(&buffer->notify); 968 nvif_object_dtor(&buffer->object); 969 } 970 971 static int 972 nouveau_svm_fault_buffer_ctor(struct nouveau_svm *svm, s32 oclass, int id) 973 { 974 struct nouveau_svm_fault_buffer *buffer = &svm->buffer[id]; 975 struct nouveau_drm *drm = svm->drm; 976 struct nvif_object *device = &drm->client.device.object; 977 struct nvif_clb069_v0 args = {}; 978 int ret; 979 980 buffer->id = id; 981 982 ret = nvif_object_ctor(device, "svmFaultBuffer", 0, oclass, &args, 983 sizeof(args), &buffer->object); 984 if (ret < 0) { 985 SVM_ERR(svm, "Fault buffer allocation failed: %d", ret); 986 return ret; 987 } 988 989 nvif_object_map(&buffer->object, NULL, 0); 990 buffer->entries = args.entries; 991 buffer->getaddr = args.get; 992 buffer->putaddr = args.put; 993 994 ret = nvif_notify_ctor(&buffer->object, "svmFault", nouveau_svm_fault, 995 true, NVB069_V0_NTFY_FAULT, NULL, 0, 0, 996 &buffer->notify); 997 if (ret) 998 return ret; 999 1000 buffer->fault = kvcalloc(sizeof(*buffer->fault), buffer->entries, GFP_KERNEL); 1001 if (!buffer->fault) 1002 return -ENOMEM; 1003 1004 return nouveau_svm_fault_buffer_init(svm, id); 1005 } 1006 1007 void 1008 nouveau_svm_resume(struct nouveau_drm *drm) 1009 { 1010 struct nouveau_svm *svm = drm->svm; 1011 if (svm) 1012 nouveau_svm_fault_buffer_init(svm, 0); 1013 } 1014 1015 void 1016 nouveau_svm_suspend(struct nouveau_drm *drm) 1017 { 1018 struct nouveau_svm *svm = drm->svm; 1019 if (svm) 1020 nouveau_svm_fault_buffer_fini(svm, 0); 1021 } 1022 1023 void 1024 nouveau_svm_fini(struct nouveau_drm *drm) 1025 { 1026 struct nouveau_svm *svm = drm->svm; 1027 if (svm) { 1028 nouveau_svm_fault_buffer_dtor(svm, 0); 1029 kfree(drm->svm); 1030 drm->svm = NULL; 1031 } 1032 } 1033 1034 void 1035 nouveau_svm_init(struct nouveau_drm *drm) 1036 { 1037 static const struct nvif_mclass buffers[] = { 1038 { VOLTA_FAULT_BUFFER_A, 0 }, 1039 { MAXWELL_FAULT_BUFFER_A, 0 }, 1040 {} 1041 }; 1042 struct nouveau_svm *svm; 1043 int ret; 1044 1045 /* Disable on Volta and newer until channel recovery is fixed, 1046 * otherwise clients will have a trivial way to trash the GPU 1047 * for everyone. 1048 */ 1049 if (drm->client.device.info.family > NV_DEVICE_INFO_V0_PASCAL) 1050 return; 1051 1052 if (!(drm->svm = svm = kzalloc(sizeof(*drm->svm), GFP_KERNEL))) 1053 return; 1054 1055 drm->svm->drm = drm; 1056 mutex_init(&drm->svm->mutex); 1057 INIT_LIST_HEAD(&drm->svm->inst); 1058 1059 ret = nvif_mclass(&drm->client.device.object, buffers); 1060 if (ret < 0) { 1061 SVM_DBG(svm, "No supported fault buffer class"); 1062 nouveau_svm_fini(drm); 1063 return; 1064 } 1065 1066 ret = nouveau_svm_fault_buffer_ctor(svm, buffers[ret].oclass, 0); 1067 if (ret) { 1068 nouveau_svm_fini(drm); 1069 return; 1070 } 1071 1072 SVM_DBG(svm, "Initialised"); 1073 } 1074