1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2024 Intel Corporation 4 */ 5 6 #include <drm/drm_drv.h> 7 8 #include "xe_bo.h" 9 #include "xe_exec_queue_types.h" 10 #include "xe_gt_stats.h" 11 #include "xe_migrate.h" 12 #include "xe_module.h" 13 #include "xe_pm.h" 14 #include "xe_pt.h" 15 #include "xe_svm.h" 16 #include "xe_tile.h" 17 #include "xe_ttm_vram_mgr.h" 18 #include "xe_vm.h" 19 #include "xe_vm_types.h" 20 #include "xe_vram_types.h" 21 22 static bool xe_svm_range_in_vram(struct xe_svm_range *range) 23 { 24 /* 25 * Advisory only check whether the range is currently backed by VRAM 26 * memory. 27 */ 28 29 struct drm_gpusvm_pages_flags flags = { 30 /* Pairs with WRITE_ONCE in drm_gpusvm.c */ 31 .__flags = READ_ONCE(range->base.pages.flags.__flags), 32 }; 33 34 return flags.has_devmem_pages; 35 } 36 37 static bool xe_svm_range_has_vram_binding(struct xe_svm_range *range) 38 { 39 /* Not reliable without notifier lock */ 40 return xe_svm_range_in_vram(range) && range->tile_present; 41 } 42 43 static struct xe_vm *gpusvm_to_vm(struct drm_gpusvm *gpusvm) 44 { 45 return container_of(gpusvm, struct xe_vm, svm.gpusvm); 46 } 47 48 static struct xe_vm *range_to_vm(struct drm_gpusvm_range *r) 49 { 50 return gpusvm_to_vm(r->gpusvm); 51 } 52 53 #define range_debug(r__, operation__) \ 54 vm_dbg(&range_to_vm(&(r__)->base)->xe->drm, \ 55 "%s: asid=%u, gpusvm=%p, vram=%d,%d, seqno=%lu, " \ 56 "start=0x%014lx, end=0x%014lx, size=%lu", \ 57 (operation__), range_to_vm(&(r__)->base)->usm.asid, \ 58 (r__)->base.gpusvm, \ 59 xe_svm_range_in_vram((r__)) ? 1 : 0, \ 60 xe_svm_range_has_vram_binding((r__)) ? 1 : 0, \ 61 (r__)->base.pages.notifier_seq, \ 62 xe_svm_range_start((r__)), xe_svm_range_end((r__)), \ 63 xe_svm_range_size((r__))) 64 65 void xe_svm_range_debug(struct xe_svm_range *range, const char *operation) 66 { 67 range_debug(range, operation); 68 } 69 70 static struct drm_gpusvm_range * 71 xe_svm_range_alloc(struct drm_gpusvm *gpusvm) 72 { 73 struct xe_svm_range *range; 74 75 range = kzalloc(sizeof(*range), GFP_KERNEL); 76 if (!range) 77 return NULL; 78 79 INIT_LIST_HEAD(&range->garbage_collector_link); 80 xe_vm_get(gpusvm_to_vm(gpusvm)); 81 82 return &range->base; 83 } 84 85 static void xe_svm_range_free(struct drm_gpusvm_range *range) 86 { 87 xe_vm_put(range_to_vm(range)); 88 kfree(range); 89 } 90 91 static void 92 xe_svm_garbage_collector_add_range(struct xe_vm *vm, struct xe_svm_range *range, 93 const struct mmu_notifier_range *mmu_range) 94 { 95 struct xe_device *xe = vm->xe; 96 97 range_debug(range, "GARBAGE COLLECTOR ADD"); 98 99 drm_gpusvm_range_set_unmapped(&range->base, mmu_range); 100 101 spin_lock(&vm->svm.garbage_collector.lock); 102 if (list_empty(&range->garbage_collector_link)) 103 list_add_tail(&range->garbage_collector_link, 104 &vm->svm.garbage_collector.range_list); 105 spin_unlock(&vm->svm.garbage_collector.lock); 106 107 queue_work(xe->usm.pf_wq, &vm->svm.garbage_collector.work); 108 } 109 110 static void xe_svm_tlb_inval_count_stats_incr(struct xe_gt *gt) 111 { 112 xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_TLB_INVAL_COUNT, 1); 113 } 114 115 static u8 116 xe_svm_range_notifier_event_begin(struct xe_vm *vm, struct drm_gpusvm_range *r, 117 const struct mmu_notifier_range *mmu_range, 118 u64 *adj_start, u64 *adj_end) 119 { 120 struct xe_svm_range *range = to_xe_range(r); 121 struct xe_device *xe = vm->xe; 122 struct xe_tile *tile; 123 u8 tile_mask = 0; 124 u8 id; 125 126 xe_svm_assert_in_notifier(vm); 127 128 range_debug(range, "NOTIFIER"); 129 130 /* Skip if already unmapped or if no binding exist */ 131 if (range->base.pages.flags.unmapped || !range->tile_present) 132 return 0; 133 134 range_debug(range, "NOTIFIER - EXECUTE"); 135 136 /* Adjust invalidation to range boundaries */ 137 *adj_start = min(xe_svm_range_start(range), mmu_range->start); 138 *adj_end = max(xe_svm_range_end(range), mmu_range->end); 139 140 /* 141 * XXX: Ideally would zap PTEs in one shot in xe_svm_invalidate but the 142 * invalidation code can't correctly cope with sparse ranges or 143 * invalidations spanning multiple ranges. 144 */ 145 for_each_tile(tile, xe, id) 146 if (xe_pt_zap_ptes_range(tile, vm, range)) { 147 /* 148 * WRITE_ONCE pairs with READ_ONCE in 149 * xe_vm_has_valid_gpu_mapping() 150 */ 151 WRITE_ONCE(range->tile_invalidated, 152 range->tile_invalidated | BIT(id)); 153 154 if (!(tile_mask & BIT(id))) { 155 xe_svm_tlb_inval_count_stats_incr(tile->primary_gt); 156 if (tile->media_gt) 157 xe_svm_tlb_inval_count_stats_incr(tile->media_gt); 158 tile_mask |= BIT(id); 159 } 160 } 161 162 return tile_mask; 163 } 164 165 static void 166 xe_svm_range_notifier_event_end(struct xe_vm *vm, struct drm_gpusvm_range *r, 167 const struct mmu_notifier_range *mmu_range) 168 { 169 struct drm_gpusvm_ctx ctx = { .in_notifier = true, }; 170 171 xe_svm_assert_in_notifier(vm); 172 173 drm_gpusvm_range_unmap_pages(&vm->svm.gpusvm, r, &ctx); 174 if (!xe_vm_is_closed(vm) && mmu_range->event == MMU_NOTIFY_UNMAP) 175 xe_svm_garbage_collector_add_range(vm, to_xe_range(r), 176 mmu_range); 177 } 178 179 static s64 xe_svm_stats_ktime_us_delta(ktime_t start) 180 { 181 return IS_ENABLED(CONFIG_DEBUG_FS) ? 182 ktime_us_delta(ktime_get(), start) : 0; 183 } 184 185 static void xe_svm_tlb_inval_us_stats_incr(struct xe_gt *gt, ktime_t start) 186 { 187 s64 us_delta = xe_svm_stats_ktime_us_delta(start); 188 189 xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_TLB_INVAL_US, us_delta); 190 } 191 192 static ktime_t xe_svm_stats_ktime_get(void) 193 { 194 return IS_ENABLED(CONFIG_DEBUG_FS) ? ktime_get() : 0; 195 } 196 197 static void xe_svm_invalidate(struct drm_gpusvm *gpusvm, 198 struct drm_gpusvm_notifier *notifier, 199 const struct mmu_notifier_range *mmu_range) 200 { 201 struct xe_vm *vm = gpusvm_to_vm(gpusvm); 202 struct xe_device *xe = vm->xe; 203 struct drm_gpusvm_range *r, *first; 204 struct xe_tile *tile; 205 ktime_t start = xe_svm_stats_ktime_get(); 206 u64 adj_start = mmu_range->start, adj_end = mmu_range->end; 207 u8 tile_mask = 0, id; 208 long err; 209 210 xe_svm_assert_in_notifier(vm); 211 212 vm_dbg(&gpusvm_to_vm(gpusvm)->xe->drm, 213 "INVALIDATE: asid=%u, gpusvm=%p, seqno=%lu, start=0x%016lx, end=0x%016lx, event=%d", 214 vm->usm.asid, gpusvm, notifier->notifier.invalidate_seq, 215 mmu_range->start, mmu_range->end, mmu_range->event); 216 217 /* Adjust invalidation to notifier boundaries */ 218 adj_start = max(drm_gpusvm_notifier_start(notifier), adj_start); 219 adj_end = min(drm_gpusvm_notifier_end(notifier), adj_end); 220 221 first = drm_gpusvm_range_find(notifier, adj_start, adj_end); 222 if (!first) 223 return; 224 225 /* 226 * PTs may be getting destroyed so not safe to touch these but PT should 227 * be invalidated at this point in time. Regardless we still need to 228 * ensure any dma mappings are unmapped in the here. 229 */ 230 if (xe_vm_is_closed(vm)) 231 goto range_notifier_event_end; 232 233 /* 234 * XXX: Less than ideal to always wait on VM's resv slots if an 235 * invalidation is not required. Could walk range list twice to figure 236 * out if an invalidations is need, but also not ideal. 237 */ 238 err = dma_resv_wait_timeout(xe_vm_resv(vm), 239 DMA_RESV_USAGE_BOOKKEEP, 240 false, MAX_SCHEDULE_TIMEOUT); 241 XE_WARN_ON(err <= 0); 242 243 r = first; 244 drm_gpusvm_for_each_range(r, notifier, adj_start, adj_end) 245 tile_mask |= xe_svm_range_notifier_event_begin(vm, r, mmu_range, 246 &adj_start, 247 &adj_end); 248 if (!tile_mask) 249 goto range_notifier_event_end; 250 251 xe_device_wmb(xe); 252 253 err = xe_vm_range_tilemask_tlb_inval(vm, adj_start, adj_end, tile_mask); 254 WARN_ON_ONCE(err); 255 256 range_notifier_event_end: 257 r = first; 258 drm_gpusvm_for_each_range(r, notifier, adj_start, adj_end) 259 xe_svm_range_notifier_event_end(vm, r, mmu_range); 260 for_each_tile(tile, xe, id) { 261 if (tile_mask & BIT(id)) { 262 xe_svm_tlb_inval_us_stats_incr(tile->primary_gt, start); 263 if (tile->media_gt) 264 xe_svm_tlb_inval_us_stats_incr(tile->media_gt, start); 265 } 266 } 267 } 268 269 static int __xe_svm_garbage_collector(struct xe_vm *vm, 270 struct xe_svm_range *range) 271 { 272 struct dma_fence *fence; 273 274 range_debug(range, "GARBAGE COLLECTOR"); 275 276 xe_vm_lock(vm, false); 277 fence = xe_vm_range_unbind(vm, range); 278 xe_vm_unlock(vm); 279 if (IS_ERR(fence)) 280 return PTR_ERR(fence); 281 dma_fence_put(fence); 282 283 drm_gpusvm_range_remove(&vm->svm.gpusvm, &range->base); 284 285 return 0; 286 } 287 288 static void xe_vma_set_default_attributes(struct xe_vma *vma) 289 { 290 vma->attr.preferred_loc.devmem_fd = DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE; 291 vma->attr.preferred_loc.migration_policy = DRM_XE_MIGRATE_ALL_PAGES; 292 vma->attr.pat_index = vma->attr.default_pat_index; 293 vma->attr.atomic_access = DRM_XE_ATOMIC_UNDEFINED; 294 } 295 296 static int xe_svm_range_set_default_attr(struct xe_vm *vm, u64 start, u64 end) 297 { 298 struct xe_vma *vma; 299 bool has_default_attr; 300 int err; 301 302 vma = xe_vm_find_vma_by_addr(vm, start); 303 if (!vma) 304 return -EINVAL; 305 306 if (!(vma->gpuva.flags & XE_VMA_MADV_AUTORESET)) { 307 drm_dbg(&vm->xe->drm, "Skipping madvise reset for vma.\n"); 308 return 0; 309 } 310 311 vm_dbg(&vm->xe->drm, "Existing VMA start=0x%016llx, vma_end=0x%016llx", 312 xe_vma_start(vma), xe_vma_end(vma)); 313 314 has_default_attr = xe_vma_has_default_mem_attrs(vma); 315 316 if (has_default_attr) { 317 start = xe_vma_start(vma); 318 end = xe_vma_end(vma); 319 } else if (xe_vma_start(vma) == start && xe_vma_end(vma) == end) { 320 xe_vma_set_default_attributes(vma); 321 } 322 323 xe_vm_find_cpu_addr_mirror_vma_range(vm, &start, &end); 324 325 if (xe_vma_start(vma) == start && xe_vma_end(vma) == end && has_default_attr) 326 return 0; 327 328 vm_dbg(&vm->xe->drm, "New VMA start=0x%016llx, vma_end=0x%016llx", start, end); 329 330 err = xe_vm_alloc_cpu_addr_mirror_vma(vm, start, end - start); 331 if (err) { 332 drm_warn(&vm->xe->drm, "New VMA MAP failed: %pe\n", ERR_PTR(err)); 333 xe_vm_kill(vm, true); 334 return err; 335 } 336 337 /* 338 * On call from xe_svm_handle_pagefault original VMA might be changed 339 * signal this to lookup for VMA again. 340 */ 341 return -EAGAIN; 342 } 343 344 static int xe_svm_garbage_collector(struct xe_vm *vm) 345 { 346 struct xe_svm_range *range; 347 u64 range_start; 348 u64 range_end; 349 int err, ret = 0; 350 351 lockdep_assert_held_write(&vm->lock); 352 353 if (xe_vm_is_closed_or_banned(vm)) 354 return -ENOENT; 355 356 for (;;) { 357 spin_lock(&vm->svm.garbage_collector.lock); 358 range = list_first_entry_or_null(&vm->svm.garbage_collector.range_list, 359 typeof(*range), 360 garbage_collector_link); 361 if (!range) 362 break; 363 364 range_start = xe_svm_range_start(range); 365 range_end = xe_svm_range_end(range); 366 367 list_del(&range->garbage_collector_link); 368 spin_unlock(&vm->svm.garbage_collector.lock); 369 370 err = __xe_svm_garbage_collector(vm, range); 371 if (err) { 372 drm_warn(&vm->xe->drm, 373 "Garbage collection failed: %pe\n", 374 ERR_PTR(err)); 375 xe_vm_kill(vm, true); 376 return err; 377 } 378 379 err = xe_svm_range_set_default_attr(vm, range_start, range_end); 380 if (err) { 381 if (err == -EAGAIN) 382 ret = -EAGAIN; 383 else 384 return err; 385 } 386 } 387 spin_unlock(&vm->svm.garbage_collector.lock); 388 389 return ret; 390 } 391 392 static void xe_svm_garbage_collector_work_func(struct work_struct *w) 393 { 394 struct xe_vm *vm = container_of(w, struct xe_vm, 395 svm.garbage_collector.work); 396 397 down_write(&vm->lock); 398 xe_svm_garbage_collector(vm); 399 up_write(&vm->lock); 400 } 401 402 #if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) 403 404 static struct xe_vram_region *page_to_vr(struct page *page) 405 { 406 return container_of(page_pgmap(page), struct xe_vram_region, pagemap); 407 } 408 409 static u64 xe_vram_region_page_to_dpa(struct xe_vram_region *vr, 410 struct page *page) 411 { 412 u64 dpa; 413 u64 pfn = page_to_pfn(page); 414 u64 offset; 415 416 xe_assert(vr->xe, is_device_private_page(page)); 417 xe_assert(vr->xe, (pfn << PAGE_SHIFT) >= vr->hpa_base); 418 419 offset = (pfn << PAGE_SHIFT) - vr->hpa_base; 420 dpa = vr->dpa_base + offset; 421 422 return dpa; 423 } 424 425 enum xe_svm_copy_dir { 426 XE_SVM_COPY_TO_VRAM, 427 XE_SVM_COPY_TO_SRAM, 428 }; 429 430 static void xe_svm_copy_kb_stats_incr(struct xe_gt *gt, 431 const enum xe_svm_copy_dir dir, 432 int kb) 433 { 434 if (dir == XE_SVM_COPY_TO_VRAM) 435 xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_DEVICE_COPY_KB, kb); 436 else 437 xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_CPU_COPY_KB, kb); 438 } 439 440 static void xe_svm_copy_us_stats_incr(struct xe_gt *gt, 441 const enum xe_svm_copy_dir dir, 442 unsigned long npages, 443 ktime_t start) 444 { 445 s64 us_delta = xe_svm_stats_ktime_us_delta(start); 446 447 if (dir == XE_SVM_COPY_TO_VRAM) { 448 switch (npages) { 449 case 1: 450 xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_4K_DEVICE_COPY_US, 451 us_delta); 452 break; 453 case 16: 454 xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_64K_DEVICE_COPY_US, 455 us_delta); 456 break; 457 case 512: 458 xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_2M_DEVICE_COPY_US, 459 us_delta); 460 break; 461 } 462 xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_DEVICE_COPY_US, 463 us_delta); 464 } else { 465 switch (npages) { 466 case 1: 467 xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_4K_CPU_COPY_US, 468 us_delta); 469 break; 470 case 16: 471 xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_64K_CPU_COPY_US, 472 us_delta); 473 break; 474 case 512: 475 xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_2M_CPU_COPY_US, 476 us_delta); 477 break; 478 } 479 xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_CPU_COPY_US, 480 us_delta); 481 } 482 } 483 484 static int xe_svm_copy(struct page **pages, 485 struct drm_pagemap_addr *pagemap_addr, 486 unsigned long npages, const enum xe_svm_copy_dir dir) 487 { 488 struct xe_vram_region *vr = NULL; 489 struct xe_gt *gt = NULL; 490 struct xe_device *xe; 491 struct dma_fence *fence = NULL; 492 unsigned long i; 493 #define XE_VRAM_ADDR_INVALID ~0x0ull 494 u64 vram_addr = XE_VRAM_ADDR_INVALID; 495 int err = 0, pos = 0; 496 bool sram = dir == XE_SVM_COPY_TO_SRAM; 497 ktime_t start = xe_svm_stats_ktime_get(); 498 499 /* 500 * This flow is complex: it locates physically contiguous device pages, 501 * derives the starting physical address, and performs a single GPU copy 502 * to for every 8M chunk in a DMA address array. Both device pages and 503 * DMA addresses may be sparsely populated. If either is NULL, a copy is 504 * triggered based on the current search state. The last GPU copy is 505 * waited on to ensure all copies are complete. 506 */ 507 508 for (i = 0; i < npages; ++i) { 509 struct page *spage = pages[i]; 510 struct dma_fence *__fence; 511 u64 __vram_addr; 512 bool match = false, chunk, last; 513 514 #define XE_MIGRATE_CHUNK_SIZE SZ_8M 515 chunk = (i - pos) == (XE_MIGRATE_CHUNK_SIZE / PAGE_SIZE); 516 last = (i + 1) == npages; 517 518 /* No CPU page and no device pages queue'd to copy */ 519 if (!pagemap_addr[i].addr && vram_addr == XE_VRAM_ADDR_INVALID) 520 continue; 521 522 if (!vr && spage) { 523 vr = page_to_vr(spage); 524 gt = xe_migrate_exec_queue(vr->migrate)->gt; 525 xe = vr->xe; 526 } 527 XE_WARN_ON(spage && page_to_vr(spage) != vr); 528 529 /* 530 * CPU page and device page valid, capture physical address on 531 * first device page, check if physical contiguous on subsequent 532 * device pages. 533 */ 534 if (pagemap_addr[i].addr && spage) { 535 __vram_addr = xe_vram_region_page_to_dpa(vr, spage); 536 if (vram_addr == XE_VRAM_ADDR_INVALID) { 537 vram_addr = __vram_addr; 538 pos = i; 539 } 540 541 match = vram_addr + PAGE_SIZE * (i - pos) == __vram_addr; 542 /* Expected with contiguous memory */ 543 xe_assert(vr->xe, match); 544 545 if (pagemap_addr[i].order) { 546 i += NR_PAGES(pagemap_addr[i].order) - 1; 547 chunk = (i - pos) == (XE_MIGRATE_CHUNK_SIZE / PAGE_SIZE); 548 last = (i + 1) == npages; 549 } 550 } 551 552 /* 553 * Mismatched physical address, 8M copy chunk, or last page - 554 * trigger a copy. 555 */ 556 if (!match || chunk || last) { 557 /* 558 * Extra page for first copy if last page and matching 559 * physical address. 560 */ 561 int incr = (match && last) ? 1 : 0; 562 563 if (vram_addr != XE_VRAM_ADDR_INVALID) { 564 xe_svm_copy_kb_stats_incr(gt, dir, 565 (i - pos + incr) * 566 (PAGE_SIZE / SZ_1K)); 567 if (sram) { 568 vm_dbg(&xe->drm, 569 "COPY TO SRAM - 0x%016llx -> 0x%016llx, NPAGES=%ld", 570 vram_addr, 571 (u64)pagemap_addr[pos].addr, i - pos + incr); 572 __fence = xe_migrate_from_vram(vr->migrate, 573 i - pos + incr, 574 vram_addr, 575 &pagemap_addr[pos]); 576 } else { 577 vm_dbg(&xe->drm, 578 "COPY TO VRAM - 0x%016llx -> 0x%016llx, NPAGES=%ld", 579 (u64)pagemap_addr[pos].addr, vram_addr, 580 i - pos + incr); 581 __fence = xe_migrate_to_vram(vr->migrate, 582 i - pos + incr, 583 &pagemap_addr[pos], 584 vram_addr); 585 } 586 if (IS_ERR(__fence)) { 587 err = PTR_ERR(__fence); 588 goto err_out; 589 } 590 591 dma_fence_put(fence); 592 fence = __fence; 593 } 594 595 /* Setup physical address of next device page */ 596 if (pagemap_addr[i].addr && spage) { 597 vram_addr = __vram_addr; 598 pos = i; 599 } else { 600 vram_addr = XE_VRAM_ADDR_INVALID; 601 } 602 603 /* Extra mismatched device page, copy it */ 604 if (!match && last && vram_addr != XE_VRAM_ADDR_INVALID) { 605 xe_svm_copy_kb_stats_incr(gt, dir, 606 (PAGE_SIZE / SZ_1K)); 607 if (sram) { 608 vm_dbg(&xe->drm, 609 "COPY TO SRAM - 0x%016llx -> 0x%016llx, NPAGES=%d", 610 vram_addr, (u64)pagemap_addr[pos].addr, 1); 611 __fence = xe_migrate_from_vram(vr->migrate, 1, 612 vram_addr, 613 &pagemap_addr[pos]); 614 } else { 615 vm_dbg(&xe->drm, 616 "COPY TO VRAM - 0x%016llx -> 0x%016llx, NPAGES=%d", 617 (u64)pagemap_addr[pos].addr, vram_addr, 1); 618 __fence = xe_migrate_to_vram(vr->migrate, 1, 619 &pagemap_addr[pos], 620 vram_addr); 621 } 622 if (IS_ERR(__fence)) { 623 err = PTR_ERR(__fence); 624 goto err_out; 625 } 626 627 dma_fence_put(fence); 628 fence = __fence; 629 } 630 } 631 } 632 633 err_out: 634 /* Wait for all copies to complete */ 635 if (fence) { 636 dma_fence_wait(fence, false); 637 dma_fence_put(fence); 638 } 639 640 /* 641 * XXX: We can't derive the GT here (or anywhere in this functions, but 642 * compute always uses the primary GT so accumulate stats on the likely 643 * GT of the fault. 644 */ 645 if (gt) 646 xe_svm_copy_us_stats_incr(gt, dir, npages, start); 647 648 return err; 649 #undef XE_MIGRATE_CHUNK_SIZE 650 #undef XE_VRAM_ADDR_INVALID 651 } 652 653 static int xe_svm_copy_to_devmem(struct page **pages, 654 struct drm_pagemap_addr *pagemap_addr, 655 unsigned long npages) 656 { 657 return xe_svm_copy(pages, pagemap_addr, npages, XE_SVM_COPY_TO_VRAM); 658 } 659 660 static int xe_svm_copy_to_ram(struct page **pages, 661 struct drm_pagemap_addr *pagemap_addr, 662 unsigned long npages) 663 { 664 return xe_svm_copy(pages, pagemap_addr, npages, XE_SVM_COPY_TO_SRAM); 665 } 666 667 static struct xe_bo *to_xe_bo(struct drm_pagemap_devmem *devmem_allocation) 668 { 669 return container_of(devmem_allocation, struct xe_bo, devmem_allocation); 670 } 671 672 static void xe_svm_devmem_release(struct drm_pagemap_devmem *devmem_allocation) 673 { 674 struct xe_bo *bo = to_xe_bo(devmem_allocation); 675 struct xe_device *xe = xe_bo_device(bo); 676 677 xe_bo_put_async(bo); 678 xe_pm_runtime_put(xe); 679 } 680 681 static u64 block_offset_to_pfn(struct xe_vram_region *vr, u64 offset) 682 { 683 return PHYS_PFN(offset + vr->hpa_base); 684 } 685 686 static struct drm_buddy *vram_to_buddy(struct xe_vram_region *vram) 687 { 688 return &vram->ttm.mm; 689 } 690 691 static int xe_svm_populate_devmem_pfn(struct drm_pagemap_devmem *devmem_allocation, 692 unsigned long npages, unsigned long *pfn) 693 { 694 struct xe_bo *bo = to_xe_bo(devmem_allocation); 695 struct ttm_resource *res = bo->ttm.resource; 696 struct list_head *blocks = &to_xe_ttm_vram_mgr_resource(res)->blocks; 697 struct drm_buddy_block *block; 698 int j = 0; 699 700 list_for_each_entry(block, blocks, link) { 701 struct xe_vram_region *vr = block->private; 702 struct drm_buddy *buddy = vram_to_buddy(vr); 703 u64 block_pfn = block_offset_to_pfn(vr, drm_buddy_block_offset(block)); 704 int i; 705 706 for (i = 0; i < drm_buddy_block_size(buddy, block) >> PAGE_SHIFT; ++i) 707 pfn[j++] = block_pfn + i; 708 } 709 710 return 0; 711 } 712 713 static const struct drm_pagemap_devmem_ops dpagemap_devmem_ops = { 714 .devmem_release = xe_svm_devmem_release, 715 .populate_devmem_pfn = xe_svm_populate_devmem_pfn, 716 .copy_to_devmem = xe_svm_copy_to_devmem, 717 .copy_to_ram = xe_svm_copy_to_ram, 718 }; 719 720 #endif 721 722 static const struct drm_gpusvm_ops gpusvm_ops = { 723 .range_alloc = xe_svm_range_alloc, 724 .range_free = xe_svm_range_free, 725 .invalidate = xe_svm_invalidate, 726 }; 727 728 static const unsigned long fault_chunk_sizes[] = { 729 SZ_2M, 730 SZ_64K, 731 SZ_4K, 732 }; 733 734 /** 735 * xe_svm_init() - SVM initialize 736 * @vm: The VM. 737 * 738 * Initialize SVM state which is embedded within the VM. 739 * 740 * Return: 0 on success, negative error code on error. 741 */ 742 int xe_svm_init(struct xe_vm *vm) 743 { 744 int err; 745 746 if (vm->flags & XE_VM_FLAG_FAULT_MODE) { 747 spin_lock_init(&vm->svm.garbage_collector.lock); 748 INIT_LIST_HEAD(&vm->svm.garbage_collector.range_list); 749 INIT_WORK(&vm->svm.garbage_collector.work, 750 xe_svm_garbage_collector_work_func); 751 752 err = drm_gpusvm_init(&vm->svm.gpusvm, "Xe SVM", &vm->xe->drm, 753 current->mm, 0, vm->size, 754 xe_modparam.svm_notifier_size * SZ_1M, 755 &gpusvm_ops, fault_chunk_sizes, 756 ARRAY_SIZE(fault_chunk_sizes)); 757 drm_gpusvm_driver_set_lock(&vm->svm.gpusvm, &vm->lock); 758 } else { 759 err = drm_gpusvm_init(&vm->svm.gpusvm, "Xe SVM (simple)", 760 &vm->xe->drm, NULL, 0, 0, 0, NULL, 761 NULL, 0); 762 } 763 764 return err; 765 } 766 767 /** 768 * xe_svm_close() - SVM close 769 * @vm: The VM. 770 * 771 * Close SVM state (i.e., stop and flush all SVM actions). 772 */ 773 void xe_svm_close(struct xe_vm *vm) 774 { 775 xe_assert(vm->xe, xe_vm_is_closed(vm)); 776 flush_work(&vm->svm.garbage_collector.work); 777 } 778 779 /** 780 * xe_svm_fini() - SVM finalize 781 * @vm: The VM. 782 * 783 * Finalize SVM state which is embedded within the VM. 784 */ 785 void xe_svm_fini(struct xe_vm *vm) 786 { 787 xe_assert(vm->xe, xe_vm_is_closed(vm)); 788 789 drm_gpusvm_fini(&vm->svm.gpusvm); 790 } 791 792 static bool xe_svm_range_is_valid(struct xe_svm_range *range, 793 struct xe_tile *tile, 794 bool devmem_only) 795 { 796 return (xe_vm_has_valid_gpu_mapping(tile, range->tile_present, 797 range->tile_invalidated) && 798 (!devmem_only || xe_svm_range_in_vram(range))); 799 } 800 801 /** xe_svm_range_migrate_to_smem() - Move range pages from VRAM to SMEM 802 * @vm: xe_vm pointer 803 * @range: Pointer to the SVM range structure 804 * 805 * The xe_svm_range_migrate_to_smem() checks range has pages in VRAM 806 * and migrates them to SMEM 807 */ 808 void xe_svm_range_migrate_to_smem(struct xe_vm *vm, struct xe_svm_range *range) 809 { 810 if (xe_svm_range_in_vram(range)) 811 drm_gpusvm_range_evict(&vm->svm.gpusvm, &range->base); 812 } 813 814 /** 815 * xe_svm_range_validate() - Check if the SVM range is valid 816 * @vm: xe_vm pointer 817 * @range: Pointer to the SVM range structure 818 * @tile_mask: Mask representing the tiles to be checked 819 * @devmem_preferred : if true range needs to be in devmem 820 * 821 * The xe_svm_range_validate() function checks if a range is 822 * valid and located in the desired memory region. 823 * 824 * Return: true if the range is valid, false otherwise 825 */ 826 bool xe_svm_range_validate(struct xe_vm *vm, 827 struct xe_svm_range *range, 828 u8 tile_mask, bool devmem_preferred) 829 { 830 bool ret; 831 832 xe_svm_notifier_lock(vm); 833 834 ret = (range->tile_present & ~range->tile_invalidated & tile_mask) == tile_mask && 835 (devmem_preferred == range->base.pages.flags.has_devmem_pages); 836 837 xe_svm_notifier_unlock(vm); 838 839 return ret; 840 } 841 842 /** 843 * xe_svm_find_vma_start - Find start of CPU VMA 844 * @vm: xe_vm pointer 845 * @start: start address 846 * @end: end address 847 * @vma: Pointer to struct xe_vma 848 * 849 * 850 * This function searches for a cpu vma, within the specified 851 * range [start, end] in the given VM. It adjusts the range based on the 852 * xe_vma start and end addresses. If no cpu VMA is found, it returns ULONG_MAX. 853 * 854 * Return: The starting address of the VMA within the range, 855 * or ULONG_MAX if no VMA is found 856 */ 857 u64 xe_svm_find_vma_start(struct xe_vm *vm, u64 start, u64 end, struct xe_vma *vma) 858 { 859 return drm_gpusvm_find_vma_start(&vm->svm.gpusvm, 860 max(start, xe_vma_start(vma)), 861 min(end, xe_vma_end(vma))); 862 } 863 864 #if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) 865 static int xe_drm_pagemap_populate_mm(struct drm_pagemap *dpagemap, 866 unsigned long start, unsigned long end, 867 struct mm_struct *mm, 868 unsigned long timeslice_ms) 869 { 870 struct xe_vram_region *vr = container_of(dpagemap, typeof(*vr), dpagemap); 871 struct xe_device *xe = vr->xe; 872 struct device *dev = xe->drm.dev; 873 struct drm_buddy_block *block; 874 struct xe_validation_ctx vctx; 875 struct list_head *blocks; 876 struct drm_exec exec; 877 struct xe_bo *bo; 878 int err = 0, idx; 879 880 if (!drm_dev_enter(&xe->drm, &idx)) 881 return -ENODEV; 882 883 xe_pm_runtime_get(xe); 884 885 xe_validation_guard(&vctx, &xe->val, &exec, (struct xe_val_flags) {}, err) { 886 bo = xe_bo_create_locked(xe, NULL, NULL, end - start, 887 ttm_bo_type_device, 888 (IS_DGFX(xe) ? XE_BO_FLAG_VRAM(vr) : XE_BO_FLAG_SYSTEM) | 889 XE_BO_FLAG_CPU_ADDR_MIRROR, &exec); 890 drm_exec_retry_on_contention(&exec); 891 if (IS_ERR(bo)) { 892 err = PTR_ERR(bo); 893 xe_validation_retry_on_oom(&vctx, &err); 894 break; 895 } 896 897 drm_pagemap_devmem_init(&bo->devmem_allocation, dev, mm, 898 &dpagemap_devmem_ops, dpagemap, end - start); 899 900 blocks = &to_xe_ttm_vram_mgr_resource(bo->ttm.resource)->blocks; 901 list_for_each_entry(block, blocks, link) 902 block->private = vr; 903 904 xe_bo_get(bo); 905 906 /* Ensure the device has a pm ref while there are device pages active. */ 907 xe_pm_runtime_get_noresume(xe); 908 err = drm_pagemap_migrate_to_devmem(&bo->devmem_allocation, mm, 909 start, end, timeslice_ms, 910 xe_svm_devm_owner(xe)); 911 if (err) 912 xe_svm_devmem_release(&bo->devmem_allocation); 913 xe_bo_unlock(bo); 914 xe_bo_put(bo); 915 } 916 xe_pm_runtime_put(xe); 917 drm_dev_exit(idx); 918 919 return err; 920 } 921 #endif 922 923 static bool supports_4K_migration(struct xe_device *xe) 924 { 925 if (xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) 926 return false; 927 928 return true; 929 } 930 931 /** 932 * xe_svm_range_needs_migrate_to_vram() - SVM range needs migrate to VRAM or not 933 * @range: SVM range for which migration needs to be decided 934 * @vma: vma which has range 935 * @preferred_region_is_vram: preferred region for range is vram 936 * 937 * Return: True for range needing migration and migration is supported else false 938 */ 939 bool xe_svm_range_needs_migrate_to_vram(struct xe_svm_range *range, struct xe_vma *vma, 940 bool preferred_region_is_vram) 941 { 942 struct xe_vm *vm = range_to_vm(&range->base); 943 u64 range_size = xe_svm_range_size(range); 944 945 if (!range->base.pages.flags.migrate_devmem || !preferred_region_is_vram) 946 return false; 947 948 xe_assert(vm->xe, IS_DGFX(vm->xe)); 949 950 if (xe_svm_range_in_vram(range)) { 951 drm_info(&vm->xe->drm, "Range is already in VRAM\n"); 952 return false; 953 } 954 955 if (range_size < SZ_64K && !supports_4K_migration(vm->xe)) { 956 drm_dbg(&vm->xe->drm, "Platform doesn't support SZ_4K range migration\n"); 957 return false; 958 } 959 960 return true; 961 } 962 963 #define DECL_SVM_RANGE_COUNT_STATS(elem, stat) \ 964 static void xe_svm_range_##elem##_count_stats_incr(struct xe_gt *gt, \ 965 struct xe_svm_range *range) \ 966 { \ 967 switch (xe_svm_range_size(range)) { \ 968 case SZ_4K: \ 969 xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_4K_##stat##_COUNT, 1); \ 970 break; \ 971 case SZ_64K: \ 972 xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_64K_##stat##_COUNT, 1); \ 973 break; \ 974 case SZ_2M: \ 975 xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_2M_##stat##_COUNT, 1); \ 976 break; \ 977 } \ 978 } \ 979 980 DECL_SVM_RANGE_COUNT_STATS(fault, PAGEFAULT) 981 DECL_SVM_RANGE_COUNT_STATS(valid_fault, VALID_PAGEFAULT) 982 DECL_SVM_RANGE_COUNT_STATS(migrate, MIGRATE) 983 984 #define DECL_SVM_RANGE_US_STATS(elem, stat) \ 985 static void xe_svm_range_##elem##_us_stats_incr(struct xe_gt *gt, \ 986 struct xe_svm_range *range, \ 987 ktime_t start) \ 988 { \ 989 s64 us_delta = xe_svm_stats_ktime_us_delta(start); \ 990 \ 991 switch (xe_svm_range_size(range)) { \ 992 case SZ_4K: \ 993 xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_4K_##stat##_US, \ 994 us_delta); \ 995 break; \ 996 case SZ_64K: \ 997 xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_64K_##stat##_US, \ 998 us_delta); \ 999 break; \ 1000 case SZ_2M: \ 1001 xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_2M_##stat##_US, \ 1002 us_delta); \ 1003 break; \ 1004 } \ 1005 } \ 1006 1007 DECL_SVM_RANGE_US_STATS(migrate, MIGRATE) 1008 DECL_SVM_RANGE_US_STATS(get_pages, GET_PAGES) 1009 DECL_SVM_RANGE_US_STATS(bind, BIND) 1010 DECL_SVM_RANGE_US_STATS(fault, PAGEFAULT) 1011 1012 static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, 1013 struct xe_gt *gt, u64 fault_addr, 1014 bool need_vram) 1015 { 1016 int devmem_possible = IS_DGFX(vm->xe) && 1017 IS_ENABLED(CONFIG_DRM_XE_PAGEMAP); 1018 struct drm_gpusvm_ctx ctx = { 1019 .read_only = xe_vma_read_only(vma), 1020 .devmem_possible = devmem_possible, 1021 .check_pages_threshold = devmem_possible ? SZ_64K : 0, 1022 .devmem_only = need_vram && devmem_possible, 1023 .timeslice_ms = need_vram && devmem_possible ? 1024 vm->xe->atomic_svm_timeslice_ms : 0, 1025 .device_private_page_owner = xe_svm_devm_owner(vm->xe), 1026 }; 1027 struct xe_validation_ctx vctx; 1028 struct drm_exec exec; 1029 struct xe_svm_range *range; 1030 struct dma_fence *fence; 1031 struct drm_pagemap *dpagemap; 1032 struct xe_tile *tile = gt_to_tile(gt); 1033 int migrate_try_count = ctx.devmem_only ? 3 : 1; 1034 ktime_t start = xe_svm_stats_ktime_get(), bind_start, get_pages_start; 1035 int err; 1036 1037 lockdep_assert_held_write(&vm->lock); 1038 xe_assert(vm->xe, xe_vma_is_cpu_addr_mirror(vma)); 1039 1040 xe_gt_stats_incr(gt, XE_GT_STATS_ID_SVM_PAGEFAULT_COUNT, 1); 1041 1042 retry: 1043 /* Always process UNMAPs first so view SVM ranges is current */ 1044 err = xe_svm_garbage_collector(vm); 1045 if (err) 1046 return err; 1047 1048 dpagemap = xe_vma_resolve_pagemap(vma, tile); 1049 if (!dpagemap && !ctx.devmem_only) 1050 ctx.device_private_page_owner = NULL; 1051 range = xe_svm_range_find_or_insert(vm, fault_addr, vma, &ctx); 1052 1053 if (IS_ERR(range)) 1054 return PTR_ERR(range); 1055 1056 xe_svm_range_fault_count_stats_incr(gt, range); 1057 1058 if (ctx.devmem_only && !range->base.pages.flags.migrate_devmem) { 1059 err = -EACCES; 1060 goto out; 1061 } 1062 1063 if (xe_svm_range_is_valid(range, tile, ctx.devmem_only)) { 1064 xe_svm_range_valid_fault_count_stats_incr(gt, range); 1065 range_debug(range, "PAGE FAULT - VALID"); 1066 goto out; 1067 } 1068 1069 range_debug(range, "PAGE FAULT"); 1070 1071 if (--migrate_try_count >= 0 && 1072 xe_svm_range_needs_migrate_to_vram(range, vma, !!dpagemap || ctx.devmem_only)) { 1073 ktime_t migrate_start = xe_svm_stats_ktime_get(); 1074 1075 /* TODO : For multi-device dpagemap will be used to find the 1076 * remote tile and remote device. Will need to modify 1077 * xe_svm_alloc_vram to use dpagemap for future multi-device 1078 * support. 1079 */ 1080 xe_svm_range_migrate_count_stats_incr(gt, range); 1081 err = xe_svm_alloc_vram(tile, range, &ctx); 1082 xe_svm_range_migrate_us_stats_incr(gt, range, migrate_start); 1083 ctx.timeslice_ms <<= 1; /* Double timeslice if we have to retry */ 1084 if (err) { 1085 if (migrate_try_count || !ctx.devmem_only) { 1086 drm_dbg(&vm->xe->drm, 1087 "VRAM allocation failed, falling back to retrying fault, asid=%u, errno=%pe\n", 1088 vm->usm.asid, ERR_PTR(err)); 1089 1090 /* 1091 * In the devmem-only case, mixed mappings may 1092 * be found. The get_pages function will fix 1093 * these up to a single location, allowing the 1094 * page fault handler to make forward progress. 1095 */ 1096 if (ctx.devmem_only) 1097 goto get_pages; 1098 else 1099 goto retry; 1100 } else { 1101 drm_err(&vm->xe->drm, 1102 "VRAM allocation failed, retry count exceeded, asid=%u, errno=%pe\n", 1103 vm->usm.asid, ERR_PTR(err)); 1104 return err; 1105 } 1106 } 1107 } 1108 1109 get_pages: 1110 get_pages_start = xe_svm_stats_ktime_get(); 1111 1112 range_debug(range, "GET PAGES"); 1113 err = xe_svm_range_get_pages(vm, range, &ctx); 1114 /* Corner where CPU mappings have changed */ 1115 if (err == -EOPNOTSUPP || err == -EFAULT || err == -EPERM) { 1116 ctx.timeslice_ms <<= 1; /* Double timeslice if we have to retry */ 1117 if (migrate_try_count > 0 || !ctx.devmem_only) { 1118 drm_dbg(&vm->xe->drm, 1119 "Get pages failed, falling back to retrying, asid=%u, gpusvm=%p, errno=%pe\n", 1120 vm->usm.asid, &vm->svm.gpusvm, ERR_PTR(err)); 1121 range_debug(range, "PAGE FAULT - RETRY PAGES"); 1122 goto retry; 1123 } else { 1124 drm_err(&vm->xe->drm, 1125 "Get pages failed, retry count exceeded, asid=%u, gpusvm=%p, errno=%pe\n", 1126 vm->usm.asid, &vm->svm.gpusvm, ERR_PTR(err)); 1127 } 1128 } 1129 if (err) { 1130 range_debug(range, "PAGE FAULT - FAIL PAGE COLLECT"); 1131 goto out; 1132 } 1133 1134 xe_svm_range_get_pages_us_stats_incr(gt, range, get_pages_start); 1135 range_debug(range, "PAGE FAULT - BIND"); 1136 1137 bind_start = xe_svm_stats_ktime_get(); 1138 xe_validation_guard(&vctx, &vm->xe->val, &exec, (struct xe_val_flags) {}, err) { 1139 err = xe_vm_drm_exec_lock(vm, &exec); 1140 drm_exec_retry_on_contention(&exec); 1141 1142 xe_vm_set_validation_exec(vm, &exec); 1143 fence = xe_vm_range_rebind(vm, vma, range, BIT(tile->id)); 1144 xe_vm_set_validation_exec(vm, NULL); 1145 if (IS_ERR(fence)) { 1146 drm_exec_retry_on_contention(&exec); 1147 err = PTR_ERR(fence); 1148 xe_validation_retry_on_oom(&vctx, &err); 1149 xe_svm_range_bind_us_stats_incr(gt, range, bind_start); 1150 break; 1151 } 1152 } 1153 if (err) 1154 goto err_out; 1155 1156 dma_fence_wait(fence, false); 1157 dma_fence_put(fence); 1158 xe_svm_range_bind_us_stats_incr(gt, range, bind_start); 1159 1160 out: 1161 xe_svm_range_fault_us_stats_incr(gt, range, start); 1162 return 0; 1163 1164 err_out: 1165 if (err == -EAGAIN) { 1166 ctx.timeslice_ms <<= 1; /* Double timeslice if we have to retry */ 1167 range_debug(range, "PAGE FAULT - RETRY BIND"); 1168 goto retry; 1169 } 1170 1171 return err; 1172 } 1173 1174 /** 1175 * xe_svm_handle_pagefault() - SVM handle page fault 1176 * @vm: The VM. 1177 * @vma: The CPU address mirror VMA. 1178 * @gt: The gt upon the fault occurred. 1179 * @fault_addr: The GPU fault address. 1180 * @atomic: The fault atomic access bit. 1181 * 1182 * Create GPU bindings for a SVM page fault. Optionally migrate to device 1183 * memory. 1184 * 1185 * Return: 0 on success, negative error code on error. 1186 */ 1187 int xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, 1188 struct xe_gt *gt, u64 fault_addr, 1189 bool atomic) 1190 { 1191 int need_vram, ret; 1192 retry: 1193 need_vram = xe_vma_need_vram_for_atomic(vm->xe, vma, atomic); 1194 if (need_vram < 0) 1195 return need_vram; 1196 1197 ret = __xe_svm_handle_pagefault(vm, vma, gt, fault_addr, 1198 need_vram ? true : false); 1199 if (ret == -EAGAIN) { 1200 /* 1201 * Retry once on -EAGAIN to re-lookup the VMA, as the original VMA 1202 * may have been split by xe_svm_range_set_default_attr. 1203 */ 1204 vma = xe_vm_find_vma_by_addr(vm, fault_addr); 1205 if (!vma) 1206 return -EINVAL; 1207 1208 goto retry; 1209 } 1210 return ret; 1211 } 1212 1213 /** 1214 * xe_svm_has_mapping() - SVM has mappings 1215 * @vm: The VM. 1216 * @start: Start address. 1217 * @end: End address. 1218 * 1219 * Check if an address range has SVM mappings. 1220 * 1221 * Return: True if address range has a SVM mapping, False otherwise 1222 */ 1223 bool xe_svm_has_mapping(struct xe_vm *vm, u64 start, u64 end) 1224 { 1225 return drm_gpusvm_has_mapping(&vm->svm.gpusvm, start, end); 1226 } 1227 1228 /** 1229 * xe_svm_unmap_address_range - UNMAP SVM mappings and ranges 1230 * @vm: The VM 1231 * @start: start addr 1232 * @end: end addr 1233 * 1234 * This function UNMAPS svm ranges if start or end address are inside them. 1235 */ 1236 void xe_svm_unmap_address_range(struct xe_vm *vm, u64 start, u64 end) 1237 { 1238 struct drm_gpusvm_notifier *notifier, *next; 1239 1240 lockdep_assert_held_write(&vm->lock); 1241 1242 drm_gpusvm_for_each_notifier_safe(notifier, next, &vm->svm.gpusvm, start, end) { 1243 struct drm_gpusvm_range *range, *__next; 1244 1245 drm_gpusvm_for_each_range_safe(range, __next, notifier, start, end) { 1246 if (start > drm_gpusvm_range_start(range) || 1247 end < drm_gpusvm_range_end(range)) { 1248 if (IS_DGFX(vm->xe) && xe_svm_range_in_vram(to_xe_range(range))) 1249 drm_gpusvm_range_evict(&vm->svm.gpusvm, range); 1250 drm_gpusvm_range_get(range); 1251 __xe_svm_garbage_collector(vm, to_xe_range(range)); 1252 if (!list_empty(&to_xe_range(range)->garbage_collector_link)) { 1253 spin_lock(&vm->svm.garbage_collector.lock); 1254 list_del(&to_xe_range(range)->garbage_collector_link); 1255 spin_unlock(&vm->svm.garbage_collector.lock); 1256 } 1257 drm_gpusvm_range_put(range); 1258 } 1259 } 1260 } 1261 } 1262 1263 /** 1264 * xe_svm_bo_evict() - SVM evict BO to system memory 1265 * @bo: BO to evict 1266 * 1267 * SVM evict BO to system memory. GPU SVM layer ensures all device pages 1268 * are evicted before returning. 1269 * 1270 * Return: 0 on success standard error code otherwise 1271 */ 1272 int xe_svm_bo_evict(struct xe_bo *bo) 1273 { 1274 return drm_pagemap_evict_to_ram(&bo->devmem_allocation); 1275 } 1276 1277 /** 1278 * xe_svm_range_find_or_insert- Find or insert GPU SVM range 1279 * @vm: xe_vm pointer 1280 * @addr: address for which range needs to be found/inserted 1281 * @vma: Pointer to struct xe_vma which mirrors CPU 1282 * @ctx: GPU SVM context 1283 * 1284 * This function finds or inserts a newly allocated a SVM range based on the 1285 * address. 1286 * 1287 * Return: Pointer to the SVM range on success, ERR_PTR() on failure. 1288 */ 1289 struct xe_svm_range *xe_svm_range_find_or_insert(struct xe_vm *vm, u64 addr, 1290 struct xe_vma *vma, struct drm_gpusvm_ctx *ctx) 1291 { 1292 struct drm_gpusvm_range *r; 1293 1294 r = drm_gpusvm_range_find_or_insert(&vm->svm.gpusvm, max(addr, xe_vma_start(vma)), 1295 xe_vma_start(vma), xe_vma_end(vma), ctx); 1296 if (IS_ERR(r)) 1297 return ERR_CAST(r); 1298 1299 return to_xe_range(r); 1300 } 1301 1302 /** 1303 * xe_svm_range_get_pages() - Get pages for a SVM range 1304 * @vm: Pointer to the struct xe_vm 1305 * @range: Pointer to the xe SVM range structure 1306 * @ctx: GPU SVM context 1307 * 1308 * This function gets pages for a SVM range and ensures they are mapped for 1309 * DMA access. In case of failure with -EOPNOTSUPP, it evicts the range. 1310 * 1311 * Return: 0 on success, negative error code on failure. 1312 */ 1313 int xe_svm_range_get_pages(struct xe_vm *vm, struct xe_svm_range *range, 1314 struct drm_gpusvm_ctx *ctx) 1315 { 1316 int err = 0; 1317 1318 err = drm_gpusvm_range_get_pages(&vm->svm.gpusvm, &range->base, ctx); 1319 if (err == -EOPNOTSUPP) { 1320 range_debug(range, "PAGE FAULT - EVICT PAGES"); 1321 drm_gpusvm_range_evict(&vm->svm.gpusvm, &range->base); 1322 } 1323 1324 return err; 1325 } 1326 1327 /** 1328 * xe_svm_ranges_zap_ptes_in_range - clear ptes of svm ranges in input range 1329 * @vm: Pointer to the xe_vm structure 1330 * @start: Start of the input range 1331 * @end: End of the input range 1332 * 1333 * This function removes the page table entries (PTEs) associated 1334 * with the svm ranges within the given input start and end 1335 * 1336 * Return: tile_mask for which gt's need to be tlb invalidated. 1337 */ 1338 u8 xe_svm_ranges_zap_ptes_in_range(struct xe_vm *vm, u64 start, u64 end) 1339 { 1340 struct drm_gpusvm_notifier *notifier; 1341 struct xe_svm_range *range; 1342 u64 adj_start, adj_end; 1343 struct xe_tile *tile; 1344 u8 tile_mask = 0; 1345 u8 id; 1346 1347 lockdep_assert(lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 1) && 1348 lockdep_is_held_type(&vm->lock, 0)); 1349 1350 drm_gpusvm_for_each_notifier(notifier, &vm->svm.gpusvm, start, end) { 1351 struct drm_gpusvm_range *r = NULL; 1352 1353 adj_start = max(start, drm_gpusvm_notifier_start(notifier)); 1354 adj_end = min(end, drm_gpusvm_notifier_end(notifier)); 1355 drm_gpusvm_for_each_range(r, notifier, adj_start, adj_end) { 1356 range = to_xe_range(r); 1357 for_each_tile(tile, vm->xe, id) { 1358 if (xe_pt_zap_ptes_range(tile, vm, range)) { 1359 tile_mask |= BIT(id); 1360 /* 1361 * WRITE_ONCE pairs with READ_ONCE in 1362 * xe_vm_has_valid_gpu_mapping(). 1363 * Must not fail after setting 1364 * tile_invalidated and before 1365 * TLB invalidation. 1366 */ 1367 WRITE_ONCE(range->tile_invalidated, 1368 range->tile_invalidated | BIT(id)); 1369 } 1370 } 1371 } 1372 } 1373 1374 return tile_mask; 1375 } 1376 1377 #if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP) 1378 1379 static struct drm_pagemap *tile_local_pagemap(struct xe_tile *tile) 1380 { 1381 return &tile->mem.vram->dpagemap; 1382 } 1383 1384 /** 1385 * xe_vma_resolve_pagemap - Resolve the appropriate DRM pagemap for a VMA 1386 * @vma: Pointer to the xe_vma structure containing memory attributes 1387 * @tile: Pointer to the xe_tile structure used as fallback for VRAM mapping 1388 * 1389 * This function determines the correct DRM pagemap to use for a given VMA. 1390 * It first checks if a valid devmem_fd is provided in the VMA's preferred 1391 * location. If the devmem_fd is negative, it returns NULL, indicating no 1392 * pagemap is available and smem to be used as preferred location. 1393 * If the devmem_fd is equal to the default faulting 1394 * GT identifier, it returns the VRAM pagemap associated with the tile. 1395 * 1396 * Future support for multi-device configurations may use drm_pagemap_from_fd() 1397 * to resolve pagemaps from arbitrary file descriptors. 1398 * 1399 * Return: A pointer to the resolved drm_pagemap, or NULL if none is applicable. 1400 */ 1401 struct drm_pagemap *xe_vma_resolve_pagemap(struct xe_vma *vma, struct xe_tile *tile) 1402 { 1403 s32 fd = (s32)vma->attr.preferred_loc.devmem_fd; 1404 1405 if (fd == DRM_XE_PREFERRED_LOC_DEFAULT_SYSTEM) 1406 return NULL; 1407 1408 if (fd == DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE) 1409 return IS_DGFX(tile_to_xe(tile)) ? tile_local_pagemap(tile) : NULL; 1410 1411 /* TODO: Support multi-device with drm_pagemap_from_fd(fd) */ 1412 return NULL; 1413 } 1414 1415 /** 1416 * xe_svm_alloc_vram()- Allocate device memory pages for range, 1417 * migrating existing data. 1418 * @tile: tile to allocate vram from 1419 * @range: SVM range 1420 * @ctx: DRM GPU SVM context 1421 * 1422 * Return: 0 on success, error code on failure. 1423 */ 1424 int xe_svm_alloc_vram(struct xe_tile *tile, struct xe_svm_range *range, 1425 const struct drm_gpusvm_ctx *ctx) 1426 { 1427 struct drm_pagemap *dpagemap; 1428 1429 xe_assert(tile_to_xe(tile), range->base.pages.flags.migrate_devmem); 1430 range_debug(range, "ALLOCATE VRAM"); 1431 1432 dpagemap = tile_local_pagemap(tile); 1433 return drm_pagemap_populate_mm(dpagemap, xe_svm_range_start(range), 1434 xe_svm_range_end(range), 1435 range->base.gpusvm->mm, 1436 ctx->timeslice_ms); 1437 } 1438 1439 static struct drm_pagemap_addr 1440 xe_drm_pagemap_device_map(struct drm_pagemap *dpagemap, 1441 struct device *dev, 1442 struct page *page, 1443 unsigned int order, 1444 enum dma_data_direction dir) 1445 { 1446 struct device *pgmap_dev = dpagemap->dev; 1447 enum drm_interconnect_protocol prot; 1448 dma_addr_t addr; 1449 1450 if (pgmap_dev == dev) { 1451 addr = xe_vram_region_page_to_dpa(page_to_vr(page), page); 1452 prot = XE_INTERCONNECT_VRAM; 1453 } else { 1454 addr = DMA_MAPPING_ERROR; 1455 prot = 0; 1456 } 1457 1458 return drm_pagemap_addr_encode(addr, prot, order, dir); 1459 } 1460 1461 static const struct drm_pagemap_ops xe_drm_pagemap_ops = { 1462 .device_map = xe_drm_pagemap_device_map, 1463 .populate_mm = xe_drm_pagemap_populate_mm, 1464 }; 1465 1466 /** 1467 * xe_devm_add: Remap and provide memmap backing for device memory 1468 * @tile: tile that the memory region belongs to 1469 * @vr: vram memory region to remap 1470 * 1471 * This remap device memory to host physical address space and create 1472 * struct page to back device memory 1473 * 1474 * Return: 0 on success standard error code otherwise 1475 */ 1476 int xe_devm_add(struct xe_tile *tile, struct xe_vram_region *vr) 1477 { 1478 struct xe_device *xe = tile_to_xe(tile); 1479 struct device *dev = &to_pci_dev(xe->drm.dev)->dev; 1480 struct resource *res; 1481 void *addr; 1482 int ret; 1483 1484 res = devm_request_free_mem_region(dev, &iomem_resource, 1485 vr->usable_size); 1486 if (IS_ERR(res)) { 1487 ret = PTR_ERR(res); 1488 return ret; 1489 } 1490 1491 vr->pagemap.type = MEMORY_DEVICE_PRIVATE; 1492 vr->pagemap.range.start = res->start; 1493 vr->pagemap.range.end = res->end; 1494 vr->pagemap.nr_range = 1; 1495 vr->pagemap.ops = drm_pagemap_pagemap_ops_get(); 1496 vr->pagemap.owner = xe_svm_devm_owner(xe); 1497 addr = devm_memremap_pages(dev, &vr->pagemap); 1498 1499 vr->dpagemap.dev = dev; 1500 vr->dpagemap.ops = &xe_drm_pagemap_ops; 1501 1502 if (IS_ERR(addr)) { 1503 devm_release_mem_region(dev, res->start, resource_size(res)); 1504 ret = PTR_ERR(addr); 1505 drm_err(&xe->drm, "Failed to remap tile %d memory, errno %pe\n", 1506 tile->id, ERR_PTR(ret)); 1507 return ret; 1508 } 1509 vr->hpa_base = res->start; 1510 1511 drm_dbg(&xe->drm, "Added tile %d memory [%llx-%llx] to devm, remapped to %pr\n", 1512 tile->id, vr->io_start, vr->io_start + vr->usable_size, res); 1513 return 0; 1514 } 1515 #else 1516 int xe_svm_alloc_vram(struct xe_tile *tile, 1517 struct xe_svm_range *range, 1518 const struct drm_gpusvm_ctx *ctx) 1519 { 1520 return -EOPNOTSUPP; 1521 } 1522 1523 int xe_devm_add(struct xe_tile *tile, struct xe_vram_region *vr) 1524 { 1525 return 0; 1526 } 1527 1528 struct drm_pagemap *xe_vma_resolve_pagemap(struct xe_vma *vma, struct xe_tile *tile) 1529 { 1530 return NULL; 1531 } 1532 #endif 1533 1534 /** 1535 * xe_svm_flush() - SVM flush 1536 * @vm: The VM. 1537 * 1538 * Flush all SVM actions. 1539 */ 1540 void xe_svm_flush(struct xe_vm *vm) 1541 { 1542 if (xe_vm_in_fault_mode(vm)) 1543 flush_work(&vm->svm.garbage_collector.work); 1544 } 1545