1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2025 Intel Corporation 4 */ 5 6 #include "xe_vm_madvise.h" 7 8 #include <linux/nospec.h> 9 #include <drm/xe_drm.h> 10 11 #include "xe_bo.h" 12 #include "xe_pat.h" 13 #include "xe_pt.h" 14 #include "xe_svm.h" 15 #include "xe_tlb_inval.h" 16 #include "xe_vm.h" 17 18 struct xe_vmas_in_madvise_range { 19 u64 addr; 20 u64 range; 21 struct xe_vma **vmas; 22 int num_vmas; 23 bool has_bo_vmas; 24 bool has_svm_userptr_vmas; 25 }; 26 27 /** 28 * struct xe_madvise_details - Argument to madvise_funcs 29 * @dpagemap: Reference-counted pointer to a struct drm_pagemap. 30 * @has_purged_bo: Track if any BO was purged (for purgeable state) 31 * @retained_ptr: User pointer for retained value (for purgeable state) 32 * 33 * The madvise IOCTL handler may, in addition to the user-space 34 * args, have additional info to pass into the madvise_func that 35 * handles the madvise type. Use a struct_xe_madvise_details 36 * for that and extend the struct as necessary. 37 */ 38 struct xe_madvise_details { 39 struct drm_pagemap *dpagemap; 40 bool has_purged_bo; 41 u64 retained_ptr; 42 }; 43 44 static int get_vmas(struct xe_vm *vm, struct xe_vmas_in_madvise_range *madvise_range) 45 { 46 u64 addr = madvise_range->addr; 47 u64 range = madvise_range->range; 48 49 struct xe_vma **__vmas; 50 struct drm_gpuva *gpuva; 51 int max_vmas = 8; 52 53 lockdep_assert_held(&vm->lock); 54 55 madvise_range->num_vmas = 0; 56 madvise_range->vmas = kmalloc_objs(*madvise_range->vmas, max_vmas); 57 if (!madvise_range->vmas) 58 return -ENOMEM; 59 60 vm_dbg(&vm->xe->drm, "VMA's in range: start=0x%016llx, end=0x%016llx", addr, addr + range); 61 62 drm_gpuvm_for_each_va_range(gpuva, &vm->gpuvm, addr, addr + range) { 63 struct xe_vma *vma = gpuva_to_vma(gpuva); 64 65 if (xe_vma_bo(vma)) 66 madvise_range->has_bo_vmas = true; 67 else if (xe_vma_is_cpu_addr_mirror(vma) || xe_vma_is_userptr(vma)) 68 madvise_range->has_svm_userptr_vmas = true; 69 70 if (madvise_range->num_vmas == max_vmas) { 71 max_vmas <<= 1; 72 __vmas = krealloc(madvise_range->vmas, 73 max_vmas * sizeof(*madvise_range->vmas), 74 GFP_KERNEL); 75 if (!__vmas) { 76 kfree(madvise_range->vmas); 77 return -ENOMEM; 78 } 79 madvise_range->vmas = __vmas; 80 } 81 82 madvise_range->vmas[madvise_range->num_vmas] = vma; 83 (madvise_range->num_vmas)++; 84 } 85 86 if (!madvise_range->num_vmas) 87 kfree(madvise_range->vmas); 88 89 vm_dbg(&vm->xe->drm, "madvise_range-num_vmas = %d\n", madvise_range->num_vmas); 90 91 return 0; 92 } 93 94 static void madvise_preferred_mem_loc(struct xe_device *xe, struct xe_vm *vm, 95 struct xe_vma **vmas, int num_vmas, 96 struct drm_xe_madvise *op, 97 struct xe_madvise_details *details) 98 { 99 int i; 100 101 xe_assert(vm->xe, op->type == DRM_XE_MEM_RANGE_ATTR_PREFERRED_LOC); 102 103 for (i = 0; i < num_vmas; i++) { 104 struct xe_vma *vma = vmas[i]; 105 struct xe_vma_preferred_loc *loc = &vma->attr.preferred_loc; 106 107 /*TODO: Extend attributes to bo based vmas */ 108 if ((loc->devmem_fd == op->preferred_mem_loc.devmem_fd && 109 loc->migration_policy == op->preferred_mem_loc.migration_policy) || 110 !xe_vma_is_cpu_addr_mirror(vma)) { 111 vma->skip_invalidation = true; 112 } else { 113 vma->skip_invalidation = false; 114 loc->devmem_fd = op->preferred_mem_loc.devmem_fd; 115 /* Till multi-device support is not added migration_policy 116 * is of no use and can be ignored. 117 */ 118 loc->migration_policy = op->preferred_mem_loc.migration_policy; 119 drm_pagemap_put(loc->dpagemap); 120 loc->dpagemap = NULL; 121 if (details->dpagemap) 122 loc->dpagemap = drm_pagemap_get(details->dpagemap); 123 } 124 } 125 } 126 127 static void madvise_atomic(struct xe_device *xe, struct xe_vm *vm, 128 struct xe_vma **vmas, int num_vmas, 129 struct drm_xe_madvise *op, 130 struct xe_madvise_details *details) 131 { 132 struct xe_bo *bo; 133 int i; 134 135 xe_assert(vm->xe, op->type == DRM_XE_MEM_RANGE_ATTR_ATOMIC); 136 xe_assert(vm->xe, op->atomic.val <= DRM_XE_ATOMIC_CPU); 137 138 for (i = 0; i < num_vmas; i++) { 139 if (xe_vma_is_userptr(vmas[i]) && 140 !(op->atomic.val == DRM_XE_ATOMIC_DEVICE && 141 xe->info.has_device_atomics_on_smem)) { 142 vmas[i]->skip_invalidation = true; 143 continue; 144 } 145 146 if (vmas[i]->attr.atomic_access == op->atomic.val) { 147 vmas[i]->skip_invalidation = true; 148 } else { 149 vmas[i]->skip_invalidation = false; 150 vmas[i]->attr.atomic_access = op->atomic.val; 151 } 152 153 bo = xe_vma_bo(vmas[i]); 154 if (!bo || bo->attr.atomic_access == op->atomic.val) 155 continue; 156 157 vmas[i]->skip_invalidation = false; 158 xe_bo_assert_held(bo); 159 bo->attr.atomic_access = op->atomic.val; 160 161 /* Invalidate cpu page table, so bo can migrate to smem in next access */ 162 if (xe_bo_is_vram(bo) && 163 (bo->attr.atomic_access == DRM_XE_ATOMIC_CPU || 164 bo->attr.atomic_access == DRM_XE_ATOMIC_GLOBAL)) 165 ttm_bo_unmap_virtual(&bo->ttm); 166 } 167 } 168 169 static void madvise_pat_index(struct xe_device *xe, struct xe_vm *vm, 170 struct xe_vma **vmas, int num_vmas, 171 struct drm_xe_madvise *op, 172 struct xe_madvise_details *details) 173 { 174 int i; 175 176 xe_assert(vm->xe, op->type == DRM_XE_MEM_RANGE_ATTR_PAT); 177 178 for (i = 0; i < num_vmas; i++) { 179 if (vmas[i]->attr.pat_index == op->pat_index.val) { 180 vmas[i]->skip_invalidation = true; 181 } else { 182 vmas[i]->skip_invalidation = false; 183 vmas[i]->attr.pat_index = op->pat_index.val; 184 } 185 } 186 } 187 188 /** 189 * madvise_purgeable - Handle purgeable buffer object advice 190 * @xe: XE device 191 * @vm: VM 192 * @vmas: Array of VMAs 193 * @num_vmas: Number of VMAs 194 * @op: Madvise operation 195 * @details: Madvise details for return values 196 * 197 * Handles DONTNEED/WILLNEED/PURGED states. Tracks if any BO was purged 198 * in details->has_purged_bo for later copy to userspace. 199 */ 200 static void madvise_purgeable(struct xe_device *xe, struct xe_vm *vm, 201 struct xe_vma **vmas, int num_vmas, 202 struct drm_xe_madvise *op, 203 struct xe_madvise_details *details) 204 { 205 int i; 206 207 xe_assert(vm->xe, op->type == DRM_XE_VMA_ATTR_PURGEABLE_STATE); 208 209 for (i = 0; i < num_vmas; i++) { 210 struct xe_bo *bo = xe_vma_bo(vmas[i]); 211 212 if (!bo) { 213 /* Purgeable state applies to BOs only, skip non-BO VMAs */ 214 vmas[i]->skip_invalidation = true; 215 continue; 216 } 217 218 /* BO must be locked before modifying madv state */ 219 xe_bo_assert_held(bo); 220 221 /* 222 * Once purged, always purged. Cannot transition back to WILLNEED. 223 * This matches i915 semantics where purged BOs are permanently invalid. 224 */ 225 if (xe_bo_is_purged(bo)) { 226 details->has_purged_bo = true; 227 vmas[i]->skip_invalidation = true; 228 continue; 229 } 230 231 switch (op->purge_state_val.val) { 232 case DRM_XE_VMA_PURGEABLE_STATE_WILLNEED: 233 vmas[i]->skip_invalidation = true; 234 /* Only act on a real DONTNEED -> WILLNEED transition. */ 235 if (vmas[i]->attr.purgeable_state == XE_MADV_PURGEABLE_DONTNEED) { 236 vmas[i]->attr.purgeable_state = XE_MADV_PURGEABLE_WILLNEED; 237 xe_bo_willneed_get_locked(bo); 238 } 239 break; 240 case DRM_XE_VMA_PURGEABLE_STATE_DONTNEED: 241 /* 242 * Don't zap PTEs at DONTNEED time -- pages are still 243 * alive. The zap happens in xe_bo_move_notify() right 244 * before the shrinker frees them. 245 */ 246 vmas[i]->skip_invalidation = true; 247 248 /* Only act on a real WILLNEED -> DONTNEED transition. */ 249 if (vmas[i]->attr.purgeable_state == XE_MADV_PURGEABLE_WILLNEED) { 250 vmas[i]->attr.purgeable_state = XE_MADV_PURGEABLE_DONTNEED; 251 xe_bo_willneed_put_locked(bo); 252 } 253 break; 254 default: 255 /* Should never hit - values validated in madvise_args_are_sane() */ 256 xe_assert(vm->xe, 0); 257 return; 258 } 259 } 260 } 261 262 typedef void (*madvise_func)(struct xe_device *xe, struct xe_vm *vm, 263 struct xe_vma **vmas, int num_vmas, 264 struct drm_xe_madvise *op, 265 struct xe_madvise_details *details); 266 267 static const madvise_func madvise_funcs[] = { 268 [DRM_XE_MEM_RANGE_ATTR_PREFERRED_LOC] = madvise_preferred_mem_loc, 269 [DRM_XE_MEM_RANGE_ATTR_ATOMIC] = madvise_atomic, 270 [DRM_XE_MEM_RANGE_ATTR_PAT] = madvise_pat_index, 271 [DRM_XE_VMA_ATTR_PURGEABLE_STATE] = madvise_purgeable, 272 }; 273 274 static u8 xe_zap_ptes_in_madvise_range(struct xe_vm *vm, u64 start, u64 end) 275 { 276 struct drm_gpuva *gpuva; 277 struct xe_tile *tile; 278 u8 id, tile_mask = 0; 279 280 lockdep_assert_held_write(&vm->lock); 281 282 /* Wait for pending binds */ 283 if (dma_resv_wait_timeout(xe_vm_resv(vm), DMA_RESV_USAGE_BOOKKEEP, 284 false, MAX_SCHEDULE_TIMEOUT) <= 0) 285 XE_WARN_ON(1); 286 287 drm_gpuvm_for_each_va_range(gpuva, &vm->gpuvm, start, end) { 288 struct xe_vma *vma = gpuva_to_vma(gpuva); 289 290 if (vma->skip_invalidation || xe_vma_is_null(vma)) 291 continue; 292 293 if (xe_vma_is_cpu_addr_mirror(vma)) { 294 tile_mask |= xe_svm_ranges_zap_ptes_in_range(vm, 295 xe_vma_start(vma), 296 xe_vma_end(vma)); 297 } else { 298 for_each_tile(tile, vm->xe, id) { 299 if (xe_pt_zap_ptes(tile, vma)) { 300 tile_mask |= BIT(id); 301 302 /* 303 * WRITE_ONCE pairs with READ_ONCE 304 * in xe_vm_has_valid_gpu_mapping() 305 */ 306 WRITE_ONCE(vma->tile_invalidated, 307 vma->tile_invalidated | BIT(id)); 308 } 309 } 310 } 311 } 312 313 return tile_mask; 314 } 315 316 static int xe_vm_invalidate_madvise_range(struct xe_vm *vm, u64 start, u64 end) 317 { 318 u8 tile_mask = xe_zap_ptes_in_madvise_range(vm, start, end); 319 struct xe_tlb_inval_batch batch; 320 int err; 321 322 if (!tile_mask) 323 return 0; 324 325 xe_device_wmb(vm->xe); 326 327 err = xe_tlb_inval_range_tilemask_submit(vm->xe, vm->usm.asid, start, end, 328 tile_mask, &batch); 329 if (!err) 330 xe_tlb_inval_batch_wait(&batch); 331 332 return err; 333 } 334 335 /** 336 * madvise_range_needs_invalidation() - Check whether madvise needs invalidation 337 * @args: madvise ioctl arguments 338 * 339 * Purgeable state updates only touch VMA/BO metadata. PTEs stay valid and are 340 * zapped only if the BO is later purged. 341 * 342 * Return: true when the update needs PTE invalidation. 343 */ 344 static bool madvise_range_needs_invalidation(const struct drm_xe_madvise *args) 345 { 346 return args->type != DRM_XE_VMA_ATTR_PURGEABLE_STATE; 347 } 348 349 static bool madvise_args_are_sane(struct xe_device *xe, const struct drm_xe_madvise *args) 350 { 351 if (XE_IOCTL_DBG(xe, !args)) 352 return false; 353 354 if (XE_IOCTL_DBG(xe, !IS_ALIGNED(args->start, SZ_4K))) 355 return false; 356 357 if (XE_IOCTL_DBG(xe, !IS_ALIGNED(args->range, SZ_4K))) 358 return false; 359 360 if (XE_IOCTL_DBG(xe, args->range < SZ_4K)) 361 return false; 362 363 switch (args->type) { 364 case DRM_XE_MEM_RANGE_ATTR_PREFERRED_LOC: 365 { 366 s32 fd = (s32)args->preferred_mem_loc.devmem_fd; 367 368 if (XE_IOCTL_DBG(xe, fd < DRM_XE_PREFERRED_LOC_DEFAULT_SYSTEM)) 369 return false; 370 371 if (XE_IOCTL_DBG(xe, fd <= DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE && 372 args->preferred_mem_loc.region_instance != 0)) 373 return false; 374 375 if (XE_IOCTL_DBG(xe, args->preferred_mem_loc.migration_policy > 376 DRM_XE_MIGRATE_ONLY_SYSTEM_PAGES)) 377 return false; 378 379 if (XE_IOCTL_DBG(xe, args->preferred_mem_loc.reserved)) 380 return false; 381 break; 382 } 383 case DRM_XE_MEM_RANGE_ATTR_ATOMIC: 384 if (XE_IOCTL_DBG(xe, args->atomic.val > DRM_XE_ATOMIC_CPU)) 385 return false; 386 387 if (XE_IOCTL_DBG(xe, args->atomic.pad)) 388 return false; 389 390 if (XE_IOCTL_DBG(xe, args->atomic.reserved)) 391 return false; 392 393 break; 394 case DRM_XE_MEM_RANGE_ATTR_PAT: 395 { 396 u16 pat_index, coh_mode; 397 398 if (XE_IOCTL_DBG(xe, args->pat_index.val >= xe->pat.n_entries)) 399 return false; 400 401 pat_index = array_index_nospec(args->pat_index.val, xe->pat.n_entries); 402 coh_mode = xe_pat_index_get_coh_mode(xe, pat_index); 403 if (XE_IOCTL_DBG(xe, !coh_mode)) 404 return false; 405 406 if (XE_WARN_ON(coh_mode > XE_COH_2WAY)) 407 return false; 408 409 if (XE_IOCTL_DBG(xe, args->pat_index.pad)) 410 return false; 411 412 if (XE_IOCTL_DBG(xe, args->pat_index.reserved)) 413 return false; 414 break; 415 } 416 case DRM_XE_VMA_ATTR_PURGEABLE_STATE: 417 { 418 u32 val = args->purge_state_val.val; 419 420 if (XE_IOCTL_DBG(xe, !(val == DRM_XE_VMA_PURGEABLE_STATE_WILLNEED || 421 val == DRM_XE_VMA_PURGEABLE_STATE_DONTNEED))) 422 return false; 423 424 if (XE_IOCTL_DBG(xe, args->purge_state_val.pad)) 425 return false; 426 427 break; 428 } 429 default: 430 if (XE_IOCTL_DBG(xe, 1)) 431 return false; 432 } 433 434 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) 435 return false; 436 437 return true; 438 } 439 440 static int xe_madvise_details_init(struct xe_vm *vm, const struct drm_xe_madvise *args, 441 struct xe_madvise_details *details) 442 { 443 struct xe_device *xe = vm->xe; 444 445 memset(details, 0, sizeof(*details)); 446 447 /* Store retained pointer for purgeable state */ 448 if (args->type == DRM_XE_VMA_ATTR_PURGEABLE_STATE) { 449 details->retained_ptr = args->purge_state_val.retained_ptr; 450 return 0; 451 } 452 453 if (args->type == DRM_XE_MEM_RANGE_ATTR_PREFERRED_LOC) { 454 int fd = args->preferred_mem_loc.devmem_fd; 455 struct drm_pagemap *dpagemap; 456 457 if (fd <= 0) 458 return 0; 459 460 dpagemap = xe_drm_pagemap_from_fd(args->preferred_mem_loc.devmem_fd, 461 args->preferred_mem_loc.region_instance); 462 if (XE_IOCTL_DBG(xe, IS_ERR(dpagemap))) 463 return PTR_ERR(dpagemap); 464 465 /* Don't allow a foreign placement without a fast interconnect! */ 466 if (XE_IOCTL_DBG(xe, dpagemap->pagemap->owner != vm->svm.peer.owner)) { 467 drm_pagemap_put(dpagemap); 468 return -ENOLINK; 469 } 470 details->dpagemap = dpagemap; 471 } 472 473 return 0; 474 } 475 476 static void xe_madvise_details_fini(struct xe_madvise_details *details) 477 { 478 drm_pagemap_put(details->dpagemap); 479 } 480 481 static int xe_madvise_purgeable_retained_to_user(const struct xe_madvise_details *details) 482 { 483 u32 retained; 484 485 if (!details->retained_ptr) 486 return 0; 487 488 retained = !details->has_purged_bo; 489 490 if (put_user(retained, (u32 __user *)u64_to_user_ptr(details->retained_ptr))) 491 return -EFAULT; 492 493 return 0; 494 } 495 496 static bool check_pat_args_are_sane(struct xe_device *xe, 497 struct xe_vmas_in_madvise_range *madvise_range, 498 u16 pat_index) 499 { 500 u16 coh_mode = xe_pat_index_get_coh_mode(xe, pat_index); 501 int i; 502 503 /* 504 * Using coh_none with CPU cached buffers is not allowed on iGPU. 505 * On iGPU the GPU shares the LLC with the CPU, so with coh_none 506 * the GPU bypasses CPU caches and reads directly from DRAM, 507 * potentially seeing stale sensitive data from previously freed 508 * pages. On dGPU this restriction does not apply, because the 509 * platform does not provide a non-coherent system memory access 510 * path that would violate the DMA coherency contract. 511 */ 512 if (coh_mode != XE_COH_NONE || IS_DGFX(xe)) 513 return true; 514 515 for (i = 0; i < madvise_range->num_vmas; i++) { 516 struct xe_vma *vma = madvise_range->vmas[i]; 517 struct xe_bo *bo = xe_vma_bo(vma); 518 519 if (bo) { 520 /* BO with WB caching + COH_NONE is not allowed */ 521 if (XE_IOCTL_DBG(xe, bo->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB)) 522 return false; 523 /* Imported dma-buf without caching info, assume cached */ 524 if (XE_IOCTL_DBG(xe, !bo->cpu_caching)) 525 return false; 526 } else if (XE_IOCTL_DBG(xe, xe_vma_is_cpu_addr_mirror(vma) || 527 xe_vma_is_userptr(vma))) 528 /* System memory (userptr/SVM) is always CPU cached */ 529 return false; 530 } 531 532 return true; 533 } 534 535 static bool check_bo_args_are_sane(struct xe_vm *vm, struct xe_vma **vmas, 536 int num_vmas, u32 atomic_val) 537 { 538 struct xe_device *xe = vm->xe; 539 struct xe_bo *bo; 540 int i; 541 542 for (i = 0; i < num_vmas; i++) { 543 bo = xe_vma_bo(vmas[i]); 544 if (!bo) 545 continue; 546 /* 547 * NOTE: The following atomic checks are platform-specific. For example, 548 * if a device supports CXL atomics, these may not be necessary or 549 * may behave differently. 550 */ 551 if (XE_IOCTL_DBG(xe, atomic_val == DRM_XE_ATOMIC_CPU && 552 !(bo->flags & XE_BO_FLAG_SYSTEM))) 553 return false; 554 555 if (XE_IOCTL_DBG(xe, atomic_val == DRM_XE_ATOMIC_DEVICE && 556 !(bo->flags & XE_BO_FLAG_VRAM0) && 557 !(bo->flags & XE_BO_FLAG_VRAM1) && 558 !(bo->flags & XE_BO_FLAG_SYSTEM && 559 xe->info.has_device_atomics_on_smem))) 560 return false; 561 562 if (XE_IOCTL_DBG(xe, atomic_val == DRM_XE_ATOMIC_GLOBAL && 563 (!(bo->flags & XE_BO_FLAG_SYSTEM) || 564 (!(bo->flags & XE_BO_FLAG_VRAM0) && 565 !(bo->flags & XE_BO_FLAG_VRAM1))))) 566 return false; 567 } 568 return true; 569 } 570 /** 571 * xe_vm_madvise_ioctl - Handle MADVise ioctl for a VM 572 * @dev: DRM device pointer 573 * @data: Pointer to ioctl data (drm_xe_madvise*) 574 * @file: DRM file pointer 575 * 576 * Handles the MADVISE ioctl to provide memory advice for vma's within 577 * input range. 578 * 579 * Return: 0 on success or a negative error code on failure. 580 */ 581 int xe_vm_madvise_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 582 { 583 struct xe_device *xe = to_xe_device(dev); 584 struct xe_file *xef = to_xe_file(file); 585 struct drm_xe_madvise *args = data; 586 struct xe_vmas_in_madvise_range madvise_range = { 587 /* 588 * Userspace may pass canonical (sign-extended) addresses. 589 * Strip the sign extension to get the internal non-canonical 590 * form used by the GPUVM, matching xe_vm_bind_ioctl() behavior. 591 */ 592 .addr = xe_device_uncanonicalize_addr(xe, args->start), 593 .range = args->range, 594 }; 595 struct xe_madvise_details details; 596 u16 pat_index, coh_mode; 597 struct xe_vm *vm; 598 struct drm_exec exec; 599 int err, attr_type; 600 bool do_retained; 601 602 vm = xe_vm_lookup(xef, args->vm_id); 603 if (XE_IOCTL_DBG(xe, !vm)) 604 return -EINVAL; 605 606 if (!madvise_args_are_sane(vm->xe, args)) { 607 err = -EINVAL; 608 goto put_vm; 609 } 610 611 /* Cache whether we need to write retained, and validate it's initialized to 0 */ 612 do_retained = args->type == DRM_XE_VMA_ATTR_PURGEABLE_STATE && 613 args->purge_state_val.retained_ptr; 614 if (do_retained) { 615 u32 retained; 616 u32 __user *retained_ptr; 617 618 retained_ptr = u64_to_user_ptr(args->purge_state_val.retained_ptr); 619 if (get_user(retained, retained_ptr)) { 620 err = -EFAULT; 621 goto put_vm; 622 } 623 624 if (XE_IOCTL_DBG(xe, retained != 0)) { 625 err = -EINVAL; 626 goto put_vm; 627 } 628 } 629 630 xe_svm_flush(vm); 631 632 err = down_write_killable(&vm->lock); 633 if (err) 634 goto put_vm; 635 636 if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { 637 err = -ENOENT; 638 goto unlock_vm; 639 } 640 641 err = xe_madvise_details_init(vm, args, &details); 642 if (err) 643 goto unlock_vm; 644 645 err = xe_vm_alloc_madvise_vma(vm, madvise_range.addr, args->range); 646 if (err) 647 goto madv_fini; 648 649 err = get_vmas(vm, &madvise_range); 650 if (err || !madvise_range.num_vmas) 651 goto madv_fini; 652 653 if (args->type == DRM_XE_MEM_RANGE_ATTR_PAT) { 654 pat_index = array_index_nospec(args->pat_index.val, xe->pat.n_entries); 655 coh_mode = xe_pat_index_get_coh_mode(xe, pat_index); 656 if (XE_IOCTL_DBG(xe, madvise_range.has_svm_userptr_vmas && 657 xe_device_is_l2_flush_optimized(xe) && 658 (pat_index != 19 && coh_mode != XE_COH_2WAY))) { 659 err = -EINVAL; 660 goto free_vmas; 661 } 662 } 663 664 if (args->type == DRM_XE_MEM_RANGE_ATTR_PAT) { 665 if (!check_pat_args_are_sane(xe, &madvise_range, 666 args->pat_index.val)) { 667 err = -EINVAL; 668 goto free_vmas; 669 } 670 } 671 672 if (madvise_range.has_bo_vmas) { 673 if (args->type == DRM_XE_MEM_RANGE_ATTR_ATOMIC) { 674 if (!check_bo_args_are_sane(vm, madvise_range.vmas, 675 madvise_range.num_vmas, 676 args->atomic.val)) { 677 err = -EINVAL; 678 goto free_vmas; 679 } 680 } 681 682 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES | DRM_EXEC_INTERRUPTIBLE_WAIT, 0); 683 drm_exec_until_all_locked(&exec) { 684 for (int i = 0; i < madvise_range.num_vmas; i++) { 685 struct xe_bo *bo = xe_vma_bo(madvise_range.vmas[i]); 686 687 if (!bo) 688 continue; 689 690 if (args->type == DRM_XE_MEM_RANGE_ATTR_PAT) { 691 if (XE_IOCTL_DBG(xe, bo->ttm.base.import_attach && 692 xe_device_is_l2_flush_optimized(xe) && 693 (pat_index != 19 && 694 coh_mode != XE_COH_2WAY))) { 695 err = -EINVAL; 696 goto err_fini; 697 } 698 } 699 700 err = drm_exec_lock_obj(&exec, &bo->ttm.base); 701 drm_exec_retry_on_contention(&exec); 702 if (err) 703 goto err_fini; 704 } 705 } 706 } 707 708 if (madvise_range.has_svm_userptr_vmas) { 709 err = xe_svm_notifier_lock_interruptible(vm); 710 if (err) 711 goto err_fini; 712 } 713 714 attr_type = array_index_nospec(args->type, ARRAY_SIZE(madvise_funcs)); 715 716 /* Ensure the madvise function exists for this type */ 717 if (!madvise_funcs[attr_type]) { 718 err = -EINVAL; 719 goto err_fini; 720 } 721 722 madvise_funcs[attr_type](xe, vm, madvise_range.vmas, madvise_range.num_vmas, args, 723 &details); 724 725 if (madvise_range_needs_invalidation(args)) 726 err = xe_vm_invalidate_madvise_range(vm, madvise_range.addr, 727 madvise_range.addr + args->range); 728 729 if (madvise_range.has_svm_userptr_vmas) 730 xe_svm_notifier_unlock(vm); 731 732 err_fini: 733 if (madvise_range.has_bo_vmas) 734 drm_exec_fini(&exec); 735 free_vmas: 736 kfree(madvise_range.vmas); 737 madvise_range.vmas = NULL; 738 madv_fini: 739 xe_madvise_details_fini(&details); 740 unlock_vm: 741 up_write(&vm->lock); 742 743 /* Write retained value to user after releasing all locks */ 744 if (!err && do_retained) 745 err = xe_madvise_purgeable_retained_to_user(&details); 746 put_vm: 747 xe_vm_put(vm); 748 return err; 749 } 750