1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2025 Intel Corporation
4 */
5
6 #include "xe_vm_madvise.h"
7
8 #include <linux/nospec.h>
9 #include <drm/xe_drm.h>
10
11 #include "xe_bo.h"
12 #include "xe_pat.h"
13 #include "xe_pt.h"
14 #include "xe_svm.h"
15 #include "xe_tlb_inval.h"
16 #include "xe_vm.h"
17
18 struct xe_vmas_in_madvise_range {
19 u64 addr;
20 u64 range;
21 struct xe_vma **vmas;
22 int num_vmas;
23 bool has_bo_vmas;
24 bool has_svm_userptr_vmas;
25 };
26
27 /**
28 * struct xe_madvise_details - Argument to madvise_funcs
29 * @dpagemap: Reference-counted pointer to a struct drm_pagemap.
30 * @has_purged_bo: Track if any BO was purged (for purgeable state)
31 * @retained_ptr: User pointer for retained value (for purgeable state)
32 *
33 * The madvise IOCTL handler may, in addition to the user-space
34 * args, have additional info to pass into the madvise_func that
35 * handles the madvise type. Use a struct_xe_madvise_details
36 * for that and extend the struct as necessary.
37 */
38 struct xe_madvise_details {
39 struct drm_pagemap *dpagemap;
40 bool has_purged_bo;
41 u64 retained_ptr;
42 };
43
get_vmas(struct xe_vm * vm,struct xe_vmas_in_madvise_range * madvise_range)44 static int get_vmas(struct xe_vm *vm, struct xe_vmas_in_madvise_range *madvise_range)
45 {
46 u64 addr = madvise_range->addr;
47 u64 range = madvise_range->range;
48
49 struct xe_vma **__vmas;
50 struct drm_gpuva *gpuva;
51 int max_vmas = 8;
52
53 lockdep_assert_held(&vm->lock);
54
55 madvise_range->num_vmas = 0;
56 madvise_range->vmas = kmalloc_objs(*madvise_range->vmas, max_vmas);
57 if (!madvise_range->vmas)
58 return -ENOMEM;
59
60 vm_dbg(&vm->xe->drm, "VMA's in range: start=0x%016llx, end=0x%016llx", addr, addr + range);
61
62 drm_gpuvm_for_each_va_range(gpuva, &vm->gpuvm, addr, addr + range) {
63 struct xe_vma *vma = gpuva_to_vma(gpuva);
64
65 if (xe_vma_bo(vma))
66 madvise_range->has_bo_vmas = true;
67 else if (xe_vma_is_cpu_addr_mirror(vma) || xe_vma_is_userptr(vma))
68 madvise_range->has_svm_userptr_vmas = true;
69
70 if (madvise_range->num_vmas == max_vmas) {
71 max_vmas <<= 1;
72 __vmas = krealloc(madvise_range->vmas,
73 max_vmas * sizeof(*madvise_range->vmas),
74 GFP_KERNEL);
75 if (!__vmas) {
76 kfree(madvise_range->vmas);
77 return -ENOMEM;
78 }
79 madvise_range->vmas = __vmas;
80 }
81
82 madvise_range->vmas[madvise_range->num_vmas] = vma;
83 (madvise_range->num_vmas)++;
84 }
85
86 if (!madvise_range->num_vmas)
87 kfree(madvise_range->vmas);
88
89 vm_dbg(&vm->xe->drm, "madvise_range-num_vmas = %d\n", madvise_range->num_vmas);
90
91 return 0;
92 }
93
madvise_preferred_mem_loc(struct xe_device * xe,struct xe_vm * vm,struct xe_vma ** vmas,int num_vmas,struct drm_xe_madvise * op,struct xe_madvise_details * details)94 static void madvise_preferred_mem_loc(struct xe_device *xe, struct xe_vm *vm,
95 struct xe_vma **vmas, int num_vmas,
96 struct drm_xe_madvise *op,
97 struct xe_madvise_details *details)
98 {
99 int i;
100
101 xe_assert(vm->xe, op->type == DRM_XE_MEM_RANGE_ATTR_PREFERRED_LOC);
102
103 for (i = 0; i < num_vmas; i++) {
104 struct xe_vma *vma = vmas[i];
105 struct xe_vma_preferred_loc *loc = &vma->attr.preferred_loc;
106
107 /*TODO: Extend attributes to bo based vmas */
108 if ((loc->devmem_fd == op->preferred_mem_loc.devmem_fd &&
109 loc->migration_policy == op->preferred_mem_loc.migration_policy) ||
110 !xe_vma_is_cpu_addr_mirror(vma)) {
111 vma->skip_invalidation = true;
112 } else {
113 vma->skip_invalidation = false;
114 loc->devmem_fd = op->preferred_mem_loc.devmem_fd;
115 /* Till multi-device support is not added migration_policy
116 * is of no use and can be ignored.
117 */
118 loc->migration_policy = op->preferred_mem_loc.migration_policy;
119 drm_pagemap_put(loc->dpagemap);
120 loc->dpagemap = NULL;
121 if (details->dpagemap)
122 loc->dpagemap = drm_pagemap_get(details->dpagemap);
123 }
124 }
125 }
126
madvise_atomic(struct xe_device * xe,struct xe_vm * vm,struct xe_vma ** vmas,int num_vmas,struct drm_xe_madvise * op,struct xe_madvise_details * details)127 static void madvise_atomic(struct xe_device *xe, struct xe_vm *vm,
128 struct xe_vma **vmas, int num_vmas,
129 struct drm_xe_madvise *op,
130 struct xe_madvise_details *details)
131 {
132 struct xe_bo *bo;
133 int i;
134
135 xe_assert(vm->xe, op->type == DRM_XE_MEM_RANGE_ATTR_ATOMIC);
136 xe_assert(vm->xe, op->atomic.val <= DRM_XE_ATOMIC_CPU);
137
138 for (i = 0; i < num_vmas; i++) {
139 if (xe_vma_is_userptr(vmas[i]) &&
140 !(op->atomic.val == DRM_XE_ATOMIC_DEVICE &&
141 xe->info.has_device_atomics_on_smem)) {
142 vmas[i]->skip_invalidation = true;
143 continue;
144 }
145
146 if (vmas[i]->attr.atomic_access == op->atomic.val) {
147 vmas[i]->skip_invalidation = true;
148 } else {
149 vmas[i]->skip_invalidation = false;
150 vmas[i]->attr.atomic_access = op->atomic.val;
151 }
152
153 bo = xe_vma_bo(vmas[i]);
154 if (!bo || bo->attr.atomic_access == op->atomic.val)
155 continue;
156
157 vmas[i]->skip_invalidation = false;
158 xe_bo_assert_held(bo);
159 bo->attr.atomic_access = op->atomic.val;
160
161 /* Invalidate cpu page table, so bo can migrate to smem in next access */
162 if (xe_bo_is_vram(bo) &&
163 (bo->attr.atomic_access == DRM_XE_ATOMIC_CPU ||
164 bo->attr.atomic_access == DRM_XE_ATOMIC_GLOBAL))
165 ttm_bo_unmap_virtual(&bo->ttm);
166 }
167 }
168
madvise_pat_index(struct xe_device * xe,struct xe_vm * vm,struct xe_vma ** vmas,int num_vmas,struct drm_xe_madvise * op,struct xe_madvise_details * details)169 static void madvise_pat_index(struct xe_device *xe, struct xe_vm *vm,
170 struct xe_vma **vmas, int num_vmas,
171 struct drm_xe_madvise *op,
172 struct xe_madvise_details *details)
173 {
174 int i;
175
176 xe_assert(vm->xe, op->type == DRM_XE_MEM_RANGE_ATTR_PAT);
177
178 for (i = 0; i < num_vmas; i++) {
179 if (vmas[i]->attr.pat_index == op->pat_index.val) {
180 vmas[i]->skip_invalidation = true;
181 } else {
182 vmas[i]->skip_invalidation = false;
183 vmas[i]->attr.pat_index = op->pat_index.val;
184 }
185 }
186 }
187
188 /**
189 * madvise_purgeable - Handle purgeable buffer object advice
190 * @xe: XE device
191 * @vm: VM
192 * @vmas: Array of VMAs
193 * @num_vmas: Number of VMAs
194 * @op: Madvise operation
195 * @details: Madvise details for return values
196 *
197 * Handles DONTNEED/WILLNEED/PURGED states. Tracks if any BO was purged
198 * in details->has_purged_bo for later copy to userspace.
199 */
madvise_purgeable(struct xe_device * xe,struct xe_vm * vm,struct xe_vma ** vmas,int num_vmas,struct drm_xe_madvise * op,struct xe_madvise_details * details)200 static void madvise_purgeable(struct xe_device *xe, struct xe_vm *vm,
201 struct xe_vma **vmas, int num_vmas,
202 struct drm_xe_madvise *op,
203 struct xe_madvise_details *details)
204 {
205 int i;
206
207 xe_assert(vm->xe, op->type == DRM_XE_VMA_ATTR_PURGEABLE_STATE);
208
209 for (i = 0; i < num_vmas; i++) {
210 struct xe_bo *bo = xe_vma_bo(vmas[i]);
211
212 if (!bo) {
213 /* Purgeable state applies to BOs only, skip non-BO VMAs */
214 vmas[i]->skip_invalidation = true;
215 continue;
216 }
217
218 /* BO must be locked before modifying madv state */
219 xe_bo_assert_held(bo);
220
221 /*
222 * Once purged, always purged. Cannot transition back to WILLNEED.
223 * This matches i915 semantics where purged BOs are permanently invalid.
224 */
225 if (xe_bo_is_purged(bo)) {
226 details->has_purged_bo = true;
227 vmas[i]->skip_invalidation = true;
228 continue;
229 }
230
231 switch (op->purge_state_val.val) {
232 case DRM_XE_VMA_PURGEABLE_STATE_WILLNEED:
233 vmas[i]->skip_invalidation = true;
234 /* Only act on a real DONTNEED -> WILLNEED transition. */
235 if (vmas[i]->attr.purgeable_state == XE_MADV_PURGEABLE_DONTNEED) {
236 vmas[i]->attr.purgeable_state = XE_MADV_PURGEABLE_WILLNEED;
237 xe_bo_willneed_get_locked(bo);
238 }
239 break;
240 case DRM_XE_VMA_PURGEABLE_STATE_DONTNEED:
241 /*
242 * Don't zap PTEs at DONTNEED time -- pages are still
243 * alive. The zap happens in xe_bo_move_notify() right
244 * before the shrinker frees them.
245 */
246 vmas[i]->skip_invalidation = true;
247
248 /* Only act on a real WILLNEED -> DONTNEED transition. */
249 if (vmas[i]->attr.purgeable_state == XE_MADV_PURGEABLE_WILLNEED) {
250 vmas[i]->attr.purgeable_state = XE_MADV_PURGEABLE_DONTNEED;
251 xe_bo_willneed_put_locked(bo);
252 }
253 break;
254 default:
255 /* Should never hit - values validated in madvise_args_are_sane() */
256 xe_assert(vm->xe, 0);
257 return;
258 }
259 }
260 }
261
262 typedef void (*madvise_func)(struct xe_device *xe, struct xe_vm *vm,
263 struct xe_vma **vmas, int num_vmas,
264 struct drm_xe_madvise *op,
265 struct xe_madvise_details *details);
266
267 static const madvise_func madvise_funcs[] = {
268 [DRM_XE_MEM_RANGE_ATTR_PREFERRED_LOC] = madvise_preferred_mem_loc,
269 [DRM_XE_MEM_RANGE_ATTR_ATOMIC] = madvise_atomic,
270 [DRM_XE_MEM_RANGE_ATTR_PAT] = madvise_pat_index,
271 [DRM_XE_VMA_ATTR_PURGEABLE_STATE] = madvise_purgeable,
272 };
273
xe_zap_ptes_in_madvise_range(struct xe_vm * vm,u64 start,u64 end)274 static u8 xe_zap_ptes_in_madvise_range(struct xe_vm *vm, u64 start, u64 end)
275 {
276 struct drm_gpuva *gpuva;
277 struct xe_tile *tile;
278 u8 id, tile_mask = 0;
279
280 lockdep_assert_held_write(&vm->lock);
281
282 /* Wait for pending binds */
283 if (dma_resv_wait_timeout(xe_vm_resv(vm), DMA_RESV_USAGE_BOOKKEEP,
284 false, MAX_SCHEDULE_TIMEOUT) <= 0)
285 XE_WARN_ON(1);
286
287 drm_gpuvm_for_each_va_range(gpuva, &vm->gpuvm, start, end) {
288 struct xe_vma *vma = gpuva_to_vma(gpuva);
289
290 if (vma->skip_invalidation || xe_vma_is_null(vma))
291 continue;
292
293 if (xe_vma_is_cpu_addr_mirror(vma)) {
294 tile_mask |= xe_svm_ranges_zap_ptes_in_range(vm,
295 xe_vma_start(vma),
296 xe_vma_end(vma));
297 } else {
298 for_each_tile(tile, vm->xe, id) {
299 if (xe_pt_zap_ptes(tile, vma)) {
300 tile_mask |= BIT(id);
301
302 /*
303 * WRITE_ONCE pairs with READ_ONCE
304 * in xe_vm_has_valid_gpu_mapping()
305 */
306 WRITE_ONCE(vma->tile_invalidated,
307 vma->tile_invalidated | BIT(id));
308 }
309 }
310 }
311 }
312
313 return tile_mask;
314 }
315
xe_vm_invalidate_madvise_range(struct xe_vm * vm,u64 start,u64 end)316 static int xe_vm_invalidate_madvise_range(struct xe_vm *vm, u64 start, u64 end)
317 {
318 u8 tile_mask = xe_zap_ptes_in_madvise_range(vm, start, end);
319 struct xe_tlb_inval_batch batch;
320 int err;
321
322 if (!tile_mask)
323 return 0;
324
325 xe_device_wmb(vm->xe);
326
327 err = xe_tlb_inval_range_tilemask_submit(vm->xe, vm->usm.asid, start, end,
328 tile_mask, &batch);
329 if (!err)
330 xe_tlb_inval_batch_wait(&batch);
331
332 return err;
333 }
334
335 /**
336 * madvise_range_needs_invalidation() - Check whether madvise needs invalidation
337 * @args: madvise ioctl arguments
338 *
339 * Purgeable state updates only touch VMA/BO metadata. PTEs stay valid and are
340 * zapped only if the BO is later purged.
341 *
342 * Return: true when the update needs PTE invalidation.
343 */
madvise_range_needs_invalidation(const struct drm_xe_madvise * args)344 static bool madvise_range_needs_invalidation(const struct drm_xe_madvise *args)
345 {
346 return args->type != DRM_XE_VMA_ATTR_PURGEABLE_STATE;
347 }
348
madvise_args_are_sane(struct xe_device * xe,const struct drm_xe_madvise * args)349 static bool madvise_args_are_sane(struct xe_device *xe, const struct drm_xe_madvise *args)
350 {
351 if (XE_IOCTL_DBG(xe, !args))
352 return false;
353
354 if (XE_IOCTL_DBG(xe, !IS_ALIGNED(args->start, SZ_4K)))
355 return false;
356
357 if (XE_IOCTL_DBG(xe, !IS_ALIGNED(args->range, SZ_4K)))
358 return false;
359
360 if (XE_IOCTL_DBG(xe, args->range < SZ_4K))
361 return false;
362
363 switch (args->type) {
364 case DRM_XE_MEM_RANGE_ATTR_PREFERRED_LOC:
365 {
366 s32 fd = (s32)args->preferred_mem_loc.devmem_fd;
367
368 if (XE_IOCTL_DBG(xe, fd < DRM_XE_PREFERRED_LOC_DEFAULT_SYSTEM))
369 return false;
370
371 if (XE_IOCTL_DBG(xe, fd <= DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE &&
372 args->preferred_mem_loc.region_instance != 0))
373 return false;
374
375 if (XE_IOCTL_DBG(xe, args->preferred_mem_loc.migration_policy >
376 DRM_XE_MIGRATE_ONLY_SYSTEM_PAGES))
377 return false;
378
379 if (XE_IOCTL_DBG(xe, args->preferred_mem_loc.reserved))
380 return false;
381 break;
382 }
383 case DRM_XE_MEM_RANGE_ATTR_ATOMIC:
384 if (XE_IOCTL_DBG(xe, args->atomic.val > DRM_XE_ATOMIC_CPU))
385 return false;
386
387 if (XE_IOCTL_DBG(xe, args->atomic.pad))
388 return false;
389
390 if (XE_IOCTL_DBG(xe, args->atomic.reserved))
391 return false;
392
393 break;
394 case DRM_XE_MEM_RANGE_ATTR_PAT:
395 {
396 u16 pat_index, coh_mode;
397
398 if (XE_IOCTL_DBG(xe, args->pat_index.val >= xe->pat.n_entries))
399 return false;
400
401 pat_index = array_index_nospec(args->pat_index.val, xe->pat.n_entries);
402 coh_mode = xe_pat_index_get_coh_mode(xe, pat_index);
403 if (XE_IOCTL_DBG(xe, !coh_mode))
404 return false;
405
406 if (XE_WARN_ON(coh_mode > XE_COH_2WAY))
407 return false;
408
409 if (XE_IOCTL_DBG(xe, args->pat_index.pad))
410 return false;
411
412 if (XE_IOCTL_DBG(xe, args->pat_index.reserved))
413 return false;
414 break;
415 }
416 case DRM_XE_VMA_ATTR_PURGEABLE_STATE:
417 {
418 u32 val = args->purge_state_val.val;
419
420 if (XE_IOCTL_DBG(xe, !(val == DRM_XE_VMA_PURGEABLE_STATE_WILLNEED ||
421 val == DRM_XE_VMA_PURGEABLE_STATE_DONTNEED)))
422 return false;
423
424 if (XE_IOCTL_DBG(xe, args->purge_state_val.pad))
425 return false;
426
427 break;
428 }
429 default:
430 if (XE_IOCTL_DBG(xe, 1))
431 return false;
432 }
433
434 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
435 return false;
436
437 return true;
438 }
439
xe_madvise_details_init(struct xe_vm * vm,const struct drm_xe_madvise * args,struct xe_madvise_details * details)440 static int xe_madvise_details_init(struct xe_vm *vm, const struct drm_xe_madvise *args,
441 struct xe_madvise_details *details)
442 {
443 struct xe_device *xe = vm->xe;
444
445 memset(details, 0, sizeof(*details));
446
447 /* Store retained pointer for purgeable state */
448 if (args->type == DRM_XE_VMA_ATTR_PURGEABLE_STATE) {
449 details->retained_ptr = args->purge_state_val.retained_ptr;
450 return 0;
451 }
452
453 if (args->type == DRM_XE_MEM_RANGE_ATTR_PREFERRED_LOC) {
454 int fd = args->preferred_mem_loc.devmem_fd;
455 struct drm_pagemap *dpagemap;
456
457 if (fd <= 0)
458 return 0;
459
460 dpagemap = xe_drm_pagemap_from_fd(args->preferred_mem_loc.devmem_fd,
461 args->preferred_mem_loc.region_instance);
462 if (XE_IOCTL_DBG(xe, IS_ERR(dpagemap)))
463 return PTR_ERR(dpagemap);
464
465 /* Don't allow a foreign placement without a fast interconnect! */
466 if (XE_IOCTL_DBG(xe, dpagemap->pagemap->owner != vm->svm.peer.owner)) {
467 drm_pagemap_put(dpagemap);
468 return -ENOLINK;
469 }
470 details->dpagemap = dpagemap;
471 }
472
473 return 0;
474 }
475
xe_madvise_details_fini(struct xe_madvise_details * details)476 static void xe_madvise_details_fini(struct xe_madvise_details *details)
477 {
478 drm_pagemap_put(details->dpagemap);
479 }
480
xe_madvise_purgeable_retained_to_user(const struct xe_madvise_details * details)481 static int xe_madvise_purgeable_retained_to_user(const struct xe_madvise_details *details)
482 {
483 u32 retained;
484
485 if (!details->retained_ptr)
486 return 0;
487
488 retained = !details->has_purged_bo;
489
490 if (put_user(retained, (u32 __user *)u64_to_user_ptr(details->retained_ptr)))
491 return -EFAULT;
492
493 return 0;
494 }
495
check_pat_args_are_sane(struct xe_device * xe,struct xe_vmas_in_madvise_range * madvise_range,u16 pat_index)496 static bool check_pat_args_are_sane(struct xe_device *xe,
497 struct xe_vmas_in_madvise_range *madvise_range,
498 u16 pat_index)
499 {
500 u16 coh_mode = xe_pat_index_get_coh_mode(xe, pat_index);
501 int i;
502
503 /*
504 * Using coh_none with CPU cached buffers is not allowed on iGPU.
505 * On iGPU the GPU shares the LLC with the CPU, so with coh_none
506 * the GPU bypasses CPU caches and reads directly from DRAM,
507 * potentially seeing stale sensitive data from previously freed
508 * pages. On dGPU this restriction does not apply, because the
509 * platform does not provide a non-coherent system memory access
510 * path that would violate the DMA coherency contract.
511 */
512 if (coh_mode != XE_COH_NONE || IS_DGFX(xe))
513 return true;
514
515 for (i = 0; i < madvise_range->num_vmas; i++) {
516 struct xe_vma *vma = madvise_range->vmas[i];
517 struct xe_bo *bo = xe_vma_bo(vma);
518
519 if (bo) {
520 /* BO with WB caching + COH_NONE is not allowed */
521 if (XE_IOCTL_DBG(xe, bo->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB))
522 return false;
523 /* Imported dma-buf without caching info, assume cached */
524 if (XE_IOCTL_DBG(xe, !bo->cpu_caching))
525 return false;
526 } else if (XE_IOCTL_DBG(xe, xe_vma_is_cpu_addr_mirror(vma) ||
527 xe_vma_is_userptr(vma)))
528 /* System memory (userptr/SVM) is always CPU cached */
529 return false;
530 }
531
532 return true;
533 }
534
check_bo_args_are_sane(struct xe_vm * vm,struct xe_vma ** vmas,int num_vmas,u32 atomic_val)535 static bool check_bo_args_are_sane(struct xe_vm *vm, struct xe_vma **vmas,
536 int num_vmas, u32 atomic_val)
537 {
538 struct xe_device *xe = vm->xe;
539 struct xe_bo *bo;
540 int i;
541
542 for (i = 0; i < num_vmas; i++) {
543 bo = xe_vma_bo(vmas[i]);
544 if (!bo)
545 continue;
546 /*
547 * NOTE: The following atomic checks are platform-specific. For example,
548 * if a device supports CXL atomics, these may not be necessary or
549 * may behave differently.
550 */
551 if (XE_IOCTL_DBG(xe, atomic_val == DRM_XE_ATOMIC_CPU &&
552 !(bo->flags & XE_BO_FLAG_SYSTEM)))
553 return false;
554
555 if (XE_IOCTL_DBG(xe, atomic_val == DRM_XE_ATOMIC_DEVICE &&
556 !(bo->flags & XE_BO_FLAG_VRAM0) &&
557 !(bo->flags & XE_BO_FLAG_VRAM1) &&
558 !(bo->flags & XE_BO_FLAG_SYSTEM &&
559 xe->info.has_device_atomics_on_smem)))
560 return false;
561
562 if (XE_IOCTL_DBG(xe, atomic_val == DRM_XE_ATOMIC_GLOBAL &&
563 (!(bo->flags & XE_BO_FLAG_SYSTEM) ||
564 (!(bo->flags & XE_BO_FLAG_VRAM0) &&
565 !(bo->flags & XE_BO_FLAG_VRAM1)))))
566 return false;
567 }
568 return true;
569 }
570 /**
571 * xe_vm_madvise_ioctl - Handle MADVise ioctl for a VM
572 * @dev: DRM device pointer
573 * @data: Pointer to ioctl data (drm_xe_madvise*)
574 * @file: DRM file pointer
575 *
576 * Handles the MADVISE ioctl to provide memory advice for vma's within
577 * input range.
578 *
579 * Return: 0 on success or a negative error code on failure.
580 */
xe_vm_madvise_ioctl(struct drm_device * dev,void * data,struct drm_file * file)581 int xe_vm_madvise_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
582 {
583 struct xe_device *xe = to_xe_device(dev);
584 struct xe_file *xef = to_xe_file(file);
585 struct drm_xe_madvise *args = data;
586 struct xe_vmas_in_madvise_range madvise_range = {
587 /*
588 * Userspace may pass canonical (sign-extended) addresses.
589 * Strip the sign extension to get the internal non-canonical
590 * form used by the GPUVM, matching xe_vm_bind_ioctl() behavior.
591 */
592 .addr = xe_device_uncanonicalize_addr(xe, args->start),
593 .range = args->range,
594 };
595 struct xe_madvise_details details;
596 u16 pat_index, coh_mode;
597 struct xe_vm *vm;
598 struct drm_exec exec;
599 int err, attr_type;
600 bool do_retained;
601
602 vm = xe_vm_lookup(xef, args->vm_id);
603 if (XE_IOCTL_DBG(xe, !vm))
604 return -EINVAL;
605
606 if (!madvise_args_are_sane(vm->xe, args)) {
607 err = -EINVAL;
608 goto put_vm;
609 }
610
611 /* Cache whether we need to write retained, and validate it's initialized to 0 */
612 do_retained = args->type == DRM_XE_VMA_ATTR_PURGEABLE_STATE &&
613 args->purge_state_val.retained_ptr;
614 if (do_retained) {
615 u32 retained;
616 u32 __user *retained_ptr;
617
618 retained_ptr = u64_to_user_ptr(args->purge_state_val.retained_ptr);
619 if (get_user(retained, retained_ptr)) {
620 err = -EFAULT;
621 goto put_vm;
622 }
623
624 if (XE_IOCTL_DBG(xe, retained != 0)) {
625 err = -EINVAL;
626 goto put_vm;
627 }
628 }
629
630 xe_svm_flush(vm);
631
632 err = down_write_killable(&vm->lock);
633 if (err)
634 goto put_vm;
635
636 if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) {
637 err = -ENOENT;
638 goto unlock_vm;
639 }
640
641 err = xe_madvise_details_init(vm, args, &details);
642 if (err)
643 goto unlock_vm;
644
645 err = xe_vm_alloc_madvise_vma(vm, madvise_range.addr, args->range);
646 if (err)
647 goto madv_fini;
648
649 err = get_vmas(vm, &madvise_range);
650 if (err || !madvise_range.num_vmas)
651 goto madv_fini;
652
653 if (args->type == DRM_XE_MEM_RANGE_ATTR_PAT) {
654 pat_index = array_index_nospec(args->pat_index.val, xe->pat.n_entries);
655 coh_mode = xe_pat_index_get_coh_mode(xe, pat_index);
656 if (XE_IOCTL_DBG(xe, madvise_range.has_svm_userptr_vmas &&
657 xe_device_is_l2_flush_optimized(xe) &&
658 (pat_index != 19 && coh_mode != XE_COH_2WAY))) {
659 err = -EINVAL;
660 goto free_vmas;
661 }
662 }
663
664 if (args->type == DRM_XE_MEM_RANGE_ATTR_PAT) {
665 if (!check_pat_args_are_sane(xe, &madvise_range,
666 args->pat_index.val)) {
667 err = -EINVAL;
668 goto free_vmas;
669 }
670 }
671
672 if (madvise_range.has_bo_vmas) {
673 if (args->type == DRM_XE_MEM_RANGE_ATTR_ATOMIC) {
674 if (!check_bo_args_are_sane(vm, madvise_range.vmas,
675 madvise_range.num_vmas,
676 args->atomic.val)) {
677 err = -EINVAL;
678 goto free_vmas;
679 }
680 }
681
682 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES | DRM_EXEC_INTERRUPTIBLE_WAIT, 0);
683 drm_exec_until_all_locked(&exec) {
684 for (int i = 0; i < madvise_range.num_vmas; i++) {
685 struct xe_bo *bo = xe_vma_bo(madvise_range.vmas[i]);
686
687 if (!bo)
688 continue;
689
690 if (args->type == DRM_XE_MEM_RANGE_ATTR_PAT) {
691 if (XE_IOCTL_DBG(xe, bo->ttm.base.import_attach &&
692 xe_device_is_l2_flush_optimized(xe) &&
693 (pat_index != 19 &&
694 coh_mode != XE_COH_2WAY))) {
695 err = -EINVAL;
696 goto err_fini;
697 }
698 }
699
700 err = drm_exec_lock_obj(&exec, &bo->ttm.base);
701 drm_exec_retry_on_contention(&exec);
702 if (err)
703 goto err_fini;
704 }
705 }
706 }
707
708 if (madvise_range.has_svm_userptr_vmas) {
709 err = xe_svm_notifier_lock_interruptible(vm);
710 if (err)
711 goto err_fini;
712 }
713
714 attr_type = array_index_nospec(args->type, ARRAY_SIZE(madvise_funcs));
715
716 /* Ensure the madvise function exists for this type */
717 if (!madvise_funcs[attr_type]) {
718 err = -EINVAL;
719 goto err_fini;
720 }
721
722 madvise_funcs[attr_type](xe, vm, madvise_range.vmas, madvise_range.num_vmas, args,
723 &details);
724
725 if (madvise_range_needs_invalidation(args))
726 err = xe_vm_invalidate_madvise_range(vm, madvise_range.addr,
727 madvise_range.addr + args->range);
728
729 if (madvise_range.has_svm_userptr_vmas)
730 xe_svm_notifier_unlock(vm);
731
732 err_fini:
733 if (madvise_range.has_bo_vmas)
734 drm_exec_fini(&exec);
735 free_vmas:
736 kfree(madvise_range.vmas);
737 madvise_range.vmas = NULL;
738 madv_fini:
739 xe_madvise_details_fini(&details);
740 unlock_vm:
741 up_write(&vm->lock);
742
743 /* Write retained value to user after releasing all locks */
744 if (!err && do_retained)
745 err = xe_madvise_purgeable_retained_to_user(&details);
746 put_vm:
747 xe_vm_put(vm);
748 return err;
749 }
750