1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25 #include <linux/list_sort.h>
26 #include <linux/prime_numbers.h>
27
28 #include "gem/i915_gem_context.h"
29 #include "gem/i915_gem_internal.h"
30 #include "gem/i915_gem_lmem.h"
31 #include "gem/i915_gem_region.h"
32 #include "gem/selftests/mock_context.h"
33 #include "gt/intel_context.h"
34 #include "gt/intel_gpu_commands.h"
35 #include "gt/intel_gtt.h"
36
37 #include "i915_random.h"
38 #include "i915_selftest.h"
39 #include "i915_vma_resource.h"
40
41 #include "mock_drm.h"
42 #include "mock_gem_device.h"
43 #include "mock_gtt.h"
44 #include "igt_flush_test.h"
45
cleanup_freed_objects(struct drm_i915_private * i915)46 static void cleanup_freed_objects(struct drm_i915_private *i915)
47 {
48 i915_gem_drain_freed_objects(i915);
49 }
50
fake_free_pages(struct drm_i915_gem_object * obj,struct sg_table * pages)51 static void fake_free_pages(struct drm_i915_gem_object *obj,
52 struct sg_table *pages)
53 {
54 sg_free_table(pages);
55 kfree(pages);
56 }
57
fake_get_pages(struct drm_i915_gem_object * obj)58 static int fake_get_pages(struct drm_i915_gem_object *obj)
59 {
60 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY)
61 #define PFN_BIAS 0x1000
62 struct sg_table *pages;
63 struct scatterlist *sg;
64 typeof(obj->base.size) rem;
65
66 pages = kmalloc_obj(*pages, GFP);
67 if (!pages)
68 return -ENOMEM;
69
70 rem = round_up(obj->base.size, BIT(31)) >> 31;
71 /* restricted by sg_alloc_table */
72 if (overflows_type(rem, unsigned int)) {
73 kfree(pages);
74 return -E2BIG;
75 }
76
77 if (sg_alloc_table(pages, rem, GFP)) {
78 kfree(pages);
79 return -ENOMEM;
80 }
81
82 rem = obj->base.size;
83 for (sg = pages->sgl; sg; sg = sg_next(sg)) {
84 unsigned long len = min_t(typeof(rem), rem, BIT(31));
85
86 GEM_BUG_ON(!len);
87 sg_set_page(sg, pfn_to_page(PFN_BIAS), len, 0);
88 sg_dma_address(sg) = page_to_phys(sg_page(sg));
89 sg_dma_len(sg) = len;
90
91 rem -= len;
92 }
93 GEM_BUG_ON(rem);
94
95 __i915_gem_object_set_pages(obj, pages);
96
97 return 0;
98 #undef GFP
99 }
100
fake_put_pages(struct drm_i915_gem_object * obj,struct sg_table * pages)101 static void fake_put_pages(struct drm_i915_gem_object *obj,
102 struct sg_table *pages)
103 {
104 fake_free_pages(obj, pages);
105 obj->mm.dirty = false;
106 }
107
108 static const struct drm_i915_gem_object_ops fake_ops = {
109 .name = "fake-gem",
110 .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
111 .get_pages = fake_get_pages,
112 .put_pages = fake_put_pages,
113 };
114
115 static struct drm_i915_gem_object *
fake_dma_object(struct drm_i915_private * i915,u64 size)116 fake_dma_object(struct drm_i915_private *i915, u64 size)
117 {
118 static struct lock_class_key lock_class;
119 struct drm_i915_gem_object *obj;
120
121 GEM_BUG_ON(!size);
122 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
123
124 if (overflows_type(size, obj->base.size))
125 return ERR_PTR(-E2BIG);
126
127 obj = i915_gem_object_alloc();
128 if (!obj)
129 goto err;
130
131 drm_gem_private_object_init(&i915->drm, &obj->base, size);
132 i915_gem_object_init(obj, &fake_ops, &lock_class, 0);
133
134 i915_gem_object_set_volatile(obj);
135
136 obj->write_domain = I915_GEM_DOMAIN_CPU;
137 obj->read_domains = I915_GEM_DOMAIN_CPU;
138 obj->pat_index = i915_gem_get_pat_index(i915, I915_CACHE_NONE);
139
140 /* Preallocate the "backing storage" */
141 if (i915_gem_object_pin_pages_unlocked(obj))
142 goto err_obj;
143
144 i915_gem_object_unpin_pages(obj);
145 return obj;
146
147 err_obj:
148 i915_gem_object_put(obj);
149 err:
150 return ERR_PTR(-ENOMEM);
151 }
152
igt_ppgtt_alloc(void * arg)153 static int igt_ppgtt_alloc(void *arg)
154 {
155 struct drm_i915_private *dev_priv = arg;
156 struct i915_ppgtt *ppgtt;
157 struct i915_gem_ww_ctx ww;
158 u64 size, last, limit;
159 int err = 0;
160
161 /* Allocate a ppggt and try to fill the entire range */
162
163 if (!HAS_PPGTT(dev_priv))
164 return 0;
165
166 ppgtt = i915_ppgtt_create(to_gt(dev_priv), 0);
167 if (IS_ERR(ppgtt))
168 return PTR_ERR(ppgtt);
169
170 if (!ppgtt->vm.allocate_va_range)
171 goto ppgtt_vm_put;
172
173 /*
174 * While we only allocate the page tables here and so we could
175 * address a much larger GTT than we could actually fit into
176 * RAM, a practical limit is the amount of physical pages in the system.
177 * This should ensure that we do not run into the oomkiller during
178 * the test and take down the machine wilfully.
179 */
180 limit = totalram_pages() << PAGE_SHIFT;
181 limit = min(ppgtt->vm.total, limit);
182
183 i915_gem_ww_ctx_init(&ww, false);
184 retry:
185 err = i915_vm_lock_objects(&ppgtt->vm, &ww);
186 if (err)
187 goto err_ppgtt_cleanup;
188
189 /* Check we can allocate the entire range */
190 for (size = 4096; size <= limit; size <<= 2) {
191 struct i915_vm_pt_stash stash = {};
192
193 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, size);
194 if (err)
195 goto err_ppgtt_cleanup;
196
197 err = i915_vm_map_pt_stash(&ppgtt->vm, &stash);
198 if (err) {
199 i915_vm_free_pt_stash(&ppgtt->vm, &stash);
200 goto err_ppgtt_cleanup;
201 }
202
203 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, size);
204 cond_resched();
205
206 ppgtt->vm.clear_range(&ppgtt->vm, 0, size);
207
208 i915_vm_free_pt_stash(&ppgtt->vm, &stash);
209 }
210
211 /* Check we can incrementally allocate the entire range */
212 for (last = 0, size = 4096; size <= limit; last = size, size <<= 2) {
213 struct i915_vm_pt_stash stash = {};
214
215 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, size - last);
216 if (err)
217 goto err_ppgtt_cleanup;
218
219 err = i915_vm_map_pt_stash(&ppgtt->vm, &stash);
220 if (err) {
221 i915_vm_free_pt_stash(&ppgtt->vm, &stash);
222 goto err_ppgtt_cleanup;
223 }
224
225 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash,
226 last, size - last);
227 cond_resched();
228
229 i915_vm_free_pt_stash(&ppgtt->vm, &stash);
230 }
231
232 err_ppgtt_cleanup:
233 if (err == -EDEADLK) {
234 err = i915_gem_ww_ctx_backoff(&ww);
235 if (!err)
236 goto retry;
237 }
238 i915_gem_ww_ctx_fini(&ww);
239 ppgtt_vm_put:
240 i915_vm_put(&ppgtt->vm);
241 return err;
242 }
243
lowlevel_hole(struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time)244 static int lowlevel_hole(struct i915_address_space *vm,
245 u64 hole_start, u64 hole_end,
246 unsigned long end_time)
247 {
248 const unsigned int min_alignment =
249 i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM);
250 I915_RND_STATE(seed_prng);
251 struct i915_vma_resource *mock_vma_res;
252 unsigned int size;
253
254 mock_vma_res = kzalloc(sizeof(*mock_vma_res), GFP_KERNEL);
255 if (!mock_vma_res)
256 return -ENOMEM;
257
258 /* Keep creating larger objects until one cannot fit into the hole */
259 for (size = 12; (hole_end - hole_start) >> size; size++) {
260 I915_RND_SUBSTATE(prng, seed_prng);
261 struct drm_i915_gem_object *obj;
262 unsigned int *order, count, n;
263 u64 hole_size, aligned_size;
264
265 aligned_size = max_t(u32, ilog2(min_alignment), size);
266 hole_size = (hole_end - hole_start) >> aligned_size;
267 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32))
268 hole_size = KMALLOC_MAX_SIZE / sizeof(u32);
269 count = hole_size >> 1;
270 if (!count) {
271 pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n",
272 __func__, hole_start, hole_end, size, hole_size);
273 break;
274 }
275
276 do {
277 order = i915_random_order(count, &prng);
278 if (order)
279 break;
280 } while (count >>= 1);
281 if (!count) {
282 kfree(mock_vma_res);
283 return -ENOMEM;
284 }
285 GEM_BUG_ON(!order);
286
287 GEM_BUG_ON(count * BIT_ULL(aligned_size) > vm->total);
288 GEM_BUG_ON(hole_start + count * BIT_ULL(aligned_size) > hole_end);
289
290 /*
291 * Ignore allocation failures (i.e. don't report them as
292 * a test failure) as we are purposefully allocating very
293 * large objects without checking that we have sufficient
294 * memory. We expect to hit -ENOMEM.
295 */
296
297 obj = fake_dma_object(vm->i915, BIT_ULL(size));
298 if (IS_ERR(obj)) {
299 kfree(order);
300 break;
301 }
302
303 GEM_BUG_ON(obj->base.size != BIT_ULL(size));
304
305 if (i915_gem_object_pin_pages_unlocked(obj)) {
306 i915_gem_object_put(obj);
307 kfree(order);
308 break;
309 }
310
311 for (n = 0; n < count; n++) {
312 u64 addr = hole_start + order[n] * BIT_ULL(aligned_size);
313 intel_wakeref_t wakeref;
314
315 GEM_BUG_ON(addr + BIT_ULL(aligned_size) > vm->total);
316
317 if (igt_timeout(end_time,
318 "%s timed out before %d/%d\n",
319 __func__, n, count)) {
320 hole_end = hole_start; /* quit */
321 break;
322 }
323
324 if (vm->allocate_va_range) {
325 struct i915_vm_pt_stash stash = {};
326 struct i915_gem_ww_ctx ww;
327 int err;
328
329 i915_gem_ww_ctx_init(&ww, false);
330 retry:
331 err = i915_vm_lock_objects(vm, &ww);
332 if (err)
333 goto alloc_vm_end;
334
335 err = -ENOMEM;
336 if (i915_vm_alloc_pt_stash(vm, &stash,
337 BIT_ULL(size)))
338 goto alloc_vm_end;
339
340 err = i915_vm_map_pt_stash(vm, &stash);
341 if (!err)
342 vm->allocate_va_range(vm, &stash,
343 addr, BIT_ULL(size));
344 i915_vm_free_pt_stash(vm, &stash);
345 alloc_vm_end:
346 if (err == -EDEADLK) {
347 err = i915_gem_ww_ctx_backoff(&ww);
348 if (!err)
349 goto retry;
350 }
351 i915_gem_ww_ctx_fini(&ww);
352
353 if (err)
354 break;
355 }
356
357 mock_vma_res->bi.pages = obj->mm.pages;
358 mock_vma_res->node_size = BIT_ULL(aligned_size);
359 mock_vma_res->start = addr;
360
361 with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref)
362 vm->insert_entries(vm, mock_vma_res,
363 i915_gem_get_pat_index(vm->i915,
364 I915_CACHE_NONE),
365 0);
366 }
367 count = n;
368
369 i915_random_reorder(order, count, &prng);
370 for (n = 0; n < count; n++) {
371 u64 addr = hole_start + order[n] * BIT_ULL(aligned_size);
372 intel_wakeref_t wakeref;
373
374 GEM_BUG_ON(addr + BIT_ULL(size) > vm->total);
375 with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref)
376 vm->clear_range(vm, addr, BIT_ULL(size));
377 }
378
379 i915_gem_object_unpin_pages(obj);
380 i915_gem_object_put(obj);
381
382 kfree(order);
383
384 cleanup_freed_objects(vm->i915);
385 }
386
387 kfree(mock_vma_res);
388 return 0;
389 }
390
close_object_list(struct list_head * objects,struct i915_address_space * vm)391 static void close_object_list(struct list_head *objects,
392 struct i915_address_space *vm)
393 {
394 struct drm_i915_gem_object *obj, *on;
395 int __maybe_unused ignored;
396
397 list_for_each_entry_safe(obj, on, objects, st_link) {
398 struct i915_vma *vma;
399
400 vma = i915_vma_instance(obj, vm, NULL);
401 if (!IS_ERR(vma))
402 ignored = i915_vma_unbind_unlocked(vma);
403
404 list_del(&obj->st_link);
405 i915_gem_object_put(obj);
406 }
407 }
408
fill_hole(struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time)409 static int fill_hole(struct i915_address_space *vm,
410 u64 hole_start, u64 hole_end,
411 unsigned long end_time)
412 {
413 const u64 hole_size = hole_end - hole_start;
414 struct drm_i915_gem_object *obj;
415 const unsigned int min_alignment =
416 i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM);
417 const unsigned long max_pages =
418 min_t(u64, ULONG_MAX - 1, (hole_size / 2) >> ilog2(min_alignment));
419 const unsigned long max_step = max(int_sqrt(max_pages), 2UL);
420 unsigned long npages, prime, flags;
421 struct i915_vma *vma;
422 LIST_HEAD(objects);
423 int err;
424
425 /* Try binding many VMA working inwards from either edge */
426
427 flags = PIN_OFFSET_FIXED | PIN_USER;
428 if (i915_is_ggtt(vm))
429 flags |= PIN_GLOBAL;
430
431 for_each_prime_number_from(prime, 2, max_step) {
432 for (npages = 1; npages <= max_pages; npages *= prime) {
433 const u64 full_size = npages << PAGE_SHIFT;
434 const struct {
435 const char *name;
436 u64 offset;
437 int step;
438 } phases[] = {
439 { "top-down", hole_end, -1, },
440 { "bottom-up", hole_start, 1, },
441 { }
442 }, *p;
443
444 obj = fake_dma_object(vm->i915, full_size);
445 if (IS_ERR(obj))
446 break;
447
448 list_add(&obj->st_link, &objects);
449
450 /*
451 * Align differing sized objects against the edges, and
452 * check we don't walk off into the void when binding
453 * them into the GTT.
454 */
455 for (p = phases; p->name; p++) {
456 u64 offset;
457
458 offset = p->offset;
459 list_for_each_entry(obj, &objects, st_link) {
460 u64 aligned_size = round_up(obj->base.size,
461 min_alignment);
462
463 vma = i915_vma_instance(obj, vm, NULL);
464 if (IS_ERR(vma))
465 continue;
466
467 if (p->step < 0) {
468 if (offset < hole_start + aligned_size)
469 break;
470 offset -= aligned_size;
471 }
472
473 err = i915_vma_pin(vma, 0, 0, offset | flags);
474 if (err) {
475 pr_err("%s(%s) pin (forward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n",
476 __func__, p->name, err, npages, prime, offset);
477 goto err;
478 }
479
480 if (!drm_mm_node_allocated(&vma->node) ||
481 i915_vma_misplaced(vma, 0, 0, offset | flags)) {
482 pr_err("%s(%s) (forward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
483 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
484 offset);
485 err = -EINVAL;
486 goto err;
487 }
488
489 i915_vma_unpin(vma);
490
491 if (p->step > 0) {
492 if (offset + aligned_size > hole_end)
493 break;
494 offset += aligned_size;
495 }
496 }
497
498 offset = p->offset;
499 list_for_each_entry(obj, &objects, st_link) {
500 u64 aligned_size = round_up(obj->base.size,
501 min_alignment);
502
503 vma = i915_vma_instance(obj, vm, NULL);
504 if (IS_ERR(vma))
505 continue;
506
507 if (p->step < 0) {
508 if (offset < hole_start + aligned_size)
509 break;
510 offset -= aligned_size;
511 }
512
513 if (!drm_mm_node_allocated(&vma->node) ||
514 i915_vma_misplaced(vma, 0, 0, offset | flags)) {
515 pr_err("%s(%s) (forward) moved vma.node=%llx + %llx, expected offset %llx\n",
516 __func__, p->name, vma->node.start, vma->node.size,
517 offset);
518 err = -EINVAL;
519 goto err;
520 }
521
522 err = i915_vma_unbind_unlocked(vma);
523 if (err) {
524 pr_err("%s(%s) (forward) unbind of vma.node=%llx + %llx failed with err=%d\n",
525 __func__, p->name, vma->node.start, vma->node.size,
526 err);
527 goto err;
528 }
529
530 if (p->step > 0) {
531 if (offset + aligned_size > hole_end)
532 break;
533 offset += aligned_size;
534 }
535 }
536
537 offset = p->offset;
538 list_for_each_entry_reverse(obj, &objects, st_link) {
539 u64 aligned_size = round_up(obj->base.size,
540 min_alignment);
541
542 vma = i915_vma_instance(obj, vm, NULL);
543 if (IS_ERR(vma))
544 continue;
545
546 if (p->step < 0) {
547 if (offset < hole_start + aligned_size)
548 break;
549 offset -= aligned_size;
550 }
551
552 err = i915_vma_pin(vma, 0, 0, offset | flags);
553 if (err) {
554 pr_err("%s(%s) pin (backward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n",
555 __func__, p->name, err, npages, prime, offset);
556 goto err;
557 }
558
559 if (!drm_mm_node_allocated(&vma->node) ||
560 i915_vma_misplaced(vma, 0, 0, offset | flags)) {
561 pr_err("%s(%s) (backward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
562 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
563 offset);
564 err = -EINVAL;
565 goto err;
566 }
567
568 i915_vma_unpin(vma);
569
570 if (p->step > 0) {
571 if (offset + aligned_size > hole_end)
572 break;
573 offset += aligned_size;
574 }
575 }
576
577 offset = p->offset;
578 list_for_each_entry_reverse(obj, &objects, st_link) {
579 u64 aligned_size = round_up(obj->base.size,
580 min_alignment);
581
582 vma = i915_vma_instance(obj, vm, NULL);
583 if (IS_ERR(vma))
584 continue;
585
586 if (p->step < 0) {
587 if (offset < hole_start + aligned_size)
588 break;
589 offset -= aligned_size;
590 }
591
592 if (!drm_mm_node_allocated(&vma->node) ||
593 i915_vma_misplaced(vma, 0, 0, offset | flags)) {
594 pr_err("%s(%s) (backward) moved vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
595 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
596 offset);
597 err = -EINVAL;
598 goto err;
599 }
600
601 err = i915_vma_unbind_unlocked(vma);
602 if (err) {
603 pr_err("%s(%s) (backward) unbind of vma.node=%llx + %llx failed with err=%d\n",
604 __func__, p->name, vma->node.start, vma->node.size,
605 err);
606 goto err;
607 }
608
609 if (p->step > 0) {
610 if (offset + aligned_size > hole_end)
611 break;
612 offset += aligned_size;
613 }
614 }
615 }
616
617 if (igt_timeout(end_time, "%s timed out (npages=%lu, prime=%lu)\n",
618 __func__, npages, prime)) {
619 err = -EINTR;
620 goto err;
621 }
622 }
623
624 close_object_list(&objects, vm);
625 cleanup_freed_objects(vm->i915);
626 }
627
628 return 0;
629
630 err:
631 close_object_list(&objects, vm);
632 return err;
633 }
634
walk_hole(struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time)635 static int walk_hole(struct i915_address_space *vm,
636 u64 hole_start, u64 hole_end,
637 unsigned long end_time)
638 {
639 const u64 hole_size = hole_end - hole_start;
640 const unsigned long max_pages =
641 min_t(u64, ULONG_MAX - 1, hole_size >> PAGE_SHIFT);
642 unsigned long min_alignment;
643 unsigned long flags;
644 u64 size;
645
646 /* Try binding a single VMA in different positions within the hole */
647
648 flags = PIN_OFFSET_FIXED | PIN_USER;
649 if (i915_is_ggtt(vm))
650 flags |= PIN_GLOBAL;
651
652 min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM);
653
654 for_each_prime_number_from(size, 1, max_pages) {
655 struct drm_i915_gem_object *obj;
656 struct i915_vma *vma;
657 u64 addr;
658 int err = 0;
659
660 obj = fake_dma_object(vm->i915, size << PAGE_SHIFT);
661 if (IS_ERR(obj))
662 break;
663
664 vma = i915_vma_instance(obj, vm, NULL);
665 if (IS_ERR(vma)) {
666 err = PTR_ERR(vma);
667 goto err_put;
668 }
669
670 for (addr = hole_start;
671 addr + obj->base.size < hole_end;
672 addr += round_up(obj->base.size, min_alignment)) {
673 err = i915_vma_pin(vma, 0, 0, addr | flags);
674 if (err) {
675 pr_err("%s bind failed at %llx + %llx [hole %llx- %llx] with err=%d\n",
676 __func__, addr, vma->size,
677 hole_start, hole_end, err);
678 goto err_put;
679 }
680 i915_vma_unpin(vma);
681
682 if (!drm_mm_node_allocated(&vma->node) ||
683 i915_vma_misplaced(vma, 0, 0, addr | flags)) {
684 pr_err("%s incorrect at %llx + %llx\n",
685 __func__, addr, vma->size);
686 err = -EINVAL;
687 goto err_put;
688 }
689
690 err = i915_vma_unbind_unlocked(vma);
691 if (err) {
692 pr_err("%s unbind failed at %llx + %llx with err=%d\n",
693 __func__, addr, vma->size, err);
694 goto err_put;
695 }
696
697 GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
698
699 if (igt_timeout(end_time,
700 "%s timed out at %llx\n",
701 __func__, addr)) {
702 err = -EINTR;
703 goto err_put;
704 }
705 }
706
707 err_put:
708 i915_gem_object_put(obj);
709 if (err)
710 return err;
711
712 cleanup_freed_objects(vm->i915);
713 }
714
715 return 0;
716 }
717
pot_hole(struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time)718 static int pot_hole(struct i915_address_space *vm,
719 u64 hole_start, u64 hole_end,
720 unsigned long end_time)
721 {
722 struct drm_i915_gem_object *obj;
723 struct i915_vma *vma;
724 unsigned int min_alignment;
725 unsigned long flags;
726 unsigned int pot;
727 int err = 0;
728
729 flags = PIN_OFFSET_FIXED | PIN_USER;
730 if (i915_is_ggtt(vm))
731 flags |= PIN_GLOBAL;
732
733 min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM);
734
735 obj = i915_gem_object_create_internal(vm->i915, 2 * I915_GTT_PAGE_SIZE);
736 if (IS_ERR(obj))
737 return PTR_ERR(obj);
738
739 vma = i915_vma_instance(obj, vm, NULL);
740 if (IS_ERR(vma)) {
741 err = PTR_ERR(vma);
742 goto err_obj;
743 }
744
745 /* Insert a pair of pages across every pot boundary within the hole */
746 for (pot = fls64(hole_end - 1) - 1;
747 pot > ilog2(2 * min_alignment);
748 pot--) {
749 u64 step = BIT_ULL(pot);
750 u64 addr;
751
752 for (addr = round_up(hole_start + min_alignment, step) - min_alignment;
753 hole_end > addr && hole_end - addr >= 2 * min_alignment;
754 addr += step) {
755 err = i915_vma_pin(vma, 0, 0, addr | flags);
756 if (err) {
757 pr_err("%s failed to pin object at %llx in hole [%llx - %llx], with err=%d\n",
758 __func__,
759 addr,
760 hole_start, hole_end,
761 err);
762 goto err_obj;
763 }
764
765 if (!drm_mm_node_allocated(&vma->node) ||
766 i915_vma_misplaced(vma, 0, 0, addr | flags)) {
767 pr_err("%s incorrect at %llx + %llx\n",
768 __func__, addr, vma->size);
769 i915_vma_unpin(vma);
770 err = i915_vma_unbind_unlocked(vma);
771 err = -EINVAL;
772 goto err_obj;
773 }
774
775 i915_vma_unpin(vma);
776 err = i915_vma_unbind_unlocked(vma);
777 GEM_BUG_ON(err);
778 }
779
780 if (igt_timeout(end_time,
781 "%s timed out after %d/%d\n",
782 __func__, pot, fls64(hole_end - 1) - 1)) {
783 err = -EINTR;
784 goto err_obj;
785 }
786 }
787
788 err_obj:
789 i915_gem_object_put(obj);
790 return err;
791 }
792
drunk_hole(struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time)793 static int drunk_hole(struct i915_address_space *vm,
794 u64 hole_start, u64 hole_end,
795 unsigned long end_time)
796 {
797 I915_RND_STATE(prng);
798 unsigned int min_alignment;
799 unsigned int size;
800 unsigned long flags;
801
802 flags = PIN_OFFSET_FIXED | PIN_USER;
803 if (i915_is_ggtt(vm))
804 flags |= PIN_GLOBAL;
805
806 min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM);
807
808 /* Keep creating larger objects until one cannot fit into the hole */
809 for (size = 12; (hole_end - hole_start) >> size; size++) {
810 struct drm_i915_gem_object *obj;
811 unsigned int *order, count, n;
812 struct i915_vma *vma;
813 u64 hole_size, aligned_size;
814 int err = -ENODEV;
815
816 aligned_size = max_t(u32, ilog2(min_alignment), size);
817 hole_size = (hole_end - hole_start) >> aligned_size;
818 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32))
819 hole_size = KMALLOC_MAX_SIZE / sizeof(u32);
820 count = hole_size >> 1;
821 if (!count) {
822 pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n",
823 __func__, hole_start, hole_end, size, hole_size);
824 break;
825 }
826
827 do {
828 order = i915_random_order(count, &prng);
829 if (order)
830 break;
831 } while (count >>= 1);
832 if (!count)
833 return -ENOMEM;
834 GEM_BUG_ON(!order);
835
836 /*
837 * Ignore allocation failures (i.e. don't report them as
838 * a test failure) as we are purposefully allocating very
839 * large objects without checking that we have sufficient
840 * memory. We expect to hit -ENOMEM.
841 */
842
843 obj = fake_dma_object(vm->i915, BIT_ULL(size));
844 if (IS_ERR(obj)) {
845 kfree(order);
846 break;
847 }
848
849 vma = i915_vma_instance(obj, vm, NULL);
850 if (IS_ERR(vma)) {
851 err = PTR_ERR(vma);
852 goto err_obj;
853 }
854
855 GEM_BUG_ON(vma->size != BIT_ULL(size));
856
857 for (n = 0; n < count; n++) {
858 u64 addr = hole_start + order[n] * BIT_ULL(aligned_size);
859
860 err = i915_vma_pin(vma, 0, 0, addr | flags);
861 if (err) {
862 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n",
863 __func__,
864 addr, BIT_ULL(size),
865 hole_start, hole_end,
866 err);
867 goto err_obj;
868 }
869
870 if (!drm_mm_node_allocated(&vma->node) ||
871 i915_vma_misplaced(vma, 0, 0, addr | flags)) {
872 pr_err("%s incorrect at %llx + %llx\n",
873 __func__, addr, BIT_ULL(size));
874 i915_vma_unpin(vma);
875 err = i915_vma_unbind_unlocked(vma);
876 err = -EINVAL;
877 goto err_obj;
878 }
879
880 i915_vma_unpin(vma);
881 err = i915_vma_unbind_unlocked(vma);
882 GEM_BUG_ON(err);
883
884 if (igt_timeout(end_time,
885 "%s timed out after %d/%d\n",
886 __func__, n, count)) {
887 err = -EINTR;
888 goto err_obj;
889 }
890 }
891
892 err_obj:
893 i915_gem_object_put(obj);
894 kfree(order);
895 if (err)
896 return err;
897
898 cleanup_freed_objects(vm->i915);
899 }
900
901 return 0;
902 }
903
__shrink_hole(struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time)904 static int __shrink_hole(struct i915_address_space *vm,
905 u64 hole_start, u64 hole_end,
906 unsigned long end_time)
907 {
908 struct drm_i915_gem_object *obj;
909 unsigned long flags = PIN_OFFSET_FIXED | PIN_USER;
910 unsigned int min_alignment;
911 unsigned int order = 12;
912 LIST_HEAD(objects);
913 int err = 0;
914 u64 addr;
915
916 min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM);
917
918 /* Keep creating larger objects until one cannot fit into the hole */
919 for (addr = hole_start; addr < hole_end; ) {
920 struct i915_vma *vma;
921 u64 size = BIT_ULL(order++);
922
923 size = min(size, hole_end - addr);
924 obj = fake_dma_object(vm->i915, size);
925 if (IS_ERR(obj)) {
926 err = PTR_ERR(obj);
927 break;
928 }
929
930 list_add(&obj->st_link, &objects);
931
932 vma = i915_vma_instance(obj, vm, NULL);
933 if (IS_ERR(vma)) {
934 err = PTR_ERR(vma);
935 break;
936 }
937
938 GEM_BUG_ON(vma->size != size);
939
940 err = i915_vma_pin(vma, 0, 0, addr | flags);
941 if (err) {
942 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n",
943 __func__, addr, size, hole_start, hole_end, err);
944 break;
945 }
946
947 if (!drm_mm_node_allocated(&vma->node) ||
948 i915_vma_misplaced(vma, 0, 0, addr | flags)) {
949 pr_err("%s incorrect at %llx + %llx\n",
950 __func__, addr, size);
951 i915_vma_unpin(vma);
952 err = i915_vma_unbind_unlocked(vma);
953 err = -EINVAL;
954 break;
955 }
956
957 i915_vma_unpin(vma);
958 addr += round_up(size, min_alignment);
959
960 /*
961 * Since we are injecting allocation faults at random intervals,
962 * wait for this allocation to complete before we change the
963 * faultinjection.
964 */
965 err = i915_vma_sync(vma);
966 if (err)
967 break;
968
969 if (igt_timeout(end_time,
970 "%s timed out at offset %llx [%llx - %llx]\n",
971 __func__, addr, hole_start, hole_end)) {
972 err = -EINTR;
973 break;
974 }
975 }
976
977 close_object_list(&objects, vm);
978 cleanup_freed_objects(vm->i915);
979 return err;
980 }
981
shrink_hole(struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time)982 static int shrink_hole(struct i915_address_space *vm,
983 u64 hole_start, u64 hole_end,
984 unsigned long end_time)
985 {
986 unsigned long prime;
987 int err;
988
989 vm->fault_attr.probability = 999;
990 atomic_set(&vm->fault_attr.times, -1);
991
992 for_each_prime_number_from(prime, 0, ULONG_MAX - 1) {
993 vm->fault_attr.interval = prime;
994 err = __shrink_hole(vm, hole_start, hole_end, end_time);
995 if (err)
996 break;
997 }
998
999 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
1000
1001 return err;
1002 }
1003
shrink_boom(struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time)1004 static int shrink_boom(struct i915_address_space *vm,
1005 u64 hole_start, u64 hole_end,
1006 unsigned long end_time)
1007 {
1008 unsigned int sizes[] = { SZ_2M, SZ_1G };
1009 struct drm_i915_gem_object *purge;
1010 struct drm_i915_gem_object *explode;
1011 int err;
1012 int i;
1013
1014 /*
1015 * Catch the case which shrink_hole seems to miss. The setup here
1016 * requires invoking the shrinker as we do the alloc_pt/alloc_pd, while
1017 * ensuring that all vma associated with the respective pd/pdp are
1018 * unpinned at the time.
1019 */
1020
1021 for (i = 0; i < ARRAY_SIZE(sizes); ++i) {
1022 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
1023 unsigned int size = sizes[i];
1024 struct i915_vma *vma;
1025
1026 purge = fake_dma_object(vm->i915, size);
1027 if (IS_ERR(purge))
1028 return PTR_ERR(purge);
1029
1030 vma = i915_vma_instance(purge, vm, NULL);
1031 if (IS_ERR(vma)) {
1032 err = PTR_ERR(vma);
1033 goto err_purge;
1034 }
1035
1036 err = i915_vma_pin(vma, 0, 0, flags);
1037 if (err)
1038 goto err_purge;
1039
1040 /* Should now be ripe for purging */
1041 i915_vma_unpin(vma);
1042
1043 explode = fake_dma_object(vm->i915, size);
1044 if (IS_ERR(explode)) {
1045 err = PTR_ERR(explode);
1046 goto err_purge;
1047 }
1048
1049 vm->fault_attr.probability = 100;
1050 vm->fault_attr.interval = 1;
1051 atomic_set(&vm->fault_attr.times, -1);
1052
1053 vma = i915_vma_instance(explode, vm, NULL);
1054 if (IS_ERR(vma)) {
1055 err = PTR_ERR(vma);
1056 goto err_explode;
1057 }
1058
1059 err = i915_vma_pin(vma, 0, 0, flags | size);
1060 if (err)
1061 goto err_explode;
1062
1063 i915_vma_unpin(vma);
1064
1065 i915_gem_object_put(purge);
1066 i915_gem_object_put(explode);
1067
1068 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
1069 cleanup_freed_objects(vm->i915);
1070 }
1071
1072 return 0;
1073
1074 err_explode:
1075 i915_gem_object_put(explode);
1076 err_purge:
1077 i915_gem_object_put(purge);
1078 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
1079 return err;
1080 }
1081
misaligned_case(struct i915_address_space * vm,struct intel_memory_region * mr,u64 addr,u64 size,unsigned long flags)1082 static int misaligned_case(struct i915_address_space *vm, struct intel_memory_region *mr,
1083 u64 addr, u64 size, unsigned long flags)
1084 {
1085 struct drm_i915_gem_object *obj;
1086 struct i915_vma *vma;
1087 int err = 0;
1088 u64 expected_vma_size, expected_node_size;
1089 bool is_stolen = mr->type == INTEL_MEMORY_STOLEN_SYSTEM ||
1090 mr->type == INTEL_MEMORY_STOLEN_LOCAL;
1091
1092 obj = i915_gem_object_create_region(mr, size, 0, I915_BO_ALLOC_GPU_ONLY);
1093 if (IS_ERR(obj)) {
1094 /* if iGVT-g or DMAR is active, stolen mem will be uninitialized */
1095 if (PTR_ERR(obj) == -ENODEV && is_stolen)
1096 return 0;
1097 return PTR_ERR(obj);
1098 }
1099
1100 vma = i915_vma_instance(obj, vm, NULL);
1101 if (IS_ERR(vma)) {
1102 err = PTR_ERR(vma);
1103 goto err_put;
1104 }
1105
1106 err = i915_vma_pin(vma, 0, 0, addr | flags);
1107 if (err)
1108 goto err_put;
1109 i915_vma_unpin(vma);
1110
1111 if (!drm_mm_node_allocated(&vma->node)) {
1112 err = -EINVAL;
1113 goto err_put;
1114 }
1115
1116 if (i915_vma_misplaced(vma, 0, 0, addr | flags)) {
1117 err = -EINVAL;
1118 goto err_put;
1119 }
1120
1121 /* make sure page_sizes_gtt has been populated before use */
1122 if (i915_is_ggtt(vm) && intel_vm_no_concurrent_access_wa(vm->i915))
1123 i915_vma_wait_for_bind(vma);
1124
1125 expected_vma_size = round_up(size, 1 << (ffs(vma->resource->page_sizes_gtt) - 1));
1126 expected_node_size = expected_vma_size;
1127
1128 if (HAS_64K_PAGES(vm->i915) && i915_gem_object_is_lmem(obj)) {
1129 expected_vma_size = round_up(size, I915_GTT_PAGE_SIZE_64K);
1130 expected_node_size = round_up(size, I915_GTT_PAGE_SIZE_64K);
1131 }
1132
1133 if (vma->size != expected_vma_size || vma->node.size != expected_node_size) {
1134 err = i915_vma_unbind_unlocked(vma);
1135 err = -EBADSLT;
1136 goto err_put;
1137 }
1138
1139 err = i915_vma_unbind_unlocked(vma);
1140 if (err)
1141 goto err_put;
1142
1143 GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
1144
1145 err_put:
1146 i915_gem_object_put(obj);
1147 cleanup_freed_objects(vm->i915);
1148 return err;
1149 }
1150
misaligned_pin(struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time)1151 static int misaligned_pin(struct i915_address_space *vm,
1152 u64 hole_start, u64 hole_end,
1153 unsigned long end_time)
1154 {
1155 struct intel_memory_region *mr;
1156 enum intel_region_id id;
1157 unsigned long flags = PIN_OFFSET_FIXED | PIN_USER;
1158 int err = 0;
1159 u64 hole_size = hole_end - hole_start;
1160
1161 if (i915_is_ggtt(vm))
1162 flags |= PIN_GLOBAL;
1163
1164 for_each_memory_region(mr, vm->i915, id) {
1165 u64 min_alignment = i915_vm_min_alignment(vm, mr->type);
1166 u64 size = min_alignment;
1167 u64 addr = round_down(hole_start + (hole_size / 2), min_alignment);
1168
1169 /* avoid -ENOSPC on very small hole setups */
1170 if (hole_size < 3 * min_alignment)
1171 continue;
1172
1173 /* we can't test < 4k alignment due to flags being encoded in lower bits */
1174 if (min_alignment != I915_GTT_PAGE_SIZE_4K) {
1175 err = misaligned_case(vm, mr, addr + (min_alignment / 2), size, flags);
1176 /* misaligned should error with -EINVAL*/
1177 if (!err)
1178 err = -EBADSLT;
1179 if (err != -EINVAL)
1180 return err;
1181 }
1182
1183 /* test for vma->size expansion to min page size */
1184 err = misaligned_case(vm, mr, addr, PAGE_SIZE, flags);
1185 if (err)
1186 return err;
1187
1188 /* test for intermediate size not expanding vma->size for large alignments */
1189 err = misaligned_case(vm, mr, addr, size / 2, flags);
1190 if (err)
1191 return err;
1192 }
1193
1194 return 0;
1195 }
1196
exercise_ppgtt(struct drm_i915_private * dev_priv,int (* func)(struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time))1197 static int exercise_ppgtt(struct drm_i915_private *dev_priv,
1198 int (*func)(struct i915_address_space *vm,
1199 u64 hole_start, u64 hole_end,
1200 unsigned long end_time))
1201 {
1202 struct i915_ppgtt *ppgtt;
1203 IGT_TIMEOUT(end_time);
1204 struct file *file;
1205 int err;
1206
1207 if (!HAS_FULL_PPGTT(dev_priv))
1208 return 0;
1209
1210 file = mock_file(dev_priv);
1211 if (IS_ERR(file))
1212 return PTR_ERR(file);
1213
1214 ppgtt = i915_ppgtt_create(to_gt(dev_priv), 0);
1215 if (IS_ERR(ppgtt)) {
1216 err = PTR_ERR(ppgtt);
1217 goto out_free;
1218 }
1219 GEM_BUG_ON(offset_in_page(ppgtt->vm.total));
1220 assert_vm_alive(&ppgtt->vm);
1221
1222 err = func(&ppgtt->vm, 0, ppgtt->vm.total, end_time);
1223
1224 i915_vm_put(&ppgtt->vm);
1225
1226 out_free:
1227 fput(file);
1228 return err;
1229 }
1230
igt_ppgtt_fill(void * arg)1231 static int igt_ppgtt_fill(void *arg)
1232 {
1233 return exercise_ppgtt(arg, fill_hole);
1234 }
1235
igt_ppgtt_walk(void * arg)1236 static int igt_ppgtt_walk(void *arg)
1237 {
1238 return exercise_ppgtt(arg, walk_hole);
1239 }
1240
igt_ppgtt_pot(void * arg)1241 static int igt_ppgtt_pot(void *arg)
1242 {
1243 return exercise_ppgtt(arg, pot_hole);
1244 }
1245
igt_ppgtt_drunk(void * arg)1246 static int igt_ppgtt_drunk(void *arg)
1247 {
1248 return exercise_ppgtt(arg, drunk_hole);
1249 }
1250
igt_ppgtt_lowlevel(void * arg)1251 static int igt_ppgtt_lowlevel(void *arg)
1252 {
1253 return exercise_ppgtt(arg, lowlevel_hole);
1254 }
1255
igt_ppgtt_shrink(void * arg)1256 static int igt_ppgtt_shrink(void *arg)
1257 {
1258 return exercise_ppgtt(arg, shrink_hole);
1259 }
1260
igt_ppgtt_shrink_boom(void * arg)1261 static int igt_ppgtt_shrink_boom(void *arg)
1262 {
1263 return exercise_ppgtt(arg, shrink_boom);
1264 }
1265
igt_ppgtt_misaligned_pin(void * arg)1266 static int igt_ppgtt_misaligned_pin(void *arg)
1267 {
1268 return exercise_ppgtt(arg, misaligned_pin);
1269 }
1270
sort_holes(void * priv,const struct list_head * A,const struct list_head * B)1271 static int sort_holes(void *priv, const struct list_head *A,
1272 const struct list_head *B)
1273 {
1274 struct drm_mm_node *a = list_entry(A, typeof(*a), hole_stack);
1275 struct drm_mm_node *b = list_entry(B, typeof(*b), hole_stack);
1276
1277 if (a->start < b->start)
1278 return -1;
1279 else
1280 return 1;
1281 }
1282
exercise_ggtt(struct drm_i915_private * i915,int (* func)(struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time))1283 static int exercise_ggtt(struct drm_i915_private *i915,
1284 int (*func)(struct i915_address_space *vm,
1285 u64 hole_start, u64 hole_end,
1286 unsigned long end_time))
1287 {
1288 struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
1289 u64 hole_start, hole_end, last = 0;
1290 struct drm_mm_node *node;
1291 IGT_TIMEOUT(end_time);
1292 int err = 0;
1293
1294 restart:
1295 list_sort(NULL, &ggtt->vm.mm.hole_stack, sort_holes);
1296 drm_mm_for_each_hole(node, &ggtt->vm.mm, hole_start, hole_end) {
1297 if (hole_start < last)
1298 continue;
1299
1300 if (ggtt->vm.mm.color_adjust)
1301 ggtt->vm.mm.color_adjust(node, 0,
1302 &hole_start, &hole_end);
1303 if (hole_start >= hole_end)
1304 continue;
1305
1306 err = func(&ggtt->vm, hole_start, hole_end, end_time);
1307 if (err)
1308 break;
1309
1310 /* As we have manipulated the drm_mm, the list may be corrupt */
1311 last = hole_end;
1312 goto restart;
1313 }
1314
1315 return err;
1316 }
1317
igt_ggtt_fill(void * arg)1318 static int igt_ggtt_fill(void *arg)
1319 {
1320 return exercise_ggtt(arg, fill_hole);
1321 }
1322
igt_ggtt_walk(void * arg)1323 static int igt_ggtt_walk(void *arg)
1324 {
1325 return exercise_ggtt(arg, walk_hole);
1326 }
1327
igt_ggtt_pot(void * arg)1328 static int igt_ggtt_pot(void *arg)
1329 {
1330 return exercise_ggtt(arg, pot_hole);
1331 }
1332
igt_ggtt_drunk(void * arg)1333 static int igt_ggtt_drunk(void *arg)
1334 {
1335 return exercise_ggtt(arg, drunk_hole);
1336 }
1337
igt_ggtt_lowlevel(void * arg)1338 static int igt_ggtt_lowlevel(void *arg)
1339 {
1340 return exercise_ggtt(arg, lowlevel_hole);
1341 }
1342
igt_ggtt_misaligned_pin(void * arg)1343 static int igt_ggtt_misaligned_pin(void *arg)
1344 {
1345 return exercise_ggtt(arg, misaligned_pin);
1346 }
1347
igt_ggtt_page(void * arg)1348 static int igt_ggtt_page(void *arg)
1349 {
1350 const unsigned int count = PAGE_SIZE/sizeof(u32);
1351 I915_RND_STATE(prng);
1352 struct drm_i915_private *i915 = arg;
1353 struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
1354 struct drm_i915_gem_object *obj;
1355 intel_wakeref_t wakeref;
1356 struct drm_mm_node tmp;
1357 unsigned int *order, n;
1358 int err;
1359
1360 if (!i915_ggtt_has_aperture(ggtt))
1361 return 0;
1362
1363 obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
1364 if (IS_ERR(obj))
1365 return PTR_ERR(obj);
1366
1367 err = i915_gem_object_pin_pages_unlocked(obj);
1368 if (err)
1369 goto out_free;
1370
1371 memset(&tmp, 0, sizeof(tmp));
1372 mutex_lock(&ggtt->vm.mutex);
1373 err = drm_mm_insert_node_in_range(&ggtt->vm.mm, &tmp,
1374 count * PAGE_SIZE, 0,
1375 I915_COLOR_UNEVICTABLE,
1376 0, ggtt->mappable_end,
1377 DRM_MM_INSERT_LOW);
1378 mutex_unlock(&ggtt->vm.mutex);
1379 if (err)
1380 goto out_unpin;
1381
1382 wakeref = intel_runtime_pm_get(&i915->runtime_pm);
1383
1384 for (n = 0; n < count; n++) {
1385 u64 offset = tmp.start + n * PAGE_SIZE;
1386
1387 ggtt->vm.insert_page(&ggtt->vm,
1388 i915_gem_object_get_dma_address(obj, 0),
1389 offset,
1390 i915_gem_get_pat_index(i915,
1391 I915_CACHE_NONE),
1392 0);
1393 }
1394
1395 order = i915_random_order(count, &prng);
1396 if (!order) {
1397 err = -ENOMEM;
1398 goto out_remove;
1399 }
1400
1401 for (n = 0; n < count; n++) {
1402 u64 offset = tmp.start + order[n] * PAGE_SIZE;
1403 u32 __iomem *vaddr;
1404
1405 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset);
1406 iowrite32(n, vaddr + n);
1407 io_mapping_unmap_atomic(vaddr);
1408 }
1409 intel_gt_flush_ggtt_writes(ggtt->vm.gt);
1410
1411 i915_random_reorder(order, count, &prng);
1412 for (n = 0; n < count; n++) {
1413 u64 offset = tmp.start + order[n] * PAGE_SIZE;
1414 u32 __iomem *vaddr;
1415 u32 val;
1416
1417 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset);
1418 val = ioread32(vaddr + n);
1419 io_mapping_unmap_atomic(vaddr);
1420
1421 if (val != n) {
1422 pr_err("insert page failed: found %d, expected %d\n",
1423 val, n);
1424 err = -EINVAL;
1425 break;
1426 }
1427 }
1428
1429 kfree(order);
1430 out_remove:
1431 ggtt->vm.clear_range(&ggtt->vm, tmp.start, tmp.size);
1432 intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1433 mutex_lock(&ggtt->vm.mutex);
1434 drm_mm_remove_node(&tmp);
1435 mutex_unlock(&ggtt->vm.mutex);
1436 out_unpin:
1437 i915_gem_object_unpin_pages(obj);
1438 out_free:
1439 i915_gem_object_put(obj);
1440 return err;
1441 }
1442
track_vma_bind(struct i915_vma * vma)1443 static void track_vma_bind(struct i915_vma *vma)
1444 {
1445 struct drm_i915_gem_object *obj = vma->obj;
1446
1447 __i915_gem_object_pin_pages(obj);
1448
1449 GEM_BUG_ON(atomic_read(&vma->pages_count));
1450 atomic_set(&vma->pages_count, I915_VMA_PAGES_ACTIVE);
1451 __i915_gem_object_pin_pages(obj);
1452 vma->pages = obj->mm.pages;
1453 vma->resource->bi.pages = vma->pages;
1454
1455 mutex_lock(&vma->vm->mutex);
1456 list_move_tail(&vma->vm_link, &vma->vm->bound_list);
1457 mutex_unlock(&vma->vm->mutex);
1458 }
1459
exercise_mock(struct drm_i915_private * i915,int (* func)(struct i915_address_space * vm,u64 hole_start,u64 hole_end,unsigned long end_time))1460 static int exercise_mock(struct drm_i915_private *i915,
1461 int (*func)(struct i915_address_space *vm,
1462 u64 hole_start, u64 hole_end,
1463 unsigned long end_time))
1464 {
1465 const u64 limit = totalram_pages() << PAGE_SHIFT;
1466 struct i915_address_space *vm;
1467 struct i915_gem_context *ctx;
1468 IGT_TIMEOUT(end_time);
1469 int err;
1470
1471 ctx = mock_context(i915, "mock");
1472 if (!ctx)
1473 return -ENOMEM;
1474
1475 vm = i915_gem_context_get_eb_vm(ctx);
1476 err = func(vm, 0, min(vm->total, limit), end_time);
1477 i915_vm_put(vm);
1478
1479 mock_context_close(ctx);
1480 return err;
1481 }
1482
igt_mock_fill(void * arg)1483 static int igt_mock_fill(void *arg)
1484 {
1485 struct i915_ggtt *ggtt = arg;
1486
1487 return exercise_mock(ggtt->vm.i915, fill_hole);
1488 }
1489
igt_mock_walk(void * arg)1490 static int igt_mock_walk(void *arg)
1491 {
1492 struct i915_ggtt *ggtt = arg;
1493
1494 return exercise_mock(ggtt->vm.i915, walk_hole);
1495 }
1496
igt_mock_pot(void * arg)1497 static int igt_mock_pot(void *arg)
1498 {
1499 struct i915_ggtt *ggtt = arg;
1500
1501 return exercise_mock(ggtt->vm.i915, pot_hole);
1502 }
1503
igt_mock_drunk(void * arg)1504 static int igt_mock_drunk(void *arg)
1505 {
1506 struct i915_ggtt *ggtt = arg;
1507
1508 return exercise_mock(ggtt->vm.i915, drunk_hole);
1509 }
1510
reserve_gtt_with_resource(struct i915_vma * vma,u64 offset)1511 static int reserve_gtt_with_resource(struct i915_vma *vma, u64 offset)
1512 {
1513 struct i915_address_space *vm = vma->vm;
1514 struct i915_vma_resource *vma_res;
1515 struct drm_i915_gem_object *obj = vma->obj;
1516 int err;
1517
1518 vma_res = i915_vma_resource_alloc();
1519 if (IS_ERR(vma_res))
1520 return PTR_ERR(vma_res);
1521
1522 mutex_lock(&vm->mutex);
1523 err = i915_gem_gtt_reserve(vm, NULL, &vma->node, obj->base.size,
1524 offset,
1525 obj->pat_index,
1526 0);
1527 if (!err) {
1528 i915_vma_resource_init_from_vma(vma_res, vma);
1529 vma->resource = vma_res;
1530 } else {
1531 kfree(vma_res);
1532 }
1533 mutex_unlock(&vm->mutex);
1534
1535 return err;
1536 }
1537
igt_gtt_reserve(void * arg)1538 static int igt_gtt_reserve(void *arg)
1539 {
1540 struct i915_ggtt *ggtt = arg;
1541 struct drm_i915_gem_object *obj, *on;
1542 I915_RND_STATE(prng);
1543 LIST_HEAD(objects);
1544 u64 total;
1545 int err = -ENODEV;
1546
1547 /*
1548 * i915_gem_gtt_reserve() tries to reserve the precise range
1549 * for the node, and evicts if it has to. So our test checks that
1550 * it can give us the requested space and prevent overlaps.
1551 */
1552
1553 /* Start by filling the GGTT */
1554 for (total = 0;
1555 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1556 total += 2 * I915_GTT_PAGE_SIZE) {
1557 struct i915_vma *vma;
1558
1559 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1560 2 * PAGE_SIZE);
1561 if (IS_ERR(obj)) {
1562 err = PTR_ERR(obj);
1563 goto out;
1564 }
1565
1566 err = i915_gem_object_pin_pages_unlocked(obj);
1567 if (err) {
1568 i915_gem_object_put(obj);
1569 goto out;
1570 }
1571
1572 list_add(&obj->st_link, &objects);
1573 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1574 if (IS_ERR(vma)) {
1575 err = PTR_ERR(vma);
1576 goto out;
1577 }
1578
1579 err = reserve_gtt_with_resource(vma, total);
1580 if (err) {
1581 pr_err("i915_gem_gtt_reserve (pass 1) failed at %llu/%llu with err=%d\n",
1582 total, ggtt->vm.total, err);
1583 goto out;
1584 }
1585 track_vma_bind(vma);
1586
1587 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1588 if (vma->node.start != total ||
1589 vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1590 pr_err("i915_gem_gtt_reserve (pass 1) placement failed, found (%llx + %llx), expected (%llx + %llx)\n",
1591 vma->node.start, vma->node.size,
1592 total, 2*I915_GTT_PAGE_SIZE);
1593 err = -EINVAL;
1594 goto out;
1595 }
1596 }
1597
1598 /* Now we start forcing evictions */
1599 for (total = I915_GTT_PAGE_SIZE;
1600 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1601 total += 2 * I915_GTT_PAGE_SIZE) {
1602 struct i915_vma *vma;
1603
1604 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1605 2 * PAGE_SIZE);
1606 if (IS_ERR(obj)) {
1607 err = PTR_ERR(obj);
1608 goto out;
1609 }
1610
1611 err = i915_gem_object_pin_pages_unlocked(obj);
1612 if (err) {
1613 i915_gem_object_put(obj);
1614 goto out;
1615 }
1616
1617 list_add(&obj->st_link, &objects);
1618
1619 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1620 if (IS_ERR(vma)) {
1621 err = PTR_ERR(vma);
1622 goto out;
1623 }
1624
1625 err = reserve_gtt_with_resource(vma, total);
1626 if (err) {
1627 pr_err("i915_gem_gtt_reserve (pass 2) failed at %llu/%llu with err=%d\n",
1628 total, ggtt->vm.total, err);
1629 goto out;
1630 }
1631 track_vma_bind(vma);
1632
1633 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1634 if (vma->node.start != total ||
1635 vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1636 pr_err("i915_gem_gtt_reserve (pass 2) placement failed, found (%llx + %llx), expected (%llx + %llx)\n",
1637 vma->node.start, vma->node.size,
1638 total, 2*I915_GTT_PAGE_SIZE);
1639 err = -EINVAL;
1640 goto out;
1641 }
1642 }
1643
1644 /* And then try at random */
1645 list_for_each_entry_safe(obj, on, &objects, st_link) {
1646 struct i915_vma *vma;
1647 u64 offset;
1648
1649 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1650 if (IS_ERR(vma)) {
1651 err = PTR_ERR(vma);
1652 goto out;
1653 }
1654
1655 err = i915_vma_unbind_unlocked(vma);
1656 if (err) {
1657 pr_err("i915_vma_unbind failed with err=%d!\n", err);
1658 goto out;
1659 }
1660
1661 offset = igt_random_offset(&prng,
1662 0, ggtt->vm.total,
1663 2 * I915_GTT_PAGE_SIZE,
1664 I915_GTT_MIN_ALIGNMENT);
1665
1666 err = reserve_gtt_with_resource(vma, offset);
1667 if (err) {
1668 pr_err("i915_gem_gtt_reserve (pass 3) failed at %llu/%llu with err=%d\n",
1669 total, ggtt->vm.total, err);
1670 goto out;
1671 }
1672 track_vma_bind(vma);
1673
1674 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1675 if (vma->node.start != offset ||
1676 vma->node.size != 2*I915_GTT_PAGE_SIZE) {
1677 pr_err("i915_gem_gtt_reserve (pass 3) placement failed, found (%llx + %llx), expected (%llx + %llx)\n",
1678 vma->node.start, vma->node.size,
1679 offset, 2*I915_GTT_PAGE_SIZE);
1680 err = -EINVAL;
1681 goto out;
1682 }
1683 }
1684
1685 out:
1686 list_for_each_entry_safe(obj, on, &objects, st_link) {
1687 i915_gem_object_unpin_pages(obj);
1688 i915_gem_object_put(obj);
1689 }
1690 return err;
1691 }
1692
insert_gtt_with_resource(struct i915_vma * vma)1693 static int insert_gtt_with_resource(struct i915_vma *vma)
1694 {
1695 struct i915_address_space *vm = vma->vm;
1696 struct i915_vma_resource *vma_res;
1697 struct drm_i915_gem_object *obj = vma->obj;
1698 int err;
1699
1700 vma_res = i915_vma_resource_alloc();
1701 if (IS_ERR(vma_res))
1702 return PTR_ERR(vma_res);
1703
1704 mutex_lock(&vm->mutex);
1705 err = i915_gem_gtt_insert(vm, NULL, &vma->node, obj->base.size, 0,
1706 obj->pat_index, 0, vm->total, 0);
1707 if (!err) {
1708 i915_vma_resource_init_from_vma(vma_res, vma);
1709 vma->resource = vma_res;
1710 } else {
1711 kfree(vma_res);
1712 }
1713 mutex_unlock(&vm->mutex);
1714
1715 return err;
1716 }
1717
igt_gtt_insert(void * arg)1718 static int igt_gtt_insert(void *arg)
1719 {
1720 struct i915_ggtt *ggtt = arg;
1721 struct drm_i915_gem_object *obj, *on;
1722 struct drm_mm_node tmp = {};
1723 const struct invalid_insert {
1724 u64 size;
1725 u64 alignment;
1726 u64 start, end;
1727 } invalid_insert[] = {
1728 {
1729 ggtt->vm.total + I915_GTT_PAGE_SIZE, 0,
1730 0, ggtt->vm.total,
1731 },
1732 {
1733 2*I915_GTT_PAGE_SIZE, 0,
1734 0, I915_GTT_PAGE_SIZE,
1735 },
1736 {
1737 -(u64)I915_GTT_PAGE_SIZE, 0,
1738 0, 4*I915_GTT_PAGE_SIZE,
1739 },
1740 {
1741 -(u64)2*I915_GTT_PAGE_SIZE, 2*I915_GTT_PAGE_SIZE,
1742 0, 4*I915_GTT_PAGE_SIZE,
1743 },
1744 {
1745 I915_GTT_PAGE_SIZE, I915_GTT_MIN_ALIGNMENT << 1,
1746 I915_GTT_MIN_ALIGNMENT, I915_GTT_MIN_ALIGNMENT << 1,
1747 },
1748 {}
1749 }, *ii;
1750 LIST_HEAD(objects);
1751 u64 total;
1752 int err = -ENODEV;
1753
1754 /*
1755 * i915_gem_gtt_insert() tries to allocate some free space in the GTT
1756 * to the node, evicting if required.
1757 */
1758
1759 /* Check a couple of obviously invalid requests */
1760 for (ii = invalid_insert; ii->size; ii++) {
1761 mutex_lock(&ggtt->vm.mutex);
1762 err = i915_gem_gtt_insert(&ggtt->vm, NULL, &tmp,
1763 ii->size, ii->alignment,
1764 I915_COLOR_UNEVICTABLE,
1765 ii->start, ii->end,
1766 0);
1767 mutex_unlock(&ggtt->vm.mutex);
1768 if (err != -ENOSPC) {
1769 pr_err("Invalid i915_gem_gtt_insert(.size=%llx, .alignment=%llx, .start=%llx, .end=%llx) succeeded (err=%d)\n",
1770 ii->size, ii->alignment, ii->start, ii->end,
1771 err);
1772 return -EINVAL;
1773 }
1774 }
1775
1776 /* Start by filling the GGTT */
1777 for (total = 0;
1778 total + I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1779 total += I915_GTT_PAGE_SIZE) {
1780 struct i915_vma *vma;
1781
1782 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1783 I915_GTT_PAGE_SIZE);
1784 if (IS_ERR(obj)) {
1785 err = PTR_ERR(obj);
1786 goto out;
1787 }
1788
1789 err = i915_gem_object_pin_pages_unlocked(obj);
1790 if (err) {
1791 i915_gem_object_put(obj);
1792 goto out;
1793 }
1794
1795 list_add(&obj->st_link, &objects);
1796
1797 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1798 if (IS_ERR(vma)) {
1799 err = PTR_ERR(vma);
1800 goto out;
1801 }
1802
1803 err = insert_gtt_with_resource(vma);
1804 if (err == -ENOSPC) {
1805 /* maxed out the GGTT space */
1806 i915_gem_object_put(obj);
1807 break;
1808 }
1809 if (err) {
1810 pr_err("i915_gem_gtt_insert (pass 1) failed at %llu/%llu with err=%d\n",
1811 total, ggtt->vm.total, err);
1812 goto out;
1813 }
1814 track_vma_bind(vma);
1815 __i915_vma_pin(vma);
1816
1817 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1818 }
1819
1820 list_for_each_entry(obj, &objects, st_link) {
1821 struct i915_vma *vma;
1822
1823 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1824 if (IS_ERR(vma)) {
1825 err = PTR_ERR(vma);
1826 goto out;
1827 }
1828
1829 if (!drm_mm_node_allocated(&vma->node)) {
1830 pr_err("VMA was unexpectedly evicted!\n");
1831 err = -EINVAL;
1832 goto out;
1833 }
1834
1835 __i915_vma_unpin(vma);
1836 }
1837
1838 /* If we then reinsert, we should find the same hole */
1839 list_for_each_entry_safe(obj, on, &objects, st_link) {
1840 struct i915_vma *vma;
1841 u64 offset;
1842
1843 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1844 if (IS_ERR(vma)) {
1845 err = PTR_ERR(vma);
1846 goto out;
1847 }
1848
1849 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1850 offset = vma->node.start;
1851
1852 err = i915_vma_unbind_unlocked(vma);
1853 if (err) {
1854 pr_err("i915_vma_unbind failed with err=%d!\n", err);
1855 goto out;
1856 }
1857
1858 err = insert_gtt_with_resource(vma);
1859 if (err) {
1860 pr_err("i915_gem_gtt_insert (pass 2) failed at %llu/%llu with err=%d\n",
1861 total, ggtt->vm.total, err);
1862 goto out;
1863 }
1864 track_vma_bind(vma);
1865
1866 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1867 if (vma->node.start != offset) {
1868 pr_err("i915_gem_gtt_insert did not return node to its previous location (the only hole), expected address %llx, found %llx\n",
1869 offset, vma->node.start);
1870 err = -EINVAL;
1871 goto out;
1872 }
1873 }
1874
1875 /* And then force evictions */
1876 for (total = 0;
1877 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1878 total += 2 * I915_GTT_PAGE_SIZE) {
1879 struct i915_vma *vma;
1880
1881 obj = i915_gem_object_create_internal(ggtt->vm.i915,
1882 2 * I915_GTT_PAGE_SIZE);
1883 if (IS_ERR(obj)) {
1884 err = PTR_ERR(obj);
1885 goto out;
1886 }
1887
1888 err = i915_gem_object_pin_pages_unlocked(obj);
1889 if (err) {
1890 i915_gem_object_put(obj);
1891 goto out;
1892 }
1893
1894 list_add(&obj->st_link, &objects);
1895
1896 vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1897 if (IS_ERR(vma)) {
1898 err = PTR_ERR(vma);
1899 goto out;
1900 }
1901
1902 err = insert_gtt_with_resource(vma);
1903 if (err) {
1904 pr_err("i915_gem_gtt_insert (pass 3) failed at %llu/%llu with err=%d\n",
1905 total, ggtt->vm.total, err);
1906 goto out;
1907 }
1908 track_vma_bind(vma);
1909
1910 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
1911 }
1912
1913 out:
1914 list_for_each_entry_safe(obj, on, &objects, st_link) {
1915 i915_gem_object_unpin_pages(obj);
1916 i915_gem_object_put(obj);
1917 }
1918 return err;
1919 }
1920
i915_gem_gtt_mock_selftests(void)1921 int i915_gem_gtt_mock_selftests(void)
1922 {
1923 static const struct i915_subtest tests[] = {
1924 SUBTEST(igt_mock_drunk),
1925 SUBTEST(igt_mock_walk),
1926 SUBTEST(igt_mock_pot),
1927 SUBTEST(igt_mock_fill),
1928 SUBTEST(igt_gtt_reserve),
1929 SUBTEST(igt_gtt_insert),
1930 };
1931 struct drm_i915_private *i915;
1932 struct intel_gt *gt;
1933 int err;
1934
1935 i915 = mock_gem_device();
1936 if (!i915)
1937 return -ENOMEM;
1938
1939 /* allocate the ggtt */
1940 err = intel_gt_assign_ggtt(to_gt(i915));
1941 if (err)
1942 goto out_put;
1943
1944 gt = to_gt(i915);
1945
1946 mock_init_ggtt(gt);
1947
1948 err = i915_subtests(tests, gt->ggtt);
1949
1950 mock_device_flush(i915);
1951 i915_gem_drain_freed_objects(i915);
1952 mock_fini_ggtt(gt->ggtt);
1953
1954 out_put:
1955 mock_destroy_device(i915);
1956 return err;
1957 }
1958
i915_gem_gtt_live_selftests(struct drm_i915_private * i915)1959 int i915_gem_gtt_live_selftests(struct drm_i915_private *i915)
1960 {
1961 static const struct i915_subtest tests[] = {
1962 SUBTEST(igt_ppgtt_alloc),
1963 SUBTEST(igt_ppgtt_lowlevel),
1964 SUBTEST(igt_ppgtt_drunk),
1965 SUBTEST(igt_ppgtt_walk),
1966 SUBTEST(igt_ppgtt_pot),
1967 SUBTEST(igt_ppgtt_fill),
1968 SUBTEST(igt_ppgtt_shrink),
1969 SUBTEST(igt_ppgtt_shrink_boom),
1970 SUBTEST(igt_ppgtt_misaligned_pin),
1971 SUBTEST(igt_ggtt_lowlevel),
1972 SUBTEST(igt_ggtt_drunk),
1973 SUBTEST(igt_ggtt_walk),
1974 SUBTEST(igt_ggtt_pot),
1975 SUBTEST(igt_ggtt_fill),
1976 SUBTEST(igt_ggtt_page),
1977 SUBTEST(igt_ggtt_misaligned_pin),
1978 };
1979
1980 GEM_BUG_ON(offset_in_page(to_gt(i915)->ggtt->vm.total));
1981
1982 return i915_live_subtests(tests, i915);
1983 }
1984