1 /*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2017 Intel Corporation
5 */
6
7 #include <linux/prime_numbers.h>
8 #include <linux/string_helpers.h>
9
10 #include "gem/i915_gem_internal.h"
11 #include "gem/i915_gem_pm.h"
12 #include "gt/intel_engine_pm.h"
13 #include "gt/intel_engine_regs.h"
14 #include "gt/intel_gt.h"
15 #include "gt/intel_gt_requests.h"
16 #include "gt/intel_reset.h"
17 #include "i915_selftest.h"
18
19 #include "gem/selftests/igt_gem_utils.h"
20 #include "selftests/i915_random.h"
21 #include "selftests/igt_flush_test.h"
22 #include "selftests/igt_live_test.h"
23 #include "selftests/igt_reset.h"
24 #include "selftests/igt_spinner.h"
25 #include "selftests/mock_drm.h"
26 #include "selftests/mock_gem_device.h"
27
28 #include "huge_gem_object.h"
29 #include "igt_gem_utils.h"
30
31 #define DW_PER_PAGE (PAGE_SIZE / sizeof(u32))
32
live_nop_switch(void * arg)33 static int live_nop_switch(void *arg)
34 {
35 const unsigned int nctx = 1024;
36 struct drm_i915_private *i915 = arg;
37 struct intel_engine_cs *engine;
38 struct i915_gem_context **ctx;
39 struct igt_live_test t;
40 struct file *file;
41 unsigned long n;
42 int err = -ENODEV;
43
44 /*
45 * Create as many contexts as we can feasibly get away with
46 * and check we can switch between them rapidly.
47 *
48 * Serves as very simple stress test for submission and HW switching
49 * between contexts.
50 */
51
52 if (!DRIVER_CAPS(i915)->has_logical_contexts)
53 return 0;
54
55 file = mock_file(i915);
56 if (IS_ERR(file))
57 return PTR_ERR(file);
58
59 ctx = kzalloc_objs(*ctx, nctx);
60 if (!ctx) {
61 err = -ENOMEM;
62 goto out_file;
63 }
64
65 for (n = 0; n < nctx; n++) {
66 ctx[n] = live_context(i915, file);
67 if (IS_ERR(ctx[n])) {
68 err = PTR_ERR(ctx[n]);
69 goto out_ctx;
70 }
71 }
72
73 for_each_uabi_engine(engine, i915) {
74 struct i915_request *rq = NULL;
75 unsigned long end_time, prime;
76 ktime_t times[2] = {};
77
78 times[0] = ktime_get_raw();
79 for (n = 0; n < nctx; n++) {
80 struct i915_request *this;
81
82 this = igt_request_alloc(ctx[n], engine);
83 if (IS_ERR(this)) {
84 err = PTR_ERR(this);
85 goto out_ctx;
86 }
87 if (rq) {
88 i915_request_await_dma_fence(this, &rq->fence);
89 i915_request_put(rq);
90 }
91 rq = i915_request_get(this);
92 i915_request_add(this);
93 }
94 if (i915_request_wait(rq, 0, 10 * HZ) < 0) {
95 pr_err("Failed to populated %d contexts\n", nctx);
96 intel_gt_set_wedged(engine->gt);
97 i915_request_put(rq);
98 err = -EIO;
99 goto out_ctx;
100 }
101 i915_request_put(rq);
102
103 times[1] = ktime_get_raw();
104
105 pr_info("Populated %d contexts on %s in %lluns\n",
106 nctx, engine->name, ktime_to_ns(times[1] - times[0]));
107
108 err = igt_live_test_begin(&t, i915, __func__, engine->name);
109 if (err)
110 goto out_ctx;
111
112 end_time = jiffies + i915_selftest.timeout_jiffies;
113 for_each_prime_number_from(prime, 2, 8192) {
114 times[1] = ktime_get_raw();
115
116 rq = NULL;
117 for (n = 0; n < prime; n++) {
118 struct i915_request *this;
119
120 this = igt_request_alloc(ctx[n % nctx], engine);
121 if (IS_ERR(this)) {
122 err = PTR_ERR(this);
123 goto out_ctx;
124 }
125
126 if (rq) { /* Force submission order */
127 i915_request_await_dma_fence(this, &rq->fence);
128 i915_request_put(rq);
129 }
130
131 /*
132 * This space is left intentionally blank.
133 *
134 * We do not actually want to perform any
135 * action with this request, we just want
136 * to measure the latency in allocation
137 * and submission of our breadcrumbs -
138 * ensuring that the bare request is sufficient
139 * for the system to work (i.e. proper HEAD
140 * tracking of the rings, interrupt handling,
141 * etc). It also gives us the lowest bounds
142 * for latency.
143 */
144
145 rq = i915_request_get(this);
146 i915_request_add(this);
147 }
148 GEM_BUG_ON(!rq);
149 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
150 pr_err("Switching between %ld contexts timed out\n",
151 prime);
152 intel_gt_set_wedged(engine->gt);
153 i915_request_put(rq);
154 break;
155 }
156 i915_request_put(rq);
157
158 times[1] = ktime_sub(ktime_get_raw(), times[1]);
159 if (prime == 2)
160 times[0] = times[1];
161
162 if (__igt_timeout(end_time, NULL))
163 break;
164 }
165
166 err = igt_live_test_end(&t);
167 if (err)
168 goto out_ctx;
169
170 pr_info("Switch latencies on %s: 1 = %lluns, %lu = %lluns\n",
171 engine->name,
172 ktime_to_ns(times[0]),
173 prime - 1, div64_u64(ktime_to_ns(times[1]), prime - 1));
174 }
175
176 out_ctx:
177 kfree(ctx);
178 out_file:
179 fput(file);
180 return err;
181 }
182
183 struct parallel_switch {
184 struct kthread_worker *worker;
185 struct kthread_work work;
186 struct intel_context *ce[2];
187 int result;
188 };
189
__live_parallel_switch1(struct kthread_work * work)190 static void __live_parallel_switch1(struct kthread_work *work)
191 {
192 struct parallel_switch *arg =
193 container_of(work, typeof(*arg), work);
194 IGT_TIMEOUT(end_time);
195 unsigned long count;
196
197 count = 0;
198 arg->result = 0;
199 do {
200 struct i915_request *rq = NULL;
201 int n;
202
203 for (n = 0; !arg->result && n < ARRAY_SIZE(arg->ce); n++) {
204 struct i915_request *prev = rq;
205
206 rq = i915_request_create(arg->ce[n]);
207 if (IS_ERR(rq)) {
208 i915_request_put(prev);
209 arg->result = PTR_ERR(rq);
210 break;
211 }
212
213 i915_request_get(rq);
214 if (prev) {
215 arg->result =
216 i915_request_await_dma_fence(rq,
217 &prev->fence);
218 i915_request_put(prev);
219 }
220
221 i915_request_add(rq);
222 }
223
224 if (IS_ERR_OR_NULL(rq))
225 break;
226
227 if (i915_request_wait(rq, 0, HZ) < 0)
228 arg->result = -ETIME;
229
230 i915_request_put(rq);
231
232 count++;
233 } while (!arg->result && !__igt_timeout(end_time, NULL));
234
235 pr_info("%s: %lu switches (sync) <%d>\n",
236 arg->ce[0]->engine->name, count, arg->result);
237 }
238
__live_parallel_switchN(struct kthread_work * work)239 static void __live_parallel_switchN(struct kthread_work *work)
240 {
241 struct parallel_switch *arg =
242 container_of(work, typeof(*arg), work);
243 struct i915_request *rq = NULL;
244 IGT_TIMEOUT(end_time);
245 unsigned long count;
246 int n;
247
248 count = 0;
249 arg->result = 0;
250 do {
251 for (n = 0; !arg->result && n < ARRAY_SIZE(arg->ce); n++) {
252 struct i915_request *prev = rq;
253
254 rq = i915_request_create(arg->ce[n]);
255 if (IS_ERR(rq)) {
256 i915_request_put(prev);
257 arg->result = PTR_ERR(rq);
258 break;
259 }
260
261 i915_request_get(rq);
262 if (prev) {
263 arg->result =
264 i915_request_await_dma_fence(rq,
265 &prev->fence);
266 i915_request_put(prev);
267 }
268
269 i915_request_add(rq);
270 }
271
272 count++;
273 } while (!arg->result && !__igt_timeout(end_time, NULL));
274
275 if (!IS_ERR_OR_NULL(rq))
276 i915_request_put(rq);
277
278 pr_info("%s: %lu switches (many) <%d>\n",
279 arg->ce[0]->engine->name, count, arg->result);
280 }
281
live_parallel_switch(void * arg)282 static int live_parallel_switch(void *arg)
283 {
284 struct drm_i915_private *i915 = arg;
285 static void (* const func[])(struct kthread_work *) = {
286 __live_parallel_switch1,
287 __live_parallel_switchN,
288 NULL,
289 };
290 struct parallel_switch *data = NULL;
291 struct i915_gem_engines *engines;
292 struct i915_gem_engines_iter it;
293 void (* const *fn)(struct kthread_work *);
294 struct i915_gem_context *ctx;
295 struct intel_context *ce;
296 struct file *file;
297 int n, m, count;
298 int err = 0;
299
300 /*
301 * Check we can process switches on all engines simultaneously.
302 */
303
304 if (!DRIVER_CAPS(i915)->has_logical_contexts)
305 return 0;
306
307 file = mock_file(i915);
308 if (IS_ERR(file))
309 return PTR_ERR(file);
310
311 ctx = live_context(i915, file);
312 if (IS_ERR(ctx)) {
313 err = PTR_ERR(ctx);
314 goto out_file;
315 }
316
317 engines = i915_gem_context_lock_engines(ctx);
318 count = engines->num_engines;
319
320 data = kzalloc_objs(*data, count);
321 if (!data) {
322 i915_gem_context_unlock_engines(ctx);
323 err = -ENOMEM;
324 goto out_file;
325 }
326
327 m = 0; /* Use the first context as our template for the engines */
328 for_each_gem_engine(ce, engines, it) {
329 err = intel_context_pin(ce);
330 if (err) {
331 i915_gem_context_unlock_engines(ctx);
332 goto out;
333 }
334 data[m++].ce[0] = intel_context_get(ce);
335 }
336 i915_gem_context_unlock_engines(ctx);
337
338 /* Clone the same set of engines into the other contexts */
339 for (n = 1; n < ARRAY_SIZE(data->ce); n++) {
340 ctx = live_context(i915, file);
341 if (IS_ERR(ctx)) {
342 err = PTR_ERR(ctx);
343 goto out;
344 }
345
346 for (m = 0; m < count; m++) {
347 if (!data[m].ce[0])
348 continue;
349
350 ce = intel_context_create(data[m].ce[0]->engine);
351 if (IS_ERR(ce)) {
352 err = PTR_ERR(ce);
353 goto out;
354 }
355
356 err = intel_context_pin(ce);
357 if (err) {
358 intel_context_put(ce);
359 goto out;
360 }
361
362 data[m].ce[n] = ce;
363 }
364 }
365
366 for (n = 0; n < count; n++) {
367 struct kthread_worker *worker;
368
369 if (!data[n].ce[0])
370 continue;
371
372 worker = kthread_run_worker(0, "igt/parallel:%s",
373 data[n].ce[0]->engine->name);
374 if (IS_ERR(worker)) {
375 err = PTR_ERR(worker);
376 goto out;
377 }
378
379 data[n].worker = worker;
380 }
381
382 for (fn = func; !err && *fn; fn++) {
383 struct igt_live_test t;
384
385 err = igt_live_test_begin(&t, i915, __func__, "");
386 if (err)
387 break;
388
389 for (n = 0; n < count; n++) {
390 if (!data[n].ce[0])
391 continue;
392
393 data[n].result = 0;
394 kthread_init_work(&data[n].work, *fn);
395 kthread_queue_work(data[n].worker, &data[n].work);
396 }
397
398 for (n = 0; n < count; n++) {
399 if (data[n].ce[0]) {
400 kthread_flush_work(&data[n].work);
401 if (data[n].result && !err)
402 err = data[n].result;
403 }
404 }
405
406 if (igt_live_test_end(&t)) {
407 err = err ?: -EIO;
408 break;
409 }
410 }
411
412 out:
413 for (n = 0; n < count; n++) {
414 for (m = 0; m < ARRAY_SIZE(data->ce); m++) {
415 if (!data[n].ce[m])
416 continue;
417
418 intel_context_unpin(data[n].ce[m]);
419 intel_context_put(data[n].ce[m]);
420 }
421
422 if (data[n].worker)
423 kthread_destroy_worker(data[n].worker);
424 }
425 kfree(data);
426 out_file:
427 fput(file);
428 return err;
429 }
430
real_page_count(struct drm_i915_gem_object * obj)431 static unsigned long real_page_count(struct drm_i915_gem_object *obj)
432 {
433 return huge_gem_object_phys_size(obj) >> PAGE_SHIFT;
434 }
435
fake_page_count(struct drm_i915_gem_object * obj)436 static unsigned long fake_page_count(struct drm_i915_gem_object *obj)
437 {
438 return huge_gem_object_dma_size(obj) >> PAGE_SHIFT;
439 }
440
gpu_fill(struct intel_context * ce,struct drm_i915_gem_object * obj,unsigned int dw)441 static int gpu_fill(struct intel_context *ce,
442 struct drm_i915_gem_object *obj,
443 unsigned int dw)
444 {
445 struct i915_vma *vma;
446 int err;
447
448 GEM_BUG_ON(obj->base.size > ce->vm->total);
449 GEM_BUG_ON(!intel_engine_can_store_dword(ce->engine));
450
451 vma = i915_vma_instance(obj, ce->vm, NULL);
452 if (IS_ERR(vma))
453 return PTR_ERR(vma);
454
455 err = i915_vma_pin(vma, 0, 0, PIN_HIGH | PIN_USER);
456 if (err)
457 return err;
458
459 /*
460 * Within the GTT the huge objects maps every page onto
461 * its 1024 real pages (using phys_pfn = dma_pfn % 1024).
462 * We set the nth dword within the page using the nth
463 * mapping via the GTT - this should exercise the GTT mapping
464 * whilst checking that each context provides a unique view
465 * into the object.
466 */
467 err = igt_gpu_fill_dw(ce, vma,
468 (dw * real_page_count(obj)) << PAGE_SHIFT |
469 (dw * sizeof(u32)),
470 real_page_count(obj),
471 dw);
472 i915_vma_unpin(vma);
473
474 return err;
475 }
476
cpu_fill(struct drm_i915_gem_object * obj,u32 value)477 static int cpu_fill(struct drm_i915_gem_object *obj, u32 value)
478 {
479 const bool has_llc = HAS_LLC(to_i915(obj->base.dev));
480 unsigned int need_flush;
481 unsigned long n, m;
482 int err;
483
484 i915_gem_object_lock(obj, NULL);
485 err = i915_gem_object_prepare_write(obj, &need_flush);
486 if (err)
487 goto out;
488
489 for (n = 0; n < real_page_count(obj); n++) {
490 u32 *map;
491
492 map = kmap_local_page(i915_gem_object_get_page(obj, n));
493 for (m = 0; m < DW_PER_PAGE; m++)
494 map[m] = value;
495 if (!has_llc)
496 drm_clflush_virt_range(map, PAGE_SIZE);
497 kunmap_local(map);
498 }
499
500 i915_gem_object_finish_access(obj);
501 obj->read_domains = I915_GEM_DOMAIN_GTT | I915_GEM_DOMAIN_CPU;
502 obj->write_domain = 0;
503 out:
504 i915_gem_object_unlock(obj);
505 return err;
506 }
507
cpu_check(struct drm_i915_gem_object * obj,unsigned int idx,unsigned int max)508 static noinline int cpu_check(struct drm_i915_gem_object *obj,
509 unsigned int idx, unsigned int max)
510 {
511 unsigned int needs_flush;
512 unsigned long n;
513 int err;
514
515 i915_gem_object_lock(obj, NULL);
516 err = i915_gem_object_prepare_read(obj, &needs_flush);
517 if (err)
518 goto out_unlock;
519
520 for (n = 0; n < real_page_count(obj); n++) {
521 u32 *map, m;
522
523 map = kmap_local_page(i915_gem_object_get_page(obj, n));
524 if (needs_flush & CLFLUSH_BEFORE)
525 drm_clflush_virt_range(map, PAGE_SIZE);
526
527 for (m = 0; m < max; m++) {
528 if (map[m] != m) {
529 pr_err("%pS: Invalid value at object %d page %ld/%ld, offset %d/%d: found %x expected %x\n",
530 __builtin_return_address(0), idx,
531 n, real_page_count(obj), m, max,
532 map[m], m);
533 err = -EINVAL;
534 goto out_unmap;
535 }
536 }
537
538 for (; m < DW_PER_PAGE; m++) {
539 if (map[m] != STACK_MAGIC) {
540 pr_err("%pS: Invalid value at object %d page %ld, offset %d: found %x expected %x (uninitialised)\n",
541 __builtin_return_address(0), idx, n, m,
542 map[m], STACK_MAGIC);
543 err = -EINVAL;
544 goto out_unmap;
545 }
546 }
547
548 out_unmap:
549 kunmap_local(map);
550 if (err)
551 break;
552 }
553
554 i915_gem_object_finish_access(obj);
555 out_unlock:
556 i915_gem_object_unlock(obj);
557 return err;
558 }
559
file_add_object(struct file * file,struct drm_i915_gem_object * obj)560 static int file_add_object(struct file *file, struct drm_i915_gem_object *obj)
561 {
562 int err;
563
564 GEM_BUG_ON(obj->base.handle_count);
565
566 /* tie the object to the drm_file for easy reaping */
567 err = idr_alloc(&to_drm_file(file)->object_idr,
568 &obj->base, 1, 0, GFP_KERNEL);
569 if (err < 0)
570 return err;
571
572 i915_gem_object_get(obj);
573 obj->base.handle_count++;
574 return 0;
575 }
576
577 static struct drm_i915_gem_object *
create_test_object(struct i915_address_space * vm,struct file * file,struct list_head * objects)578 create_test_object(struct i915_address_space *vm,
579 struct file *file,
580 struct list_head *objects)
581 {
582 struct drm_i915_gem_object *obj;
583 u64 size;
584 int err;
585
586 /* Keep in GEM's good graces */
587 intel_gt_retire_requests(vm->gt);
588
589 size = min(vm->total / 2, 1024ull * DW_PER_PAGE * PAGE_SIZE);
590 size = round_down(size, DW_PER_PAGE * PAGE_SIZE);
591
592 obj = huge_gem_object(vm->i915, DW_PER_PAGE * PAGE_SIZE, size);
593 if (IS_ERR(obj))
594 return obj;
595
596 err = file_add_object(file, obj);
597 i915_gem_object_put(obj);
598 if (err)
599 return ERR_PTR(err);
600
601 err = cpu_fill(obj, STACK_MAGIC);
602 if (err) {
603 pr_err("Failed to fill object with cpu, err=%d\n",
604 err);
605 return ERR_PTR(err);
606 }
607
608 list_add_tail(&obj->st_link, objects);
609 return obj;
610 }
611
max_dwords(struct drm_i915_gem_object * obj)612 static unsigned long max_dwords(struct drm_i915_gem_object *obj)
613 {
614 unsigned long npages = fake_page_count(obj);
615
616 GEM_BUG_ON(!IS_ALIGNED(npages, DW_PER_PAGE));
617 return npages / DW_PER_PAGE;
618 }
619
throttle_release(struct i915_request ** q,int count)620 static void throttle_release(struct i915_request **q, int count)
621 {
622 int i;
623
624 for (i = 0; i < count; i++) {
625 if (IS_ERR_OR_NULL(q[i]))
626 continue;
627
628 i915_request_put(fetch_and_zero(&q[i]));
629 }
630 }
631
throttle(struct intel_context * ce,struct i915_request ** q,int count)632 static int throttle(struct intel_context *ce,
633 struct i915_request **q, int count)
634 {
635 int i;
636
637 if (!IS_ERR_OR_NULL(q[0])) {
638 if (i915_request_wait(q[0],
639 I915_WAIT_INTERRUPTIBLE,
640 MAX_SCHEDULE_TIMEOUT) < 0)
641 return -EINTR;
642
643 i915_request_put(q[0]);
644 }
645
646 for (i = 0; i < count - 1; i++)
647 q[i] = q[i + 1];
648
649 q[i] = intel_context_create_request(ce);
650 if (IS_ERR(q[i]))
651 return PTR_ERR(q[i]);
652
653 i915_request_get(q[i]);
654 i915_request_add(q[i]);
655
656 return 0;
657 }
658
igt_ctx_exec(void * arg)659 static int igt_ctx_exec(void *arg)
660 {
661 struct drm_i915_private *i915 = arg;
662 struct intel_engine_cs *engine;
663 int err = -ENODEV;
664
665 /*
666 * Create a few different contexts (with different mm) and write
667 * through each ctx/mm using the GPU making sure those writes end
668 * up in the expected pages of our obj.
669 */
670
671 if (!DRIVER_CAPS(i915)->has_logical_contexts)
672 return 0;
673
674 for_each_uabi_engine(engine, i915) {
675 struct drm_i915_gem_object *obj = NULL;
676 unsigned long ncontexts, ndwords, dw;
677 struct i915_request *tq[5] = {};
678 struct igt_live_test t;
679 IGT_TIMEOUT(end_time);
680 LIST_HEAD(objects);
681 struct file *file;
682
683 if (!intel_engine_can_store_dword(engine))
684 continue;
685
686 if (!engine->context_size)
687 continue; /* No logical context support in HW */
688
689 file = mock_file(i915);
690 if (IS_ERR(file))
691 return PTR_ERR(file);
692
693 err = igt_live_test_begin(&t, i915, __func__, engine->name);
694 if (err)
695 goto out_file;
696
697 ncontexts = 0;
698 ndwords = 0;
699 dw = 0;
700 while (!time_after(jiffies, end_time)) {
701 struct i915_gem_context *ctx;
702 struct intel_context *ce;
703
704 ctx = kernel_context(i915, NULL);
705 if (IS_ERR(ctx)) {
706 err = PTR_ERR(ctx);
707 goto out_file;
708 }
709
710 ce = i915_gem_context_get_engine(ctx, engine->legacy_idx);
711 GEM_BUG_ON(IS_ERR(ce));
712
713 if (!obj) {
714 obj = create_test_object(ce->vm, file, &objects);
715 if (IS_ERR(obj)) {
716 err = PTR_ERR(obj);
717 intel_context_put(ce);
718 kernel_context_close(ctx);
719 goto out_file;
720 }
721 }
722
723 err = gpu_fill(ce, obj, dw);
724 if (err) {
725 pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) [full-ppgtt? %s], err=%d\n",
726 ndwords, dw, max_dwords(obj),
727 engine->name,
728 str_yes_no(i915_gem_context_has_full_ppgtt(ctx)),
729 err);
730 intel_context_put(ce);
731 kernel_context_close(ctx);
732 goto out_file;
733 }
734
735 err = throttle(ce, tq, ARRAY_SIZE(tq));
736 if (err) {
737 intel_context_put(ce);
738 kernel_context_close(ctx);
739 goto out_file;
740 }
741
742 if (++dw == max_dwords(obj)) {
743 obj = NULL;
744 dw = 0;
745 }
746
747 ndwords++;
748 ncontexts++;
749
750 intel_context_put(ce);
751 kernel_context_close(ctx);
752 }
753
754 pr_info("Submitted %lu contexts to %s, filling %lu dwords\n",
755 ncontexts, engine->name, ndwords);
756
757 ncontexts = dw = 0;
758 list_for_each_entry(obj, &objects, st_link) {
759 unsigned int rem =
760 min_t(unsigned int, ndwords - dw, max_dwords(obj));
761
762 err = cpu_check(obj, ncontexts++, rem);
763 if (err)
764 break;
765
766 dw += rem;
767 }
768
769 out_file:
770 throttle_release(tq, ARRAY_SIZE(tq));
771 if (igt_live_test_end(&t))
772 err = -EIO;
773
774 fput(file);
775 if (err)
776 return err;
777
778 i915_gem_drain_freed_objects(i915);
779 }
780
781 return 0;
782 }
783
igt_shared_ctx_exec(void * arg)784 static int igt_shared_ctx_exec(void *arg)
785 {
786 struct drm_i915_private *i915 = arg;
787 struct i915_request *tq[5] = {};
788 struct i915_gem_context *parent;
789 struct intel_engine_cs *engine;
790 struct igt_live_test t;
791 struct file *file;
792 int err = 0;
793
794 /*
795 * Create a few different contexts with the same mm and write
796 * through each ctx using the GPU making sure those writes end
797 * up in the expected pages of our obj.
798 */
799 if (!DRIVER_CAPS(i915)->has_logical_contexts)
800 return 0;
801
802 file = mock_file(i915);
803 if (IS_ERR(file))
804 return PTR_ERR(file);
805
806 parent = live_context(i915, file);
807 if (IS_ERR(parent)) {
808 err = PTR_ERR(parent);
809 goto out_file;
810 }
811
812 if (!parent->vm) { /* not full-ppgtt; nothing to share */
813 err = 0;
814 goto out_file;
815 }
816
817 err = igt_live_test_begin(&t, i915, __func__, "");
818 if (err)
819 goto out_file;
820
821 for_each_uabi_engine(engine, i915) {
822 unsigned long ncontexts, ndwords, dw;
823 struct drm_i915_gem_object *obj = NULL;
824 IGT_TIMEOUT(end_time);
825 LIST_HEAD(objects);
826
827 if (!intel_engine_can_store_dword(engine))
828 continue;
829
830 dw = 0;
831 ndwords = 0;
832 ncontexts = 0;
833 while (!time_after(jiffies, end_time)) {
834 struct i915_gem_context *ctx;
835 struct intel_context *ce;
836
837 ctx = kernel_context(i915, parent->vm);
838 if (IS_ERR(ctx)) {
839 err = PTR_ERR(ctx);
840 goto out_test;
841 }
842
843 ce = i915_gem_context_get_engine(ctx, engine->legacy_idx);
844 GEM_BUG_ON(IS_ERR(ce));
845
846 if (!obj) {
847 obj = create_test_object(parent->vm,
848 file, &objects);
849 if (IS_ERR(obj)) {
850 err = PTR_ERR(obj);
851 intel_context_put(ce);
852 kernel_context_close(ctx);
853 goto out_test;
854 }
855 }
856
857 err = gpu_fill(ce, obj, dw);
858 if (err) {
859 pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) [full-ppgtt? %s], err=%d\n",
860 ndwords, dw, max_dwords(obj),
861 engine->name,
862 str_yes_no(i915_gem_context_has_full_ppgtt(ctx)),
863 err);
864 intel_context_put(ce);
865 kernel_context_close(ctx);
866 goto out_test;
867 }
868
869 err = throttle(ce, tq, ARRAY_SIZE(tq));
870 if (err) {
871 intel_context_put(ce);
872 kernel_context_close(ctx);
873 goto out_test;
874 }
875
876 if (++dw == max_dwords(obj)) {
877 obj = NULL;
878 dw = 0;
879 }
880
881 ndwords++;
882 ncontexts++;
883
884 intel_context_put(ce);
885 kernel_context_close(ctx);
886 }
887 pr_info("Submitted %lu contexts to %s, filling %lu dwords\n",
888 ncontexts, engine->name, ndwords);
889
890 ncontexts = dw = 0;
891 list_for_each_entry(obj, &objects, st_link) {
892 unsigned int rem =
893 min_t(unsigned int, ndwords - dw, max_dwords(obj));
894
895 err = cpu_check(obj, ncontexts++, rem);
896 if (err)
897 goto out_test;
898
899 dw += rem;
900 }
901
902 i915_gem_drain_freed_objects(i915);
903 }
904 out_test:
905 throttle_release(tq, ARRAY_SIZE(tq));
906 if (igt_live_test_end(&t))
907 err = -EIO;
908 out_file:
909 fput(file);
910 return err;
911 }
912
rpcs_query_batch(struct drm_i915_gem_object * rpcs,struct i915_vma * vma,struct intel_engine_cs * engine)913 static int rpcs_query_batch(struct drm_i915_gem_object *rpcs,
914 struct i915_vma *vma,
915 struct intel_engine_cs *engine)
916 {
917 u32 *cmd;
918
919 GEM_BUG_ON(GRAPHICS_VER(vma->vm->i915) < 8);
920
921 cmd = i915_gem_object_pin_map(rpcs, I915_MAP_WB);
922 if (IS_ERR(cmd))
923 return PTR_ERR(cmd);
924
925 *cmd++ = MI_STORE_REGISTER_MEM_GEN8;
926 *cmd++ = i915_mmio_reg_offset(GEN8_R_PWR_CLK_STATE(engine->mmio_base));
927 *cmd++ = lower_32_bits(i915_vma_offset(vma));
928 *cmd++ = upper_32_bits(i915_vma_offset(vma));
929 *cmd = MI_BATCH_BUFFER_END;
930
931 __i915_gem_object_flush_map(rpcs, 0, 64);
932 i915_gem_object_unpin_map(rpcs);
933
934 intel_gt_chipset_flush(vma->vm->gt);
935
936 return 0;
937 }
938
939 static int
emit_rpcs_query(struct drm_i915_gem_object * obj,struct intel_context * ce,struct i915_request ** rq_out)940 emit_rpcs_query(struct drm_i915_gem_object *obj,
941 struct intel_context *ce,
942 struct i915_request **rq_out)
943 {
944 struct drm_i915_private *i915 = to_i915(obj->base.dev);
945 struct i915_request *rq;
946 struct i915_gem_ww_ctx ww;
947 struct i915_vma *batch;
948 struct i915_vma *vma;
949 struct drm_i915_gem_object *rpcs;
950 int err;
951
952 GEM_BUG_ON(!intel_engine_can_store_dword(ce->engine));
953
954 if (GRAPHICS_VER(i915) < 8)
955 return -EINVAL;
956
957 vma = i915_vma_instance(obj, ce->vm, NULL);
958 if (IS_ERR(vma))
959 return PTR_ERR(vma);
960
961 rpcs = i915_gem_object_create_internal(i915, PAGE_SIZE);
962 if (IS_ERR(rpcs))
963 return PTR_ERR(rpcs);
964
965 i915_gem_ww_ctx_init(&ww, false);
966
967 batch = i915_vma_instance(rpcs, ce->vm, NULL);
968 if (IS_ERR(batch)) {
969 err = PTR_ERR(batch);
970 goto err_put;
971 }
972
973 retry:
974 err = i915_gem_object_lock(obj, &ww);
975 if (!err)
976 err = i915_gem_object_lock(rpcs, &ww);
977 if (!err)
978 err = i915_gem_object_set_to_gtt_domain(obj, false);
979 if (!err)
980 err = i915_vma_pin_ww(vma, &ww, 0, 0, PIN_USER);
981 if (err)
982 goto err_put;
983
984 err = i915_vma_pin_ww(batch, &ww, 0, 0, PIN_USER);
985 if (err)
986 goto err_vma;
987
988 err = rpcs_query_batch(rpcs, vma, ce->engine);
989 if (err)
990 goto err_batch;
991
992 rq = i915_request_create(ce);
993 if (IS_ERR(rq)) {
994 err = PTR_ERR(rq);
995 goto err_batch;
996 }
997
998 err = i915_vma_move_to_active(batch, rq, 0);
999 if (err)
1000 goto skip_request;
1001
1002 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
1003 if (err)
1004 goto skip_request;
1005
1006 if (rq->engine->emit_init_breadcrumb) {
1007 err = rq->engine->emit_init_breadcrumb(rq);
1008 if (err)
1009 goto skip_request;
1010 }
1011
1012 err = rq->engine->emit_bb_start(rq,
1013 i915_vma_offset(batch),
1014 i915_vma_size(batch),
1015 0);
1016 if (err)
1017 goto skip_request;
1018
1019 *rq_out = i915_request_get(rq);
1020
1021 skip_request:
1022 if (err)
1023 i915_request_set_error_once(rq, err);
1024 i915_request_add(rq);
1025 err_batch:
1026 i915_vma_unpin(batch);
1027 err_vma:
1028 i915_vma_unpin(vma);
1029 err_put:
1030 if (err == -EDEADLK) {
1031 err = i915_gem_ww_ctx_backoff(&ww);
1032 if (!err)
1033 goto retry;
1034 }
1035 i915_gem_ww_ctx_fini(&ww);
1036 i915_gem_object_put(rpcs);
1037 return err;
1038 }
1039
1040 #define TEST_IDLE BIT(0)
1041 #define TEST_BUSY BIT(1)
1042 #define TEST_RESET BIT(2)
1043
1044 static int
__sseu_prepare(const char * name,unsigned int flags,struct intel_context * ce,struct igt_spinner ** spin)1045 __sseu_prepare(const char *name,
1046 unsigned int flags,
1047 struct intel_context *ce,
1048 struct igt_spinner **spin)
1049 {
1050 struct i915_request *rq;
1051 int ret;
1052
1053 *spin = NULL;
1054 if (!(flags & (TEST_BUSY | TEST_RESET)))
1055 return 0;
1056
1057 *spin = kzalloc_obj(**spin);
1058 if (!*spin)
1059 return -ENOMEM;
1060
1061 ret = igt_spinner_init(*spin, ce->engine->gt);
1062 if (ret)
1063 goto err_free;
1064
1065 rq = igt_spinner_create_request(*spin, ce, MI_NOOP);
1066 if (IS_ERR(rq)) {
1067 ret = PTR_ERR(rq);
1068 goto err_fini;
1069 }
1070
1071 i915_request_add(rq);
1072
1073 if (!igt_wait_for_spinner(*spin, rq)) {
1074 pr_err("%s: Spinner failed to start!\n", name);
1075 ret = -ETIMEDOUT;
1076 goto err_end;
1077 }
1078
1079 return 0;
1080
1081 err_end:
1082 igt_spinner_end(*spin);
1083 err_fini:
1084 igt_spinner_fini(*spin);
1085 err_free:
1086 kfree(fetch_and_zero(spin));
1087 return ret;
1088 }
1089
1090 static int
__read_slice_count(struct intel_context * ce,struct drm_i915_gem_object * obj,struct igt_spinner * spin,u32 * rpcs)1091 __read_slice_count(struct intel_context *ce,
1092 struct drm_i915_gem_object *obj,
1093 struct igt_spinner *spin,
1094 u32 *rpcs)
1095 {
1096 struct i915_request *rq = NULL;
1097 u32 s_mask, s_shift;
1098 unsigned int cnt;
1099 u32 *buf, val;
1100 long ret;
1101
1102 ret = emit_rpcs_query(obj, ce, &rq);
1103 if (ret)
1104 return ret;
1105
1106 if (spin)
1107 igt_spinner_end(spin);
1108
1109 ret = i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT);
1110 i915_request_put(rq);
1111 if (ret < 0)
1112 return ret;
1113
1114 buf = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
1115 if (IS_ERR(buf)) {
1116 ret = PTR_ERR(buf);
1117 return ret;
1118 }
1119
1120 if (GRAPHICS_VER(ce->engine->i915) >= 11) {
1121 s_mask = GEN11_RPCS_S_CNT_MASK;
1122 s_shift = GEN11_RPCS_S_CNT_SHIFT;
1123 } else {
1124 s_mask = GEN8_RPCS_S_CNT_MASK;
1125 s_shift = GEN8_RPCS_S_CNT_SHIFT;
1126 }
1127
1128 val = *buf;
1129 cnt = (val & s_mask) >> s_shift;
1130 *rpcs = val;
1131
1132 i915_gem_object_unpin_map(obj);
1133
1134 return cnt;
1135 }
1136
1137 static int
__check_rpcs(const char * name,u32 rpcs,int slices,unsigned int expected,const char * prefix,const char * suffix)1138 __check_rpcs(const char *name, u32 rpcs, int slices, unsigned int expected,
1139 const char *prefix, const char *suffix)
1140 {
1141 if (slices == expected)
1142 return 0;
1143
1144 if (slices < 0) {
1145 pr_err("%s: %s read slice count failed with %d%s\n",
1146 name, prefix, slices, suffix);
1147 return slices;
1148 }
1149
1150 pr_err("%s: %s slice count %d is not %u%s\n",
1151 name, prefix, slices, expected, suffix);
1152
1153 pr_info("RPCS=0x%x; %u%sx%u%s\n",
1154 rpcs, slices,
1155 (rpcs & GEN8_RPCS_S_CNT_ENABLE) ? "*" : "",
1156 (rpcs & GEN8_RPCS_SS_CNT_MASK) >> GEN8_RPCS_SS_CNT_SHIFT,
1157 (rpcs & GEN8_RPCS_SS_CNT_ENABLE) ? "*" : "");
1158
1159 return -EINVAL;
1160 }
1161
1162 static int
__sseu_finish(const char * name,unsigned int flags,struct intel_context * ce,struct drm_i915_gem_object * obj,unsigned int expected,struct igt_spinner * spin)1163 __sseu_finish(const char *name,
1164 unsigned int flags,
1165 struct intel_context *ce,
1166 struct drm_i915_gem_object *obj,
1167 unsigned int expected,
1168 struct igt_spinner *spin)
1169 {
1170 unsigned int slices = hweight32(ce->engine->sseu.slice_mask);
1171 u32 rpcs = 0;
1172 int ret = 0;
1173
1174 if (flags & TEST_RESET) {
1175 ret = intel_engine_reset(ce->engine, "sseu");
1176 if (ret)
1177 goto out;
1178 }
1179
1180 ret = __read_slice_count(ce, obj,
1181 flags & TEST_RESET ? NULL : spin, &rpcs);
1182 ret = __check_rpcs(name, rpcs, ret, expected, "Context", "!");
1183 if (ret)
1184 goto out;
1185
1186 ret = __read_slice_count(ce->engine->kernel_context, obj, NULL, &rpcs);
1187 ret = __check_rpcs(name, rpcs, ret, slices, "Kernel context", "!");
1188
1189 out:
1190 if (spin)
1191 igt_spinner_end(spin);
1192
1193 if ((flags & TEST_IDLE) && ret == 0) {
1194 ret = igt_flush_test(ce->engine->i915);
1195 if (ret)
1196 return ret;
1197
1198 ret = __read_slice_count(ce, obj, NULL, &rpcs);
1199 ret = __check_rpcs(name, rpcs, ret, expected,
1200 "Context", " after idle!");
1201 }
1202
1203 return ret;
1204 }
1205
1206 static int
__sseu_test(const char * name,unsigned int flags,struct intel_context * ce,struct drm_i915_gem_object * obj,struct intel_sseu sseu)1207 __sseu_test(const char *name,
1208 unsigned int flags,
1209 struct intel_context *ce,
1210 struct drm_i915_gem_object *obj,
1211 struct intel_sseu sseu)
1212 {
1213 struct igt_spinner *spin = NULL;
1214 int ret;
1215
1216 intel_engine_pm_get(ce->engine);
1217
1218 ret = __sseu_prepare(name, flags, ce, &spin);
1219 if (ret)
1220 goto out_pm;
1221
1222 ret = intel_context_reconfigure_sseu(ce, sseu);
1223 if (ret)
1224 goto out_spin;
1225
1226 ret = __sseu_finish(name, flags, ce, obj,
1227 hweight32(sseu.slice_mask), spin);
1228
1229 out_spin:
1230 if (spin) {
1231 igt_spinner_end(spin);
1232 igt_spinner_fini(spin);
1233 kfree(spin);
1234 }
1235 out_pm:
1236 intel_engine_pm_put(ce->engine);
1237 return ret;
1238 }
1239
1240 static int
__igt_ctx_sseu(struct drm_i915_private * i915,const char * name,unsigned int flags)1241 __igt_ctx_sseu(struct drm_i915_private *i915,
1242 const char *name,
1243 unsigned int flags)
1244 {
1245 struct drm_i915_gem_object *obj;
1246 int inst = 0;
1247 int ret = 0;
1248
1249 if (GRAPHICS_VER(i915) < 9)
1250 return 0;
1251
1252 if (flags & TEST_RESET)
1253 igt_global_reset_lock(to_gt(i915));
1254
1255 obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
1256 if (IS_ERR(obj)) {
1257 ret = PTR_ERR(obj);
1258 goto out_unlock;
1259 }
1260
1261 do {
1262 struct intel_engine_cs *engine;
1263 struct intel_context *ce;
1264 struct intel_sseu pg_sseu;
1265
1266 engine = intel_engine_lookup_user(i915,
1267 I915_ENGINE_CLASS_RENDER,
1268 inst++);
1269 if (!engine)
1270 break;
1271
1272 if (hweight32(engine->sseu.slice_mask) < 2)
1273 continue;
1274
1275 if (!engine->gt->info.sseu.has_slice_pg)
1276 continue;
1277
1278 /*
1279 * Gen11 VME friendly power-gated configuration with
1280 * half enabled sub-slices.
1281 */
1282 pg_sseu = engine->sseu;
1283 pg_sseu.slice_mask = 1;
1284 pg_sseu.subslice_mask =
1285 ~(~0 << (hweight32(engine->sseu.subslice_mask) / 2));
1286
1287 pr_info("%s: SSEU subtest '%s', flags=%x, def_slices=%u, pg_slices=%u\n",
1288 engine->name, name, flags,
1289 hweight32(engine->sseu.slice_mask),
1290 hweight32(pg_sseu.slice_mask));
1291
1292 ce = intel_context_create(engine);
1293 if (IS_ERR(ce)) {
1294 ret = PTR_ERR(ce);
1295 goto out_put;
1296 }
1297
1298 ret = intel_context_pin(ce);
1299 if (ret)
1300 goto out_ce;
1301
1302 /* First set the default mask. */
1303 ret = __sseu_test(name, flags, ce, obj, engine->sseu);
1304 if (ret)
1305 goto out_unpin;
1306
1307 /* Then set a power-gated configuration. */
1308 ret = __sseu_test(name, flags, ce, obj, pg_sseu);
1309 if (ret)
1310 goto out_unpin;
1311
1312 /* Back to defaults. */
1313 ret = __sseu_test(name, flags, ce, obj, engine->sseu);
1314 if (ret)
1315 goto out_unpin;
1316
1317 /* One last power-gated configuration for the road. */
1318 ret = __sseu_test(name, flags, ce, obj, pg_sseu);
1319 if (ret)
1320 goto out_unpin;
1321
1322 out_unpin:
1323 intel_context_unpin(ce);
1324 out_ce:
1325 intel_context_put(ce);
1326 } while (!ret);
1327
1328 if (igt_flush_test(i915))
1329 ret = -EIO;
1330
1331 out_put:
1332 i915_gem_object_put(obj);
1333
1334 out_unlock:
1335 if (flags & TEST_RESET)
1336 igt_global_reset_unlock(to_gt(i915));
1337
1338 if (ret)
1339 pr_err("%s: Failed with %d!\n", name, ret);
1340
1341 return ret;
1342 }
1343
igt_ctx_sseu(void * arg)1344 static int igt_ctx_sseu(void *arg)
1345 {
1346 struct {
1347 const char *name;
1348 unsigned int flags;
1349 } *phase, phases[] = {
1350 { .name = "basic", .flags = 0 },
1351 { .name = "idle", .flags = TEST_IDLE },
1352 { .name = "busy", .flags = TEST_BUSY },
1353 { .name = "busy-reset", .flags = TEST_BUSY | TEST_RESET },
1354 { .name = "busy-idle", .flags = TEST_BUSY | TEST_IDLE },
1355 { .name = "reset-idle", .flags = TEST_RESET | TEST_IDLE },
1356 };
1357 unsigned int i;
1358 int ret = 0;
1359
1360 for (i = 0, phase = phases; ret == 0 && i < ARRAY_SIZE(phases);
1361 i++, phase++)
1362 ret = __igt_ctx_sseu(arg, phase->name, phase->flags);
1363
1364 return ret;
1365 }
1366
igt_ctx_readonly(void * arg)1367 static int igt_ctx_readonly(void *arg)
1368 {
1369 struct drm_i915_private *i915 = arg;
1370 unsigned long idx, ndwords, dw, num_engines;
1371 struct drm_i915_gem_object *obj = NULL;
1372 struct i915_request *tq[5] = {};
1373 struct i915_gem_engines_iter it;
1374 struct i915_address_space *vm;
1375 struct i915_gem_context *ctx;
1376 struct intel_context *ce;
1377 struct igt_live_test t;
1378 I915_RND_STATE(prng);
1379 IGT_TIMEOUT(end_time);
1380 LIST_HEAD(objects);
1381 struct file *file;
1382 int err = -ENODEV;
1383
1384 /*
1385 * Create a few read-only objects (with the occasional writable object)
1386 * and try to write into these object checking that the GPU discards
1387 * any write to a read-only object.
1388 */
1389
1390 file = mock_file(i915);
1391 if (IS_ERR(file))
1392 return PTR_ERR(file);
1393
1394 err = igt_live_test_begin(&t, i915, __func__, "");
1395 if (err)
1396 goto out_file;
1397
1398 ctx = live_context(i915, file);
1399 if (IS_ERR(ctx)) {
1400 err = PTR_ERR(ctx);
1401 goto out_file;
1402 }
1403
1404 vm = ctx->vm ?: &to_gt(i915)->ggtt->alias->vm;
1405 if (!vm || !vm->has_read_only) {
1406 err = 0;
1407 goto out_file;
1408 }
1409
1410 num_engines = 0;
1411 for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it)
1412 if (intel_engine_can_store_dword(ce->engine))
1413 num_engines++;
1414 i915_gem_context_unlock_engines(ctx);
1415
1416 ndwords = 0;
1417 dw = 0;
1418 while (!time_after(jiffies, end_time)) {
1419 for_each_gem_engine(ce,
1420 i915_gem_context_lock_engines(ctx), it) {
1421 if (!intel_engine_can_store_dword(ce->engine))
1422 continue;
1423
1424 if (!obj) {
1425 obj = create_test_object(ce->vm, file, &objects);
1426 if (IS_ERR(obj)) {
1427 err = PTR_ERR(obj);
1428 i915_gem_context_unlock_engines(ctx);
1429 goto out_file;
1430 }
1431
1432 if (prandom_u32_state(&prng) & 1)
1433 i915_gem_object_set_readonly(obj);
1434 }
1435
1436 err = gpu_fill(ce, obj, dw);
1437 if (err) {
1438 pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) [full-ppgtt? %s], err=%d\n",
1439 ndwords, dw, max_dwords(obj),
1440 ce->engine->name,
1441 str_yes_no(i915_gem_context_has_full_ppgtt(ctx)),
1442 err);
1443 i915_gem_context_unlock_engines(ctx);
1444 goto out_file;
1445 }
1446
1447 err = throttle(ce, tq, ARRAY_SIZE(tq));
1448 if (err) {
1449 i915_gem_context_unlock_engines(ctx);
1450 goto out_file;
1451 }
1452
1453 if (++dw == max_dwords(obj)) {
1454 obj = NULL;
1455 dw = 0;
1456 }
1457 ndwords++;
1458 }
1459 i915_gem_context_unlock_engines(ctx);
1460 }
1461 pr_info("Submitted %lu dwords (across %lu engines)\n",
1462 ndwords, num_engines);
1463
1464 dw = 0;
1465 idx = 0;
1466 list_for_each_entry(obj, &objects, st_link) {
1467 unsigned int rem =
1468 min_t(unsigned int, ndwords - dw, max_dwords(obj));
1469 unsigned int num_writes;
1470
1471 num_writes = rem;
1472 if (i915_gem_object_is_readonly(obj))
1473 num_writes = 0;
1474
1475 err = cpu_check(obj, idx++, num_writes);
1476 if (err)
1477 break;
1478
1479 dw += rem;
1480 }
1481
1482 out_file:
1483 throttle_release(tq, ARRAY_SIZE(tq));
1484 if (igt_live_test_end(&t))
1485 err = -EIO;
1486
1487 fput(file);
1488 return err;
1489 }
1490
check_scratch(struct i915_address_space * vm,u64 offset)1491 static int check_scratch(struct i915_address_space *vm, u64 offset)
1492 {
1493 struct drm_mm_node *node;
1494
1495 mutex_lock(&vm->mutex);
1496 node = __drm_mm_interval_first(&vm->mm,
1497 offset, offset + sizeof(u32) - 1);
1498 mutex_unlock(&vm->mutex);
1499 if (!node || node->start > offset)
1500 return 0;
1501
1502 GEM_BUG_ON(offset >= node->start + node->size);
1503
1504 pr_err("Target offset 0x%08x_%08x overlaps with a node in the mm!\n",
1505 upper_32_bits(offset), lower_32_bits(offset));
1506 return -EINVAL;
1507 }
1508
write_to_scratch(struct i915_gem_context * ctx,struct intel_engine_cs * engine,struct drm_i915_gem_object * obj,u64 offset,u32 value)1509 static int write_to_scratch(struct i915_gem_context *ctx,
1510 struct intel_engine_cs *engine,
1511 struct drm_i915_gem_object *obj,
1512 u64 offset, u32 value)
1513 {
1514 struct drm_i915_private *i915 = ctx->i915;
1515 struct i915_address_space *vm;
1516 struct i915_request *rq;
1517 struct i915_vma *vma;
1518 u32 *cmd;
1519 int err;
1520
1521 GEM_BUG_ON(offset < I915_GTT_PAGE_SIZE);
1522
1523 err = check_scratch(ctx->vm, offset);
1524 if (err)
1525 return err;
1526
1527 cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
1528 if (IS_ERR(cmd))
1529 return PTR_ERR(cmd);
1530
1531 *cmd++ = MI_STORE_DWORD_IMM_GEN4;
1532 if (GRAPHICS_VER(i915) >= 8) {
1533 *cmd++ = lower_32_bits(offset);
1534 *cmd++ = upper_32_bits(offset);
1535 } else {
1536 *cmd++ = 0;
1537 *cmd++ = offset;
1538 }
1539 *cmd++ = value;
1540 *cmd = MI_BATCH_BUFFER_END;
1541 __i915_gem_object_flush_map(obj, 0, 64);
1542 i915_gem_object_unpin_map(obj);
1543
1544 intel_gt_chipset_flush(engine->gt);
1545
1546 vm = i915_gem_context_get_eb_vm(ctx);
1547 vma = i915_vma_instance(obj, vm, NULL);
1548 if (IS_ERR(vma)) {
1549 err = PTR_ERR(vma);
1550 goto out_vm;
1551 }
1552
1553 err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_OFFSET_FIXED);
1554 if (err)
1555 goto out_vm;
1556
1557 rq = igt_request_alloc(ctx, engine);
1558 if (IS_ERR(rq)) {
1559 err = PTR_ERR(rq);
1560 goto err_unpin;
1561 }
1562
1563 err = igt_vma_move_to_active_unlocked(vma, rq, 0);
1564 if (err)
1565 goto skip_request;
1566
1567 if (rq->engine->emit_init_breadcrumb) {
1568 err = rq->engine->emit_init_breadcrumb(rq);
1569 if (err)
1570 goto skip_request;
1571 }
1572
1573 err = engine->emit_bb_start(rq, i915_vma_offset(vma),
1574 i915_vma_size(vma), 0);
1575 if (err)
1576 goto skip_request;
1577
1578 i915_vma_unpin(vma);
1579
1580 i915_request_add(rq);
1581
1582 goto out_vm;
1583 skip_request:
1584 i915_request_set_error_once(rq, err);
1585 i915_request_add(rq);
1586 err_unpin:
1587 i915_vma_unpin(vma);
1588 out_vm:
1589 i915_vm_put(vm);
1590
1591 if (!err)
1592 err = i915_gem_object_wait(obj, 0, MAX_SCHEDULE_TIMEOUT);
1593
1594 return err;
1595 }
1596
read_from_scratch(struct i915_gem_context * ctx,struct intel_engine_cs * engine,struct drm_i915_gem_object * obj,u64 offset,u32 * value)1597 static int read_from_scratch(struct i915_gem_context *ctx,
1598 struct intel_engine_cs *engine,
1599 struct drm_i915_gem_object *obj,
1600 u64 offset, u32 *value)
1601 {
1602 struct drm_i915_private *i915 = ctx->i915;
1603 struct i915_address_space *vm;
1604 const u32 result = 0x100;
1605 struct i915_request *rq;
1606 struct i915_vma *vma;
1607 unsigned int flags;
1608 u32 *cmd;
1609 int err;
1610
1611 GEM_BUG_ON(offset < I915_GTT_PAGE_SIZE);
1612
1613 err = check_scratch(ctx->vm, offset);
1614 if (err)
1615 return err;
1616
1617 if (GRAPHICS_VER(i915) >= 8) {
1618 const u32 GPR0 = engine->mmio_base + 0x600;
1619
1620 vm = i915_gem_context_get_eb_vm(ctx);
1621 vma = i915_vma_instance(obj, vm, NULL);
1622 if (IS_ERR(vma)) {
1623 err = PTR_ERR(vma);
1624 goto out_vm;
1625 }
1626
1627 err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_OFFSET_FIXED);
1628 if (err)
1629 goto out_vm;
1630
1631 cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
1632 if (IS_ERR(cmd)) {
1633 err = PTR_ERR(cmd);
1634 goto err_unpin;
1635 }
1636
1637 memset(cmd, POISON_INUSE, PAGE_SIZE);
1638 *cmd++ = MI_LOAD_REGISTER_MEM_GEN8;
1639 *cmd++ = GPR0;
1640 *cmd++ = lower_32_bits(offset);
1641 *cmd++ = upper_32_bits(offset);
1642 *cmd++ = MI_STORE_REGISTER_MEM_GEN8;
1643 *cmd++ = GPR0;
1644 *cmd++ = result;
1645 *cmd++ = 0;
1646 *cmd = MI_BATCH_BUFFER_END;
1647
1648 i915_gem_object_flush_map(obj);
1649 i915_gem_object_unpin_map(obj);
1650
1651 flags = 0;
1652 } else {
1653 const u32 reg = engine->mmio_base + 0x420;
1654
1655 /* hsw: register access even to 3DPRIM! is protected */
1656 vm = i915_vm_get(&engine->gt->ggtt->vm);
1657 vma = i915_vma_instance(obj, vm, NULL);
1658 if (IS_ERR(vma)) {
1659 err = PTR_ERR(vma);
1660 goto out_vm;
1661 }
1662
1663 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
1664 if (err)
1665 goto out_vm;
1666
1667 cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
1668 if (IS_ERR(cmd)) {
1669 err = PTR_ERR(cmd);
1670 goto err_unpin;
1671 }
1672
1673 memset(cmd, POISON_INUSE, PAGE_SIZE);
1674 *cmd++ = MI_LOAD_REGISTER_MEM;
1675 *cmd++ = reg;
1676 *cmd++ = offset;
1677 *cmd++ = MI_STORE_REGISTER_MEM | MI_USE_GGTT;
1678 *cmd++ = reg;
1679 *cmd++ = i915_vma_offset(vma) + result;
1680 *cmd = MI_BATCH_BUFFER_END;
1681
1682 i915_gem_object_flush_map(obj);
1683 i915_gem_object_unpin_map(obj);
1684
1685 flags = I915_DISPATCH_SECURE;
1686 }
1687
1688 intel_gt_chipset_flush(engine->gt);
1689
1690 rq = igt_request_alloc(ctx, engine);
1691 if (IS_ERR(rq)) {
1692 err = PTR_ERR(rq);
1693 goto err_unpin;
1694 }
1695
1696 err = igt_vma_move_to_active_unlocked(vma, rq, EXEC_OBJECT_WRITE);
1697 if (err)
1698 goto skip_request;
1699
1700 if (rq->engine->emit_init_breadcrumb) {
1701 err = rq->engine->emit_init_breadcrumb(rq);
1702 if (err)
1703 goto skip_request;
1704 }
1705
1706 err = engine->emit_bb_start(rq, i915_vma_offset(vma),
1707 i915_vma_size(vma), flags);
1708 if (err)
1709 goto skip_request;
1710
1711 i915_vma_unpin(vma);
1712
1713 i915_request_add(rq);
1714
1715 i915_gem_object_lock(obj, NULL);
1716 err = i915_gem_object_set_to_cpu_domain(obj, false);
1717 i915_gem_object_unlock(obj);
1718 if (err)
1719 goto out_vm;
1720
1721 cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
1722 if (IS_ERR(cmd)) {
1723 err = PTR_ERR(cmd);
1724 goto out_vm;
1725 }
1726
1727 *value = cmd[result / sizeof(*cmd)];
1728 i915_gem_object_unpin_map(obj);
1729
1730 goto out_vm;
1731 skip_request:
1732 i915_request_set_error_once(rq, err);
1733 i915_request_add(rq);
1734 err_unpin:
1735 i915_vma_unpin(vma);
1736 out_vm:
1737 i915_vm_put(vm);
1738
1739 if (!err)
1740 err = i915_gem_object_wait(obj, 0, MAX_SCHEDULE_TIMEOUT);
1741
1742 return err;
1743 }
1744
check_scratch_page(struct i915_gem_context * ctx,u32 * out)1745 static int check_scratch_page(struct i915_gem_context *ctx, u32 *out)
1746 {
1747 struct i915_address_space *vm;
1748 u32 *vaddr;
1749 int err = 0;
1750
1751 vm = ctx->vm;
1752 if (!vm)
1753 return -ENODEV;
1754
1755 if (!vm->scratch[0]) {
1756 pr_err("No scratch page!\n");
1757 return -EINVAL;
1758 }
1759
1760 vaddr = __px_vaddr(vm->scratch[0]);
1761
1762 memcpy(out, vaddr, sizeof(*out));
1763 if (memchr_inv(vaddr, *out, PAGE_SIZE)) {
1764 pr_err("Inconsistent initial state of scratch page!\n");
1765 err = -EINVAL;
1766 }
1767
1768 return err;
1769 }
1770
igt_vm_isolation(void * arg)1771 static int igt_vm_isolation(void *arg)
1772 {
1773 struct drm_i915_private *i915 = arg;
1774 struct i915_gem_context *ctx_a, *ctx_b;
1775 struct drm_i915_gem_object *obj_a, *obj_b;
1776 unsigned long num_engines, count;
1777 struct intel_engine_cs *engine;
1778 struct igt_live_test t;
1779 I915_RND_STATE(prng);
1780 struct file *file;
1781 u64 vm_total;
1782 u32 expected;
1783 int err;
1784
1785 if (GRAPHICS_VER(i915) < 7)
1786 return 0;
1787
1788 /*
1789 * The simple goal here is that a write into one context is not
1790 * observed in a second (separate page tables and scratch).
1791 */
1792
1793 file = mock_file(i915);
1794 if (IS_ERR(file))
1795 return PTR_ERR(file);
1796
1797 err = igt_live_test_begin(&t, i915, __func__, "");
1798 if (err)
1799 goto out_file;
1800
1801 ctx_a = live_context(i915, file);
1802 if (IS_ERR(ctx_a)) {
1803 err = PTR_ERR(ctx_a);
1804 goto out_file;
1805 }
1806
1807 ctx_b = live_context(i915, file);
1808 if (IS_ERR(ctx_b)) {
1809 err = PTR_ERR(ctx_b);
1810 goto out_file;
1811 }
1812
1813 /* We can only test vm isolation, if the vm are distinct */
1814 if (ctx_a->vm == ctx_b->vm)
1815 goto out_file;
1816
1817 /* Read the initial state of the scratch page */
1818 err = check_scratch_page(ctx_a, &expected);
1819 if (err)
1820 goto out_file;
1821
1822 err = check_scratch_page(ctx_b, &expected);
1823 if (err)
1824 goto out_file;
1825
1826 vm_total = ctx_a->vm->total;
1827 GEM_BUG_ON(ctx_b->vm->total != vm_total);
1828
1829 obj_a = i915_gem_object_create_internal(i915, PAGE_SIZE);
1830 if (IS_ERR(obj_a)) {
1831 err = PTR_ERR(obj_a);
1832 goto out_file;
1833 }
1834
1835 obj_b = i915_gem_object_create_internal(i915, PAGE_SIZE);
1836 if (IS_ERR(obj_b)) {
1837 err = PTR_ERR(obj_b);
1838 goto put_a;
1839 }
1840
1841 count = 0;
1842 num_engines = 0;
1843 for_each_uabi_engine(engine, i915) {
1844 IGT_TIMEOUT(end_time);
1845 unsigned long this = 0;
1846
1847 if (!intel_engine_can_store_dword(engine))
1848 continue;
1849
1850 /* Not all engines have their own GPR! */
1851 if (GRAPHICS_VER(i915) < 8 && engine->class != RENDER_CLASS)
1852 continue;
1853
1854 while (!__igt_timeout(end_time, NULL)) {
1855 u32 value = 0xc5c5c5c5;
1856 u64 offset;
1857
1858 /* Leave enough space at offset 0 for the batch */
1859 offset = igt_random_offset(&prng,
1860 I915_GTT_PAGE_SIZE, vm_total,
1861 sizeof(u32), alignof_dword);
1862
1863 err = write_to_scratch(ctx_a, engine, obj_a,
1864 offset, 0xdeadbeef);
1865 if (err == 0)
1866 err = read_from_scratch(ctx_b, engine, obj_b,
1867 offset, &value);
1868 if (err)
1869 goto put_b;
1870
1871 if (value != expected) {
1872 pr_err("%s: Read %08x from scratch (offset 0x%08x_%08x), after %lu reads!\n",
1873 engine->name, value,
1874 upper_32_bits(offset),
1875 lower_32_bits(offset),
1876 this);
1877 err = -EINVAL;
1878 goto put_b;
1879 }
1880
1881 this++;
1882 }
1883 count += this;
1884 num_engines++;
1885 }
1886 pr_info("Checked %lu scratch offsets across %lu engines\n",
1887 count, num_engines);
1888
1889 put_b:
1890 i915_gem_object_put(obj_b);
1891 put_a:
1892 i915_gem_object_put(obj_a);
1893 out_file:
1894 if (igt_live_test_end(&t))
1895 err = -EIO;
1896 fput(file);
1897 return err;
1898 }
1899
i915_gem_context_live_selftests(struct drm_i915_private * i915)1900 int i915_gem_context_live_selftests(struct drm_i915_private *i915)
1901 {
1902 static const struct i915_subtest tests[] = {
1903 SUBTEST(live_nop_switch),
1904 SUBTEST(live_parallel_switch),
1905 SUBTEST(igt_ctx_exec),
1906 SUBTEST(igt_ctx_readonly),
1907 SUBTEST(igt_ctx_sseu),
1908 SUBTEST(igt_shared_ctx_exec),
1909 SUBTEST(igt_vm_isolation),
1910 };
1911
1912 if (intel_gt_is_wedged(to_gt(i915)))
1913 return 0;
1914
1915 return i915_live_subtests(tests, i915);
1916 }
1917