1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2017-2018 Intel Corporation
4 */
5
6 #include <linux/prime_numbers.h>
7 #include <linux/string_helpers.h>
8
9 #include "intel_context.h"
10 #include "intel_engine_heartbeat.h"
11 #include "intel_engine_pm.h"
12 #include "intel_engine_regs.h"
13 #include "intel_gpu_commands.h"
14 #include "intel_gt.h"
15 #include "intel_gt_requests.h"
16 #include "intel_ring.h"
17 #include "selftest_engine_heartbeat.h"
18
19 #include "../selftests/i915_random.h"
20 #include "../i915_selftest.h"
21
22 #include "selftests/igt_flush_test.h"
23 #include "selftests/lib_sw_fence.h"
24 #include "selftests/mock_gem_device.h"
25 #include "selftests/mock_timeline.h"
26
hwsp_page(struct intel_timeline * tl)27 static struct page *hwsp_page(struct intel_timeline *tl)
28 {
29 struct drm_i915_gem_object *obj = tl->hwsp_ggtt->obj;
30
31 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
32 return sg_page(obj->mm.pages->sgl);
33 }
34
hwsp_cacheline(struct intel_timeline * tl)35 static unsigned long hwsp_cacheline(struct intel_timeline *tl)
36 {
37 unsigned long address = (unsigned long)page_address(hwsp_page(tl));
38
39 return (address + offset_in_page(tl->hwsp_offset)) / TIMELINE_SEQNO_BYTES;
40 }
41
selftest_tl_pin(struct intel_timeline * tl)42 static int selftest_tl_pin(struct intel_timeline *tl)
43 {
44 struct i915_gem_ww_ctx ww;
45 int err;
46
47 i915_gem_ww_ctx_init(&ww, false);
48 retry:
49 err = i915_gem_object_lock(tl->hwsp_ggtt->obj, &ww);
50 if (!err)
51 err = intel_timeline_pin(tl, &ww);
52
53 if (err == -EDEADLK) {
54 err = i915_gem_ww_ctx_backoff(&ww);
55 if (!err)
56 goto retry;
57 }
58 i915_gem_ww_ctx_fini(&ww);
59 return err;
60 }
61
62 /* Only half of seqno's are usable, see __intel_timeline_get_seqno() */
63 #define CACHELINES_PER_PAGE (PAGE_SIZE / TIMELINE_SEQNO_BYTES / 2)
64
65 struct mock_hwsp_freelist {
66 struct intel_gt *gt;
67 struct radix_tree_root cachelines;
68 struct intel_timeline **history;
69 unsigned long count, max;
70 struct rnd_state prng;
71 };
72
73 enum {
74 SHUFFLE = BIT(0),
75 };
76
__mock_hwsp_record(struct mock_hwsp_freelist * state,unsigned int idx,struct intel_timeline * tl)77 static void __mock_hwsp_record(struct mock_hwsp_freelist *state,
78 unsigned int idx,
79 struct intel_timeline *tl)
80 {
81 tl = xchg(&state->history[idx], tl);
82 if (tl) {
83 radix_tree_delete(&state->cachelines, hwsp_cacheline(tl));
84 intel_timeline_unpin(tl);
85 intel_timeline_put(tl);
86 }
87 }
88
__mock_hwsp_timeline(struct mock_hwsp_freelist * state,unsigned int count,unsigned int flags)89 static int __mock_hwsp_timeline(struct mock_hwsp_freelist *state,
90 unsigned int count,
91 unsigned int flags)
92 {
93 struct intel_timeline *tl;
94 unsigned int idx;
95
96 while (count--) {
97 unsigned long cacheline;
98 int err;
99
100 tl = intel_timeline_create(state->gt);
101 if (IS_ERR(tl))
102 return PTR_ERR(tl);
103
104 err = selftest_tl_pin(tl);
105 if (err) {
106 intel_timeline_put(tl);
107 return err;
108 }
109
110 cacheline = hwsp_cacheline(tl);
111 err = radix_tree_insert(&state->cachelines, cacheline, tl);
112 if (err) {
113 if (err == -EEXIST) {
114 pr_err("HWSP cacheline %lu already used; duplicate allocation!\n",
115 cacheline);
116 }
117 intel_timeline_unpin(tl);
118 intel_timeline_put(tl);
119 return err;
120 }
121
122 idx = state->count++ % state->max;
123 __mock_hwsp_record(state, idx, tl);
124 }
125
126 if (flags & SHUFFLE)
127 i915_prandom_shuffle(state->history,
128 sizeof(*state->history),
129 min(state->count, state->max),
130 &state->prng);
131
132 count = i915_prandom_u32_max_state(min(state->count, state->max),
133 &state->prng);
134 while (count--) {
135 idx = --state->count % state->max;
136 __mock_hwsp_record(state, idx, NULL);
137 }
138
139 return 0;
140 }
141
mock_hwsp_freelist(void * arg)142 static int mock_hwsp_freelist(void *arg)
143 {
144 struct mock_hwsp_freelist state;
145 struct drm_i915_private *i915;
146 const struct {
147 const char *name;
148 unsigned int flags;
149 } phases[] = {
150 { "linear", 0 },
151 { "shuffled", SHUFFLE },
152 { },
153 }, *p;
154 unsigned int na;
155 int err = 0;
156
157 i915 = mock_gem_device();
158 if (!i915)
159 return -ENOMEM;
160
161 INIT_RADIX_TREE(&state.cachelines, GFP_KERNEL);
162 state.prng = I915_RND_STATE_INITIALIZER(i915_selftest.random_seed);
163
164 state.gt = to_gt(i915);
165
166 /*
167 * Create a bunch of timelines and check that their HWSP do not overlap.
168 * Free some, and try again.
169 */
170
171 state.max = PAGE_SIZE / sizeof(*state.history);
172 state.count = 0;
173 state.history = kzalloc_objs(*state.history, state.max);
174 if (!state.history) {
175 err = -ENOMEM;
176 goto err_put;
177 }
178
179 for (p = phases; p->name; p++) {
180 pr_debug("%s(%s)\n", __func__, p->name);
181 for_each_prime_number_from(na, 1, 2 * CACHELINES_PER_PAGE) {
182 err = __mock_hwsp_timeline(&state, na, p->flags);
183 if (err)
184 goto out;
185 }
186 }
187
188 out:
189 for (na = 0; na < state.max; na++)
190 __mock_hwsp_record(&state, na, NULL);
191 kfree(state.history);
192 err_put:
193 mock_destroy_device(i915);
194 return err;
195 }
196
197 struct __igt_sync {
198 const char *name;
199 u32 seqno;
200 bool expected;
201 bool set;
202 };
203
__igt_sync(struct intel_timeline * tl,u64 ctx,const struct __igt_sync * p,const char * name)204 static int __igt_sync(struct intel_timeline *tl,
205 u64 ctx,
206 const struct __igt_sync *p,
207 const char *name)
208 {
209 int ret;
210
211 if (__intel_timeline_sync_is_later(tl, ctx, p->seqno) != p->expected) {
212 pr_err("%s: %s(ctx=%llu, seqno=%u) expected passed %s but failed\n",
213 name, p->name, ctx, p->seqno, str_yes_no(p->expected));
214 return -EINVAL;
215 }
216
217 if (p->set) {
218 ret = __intel_timeline_sync_set(tl, ctx, p->seqno);
219 if (ret)
220 return ret;
221 }
222
223 return 0;
224 }
225
igt_sync(void * arg)226 static int igt_sync(void *arg)
227 {
228 const struct __igt_sync pass[] = {
229 { "unset", 0, false, false },
230 { "new", 0, false, true },
231 { "0a", 0, true, true },
232 { "1a", 1, false, true },
233 { "1b", 1, true, true },
234 { "0b", 0, true, false },
235 { "2a", 2, false, true },
236 { "4", 4, false, true },
237 { "INT_MAX", INT_MAX, false, true },
238 { "INT_MAX-1", INT_MAX-1, true, false },
239 { "INT_MAX+1", (u32)INT_MAX+1, false, true },
240 { "INT_MAX", INT_MAX, true, false },
241 { "UINT_MAX", UINT_MAX, false, true },
242 { "wrap", 0, false, true },
243 { "unwrap", UINT_MAX, true, false },
244 {},
245 }, *p;
246 struct intel_timeline tl;
247 int order, offset;
248 int ret = -ENODEV;
249
250 mock_timeline_init(&tl, 0);
251 for (p = pass; p->name; p++) {
252 for (order = 1; order < 64; order++) {
253 for (offset = -1; offset <= (order > 1); offset++) {
254 u64 ctx = BIT_ULL(order) + offset;
255
256 ret = __igt_sync(&tl, ctx, p, "1");
257 if (ret)
258 goto out;
259 }
260 }
261 }
262 mock_timeline_fini(&tl);
263
264 mock_timeline_init(&tl, 0);
265 for (order = 1; order < 64; order++) {
266 for (offset = -1; offset <= (order > 1); offset++) {
267 u64 ctx = BIT_ULL(order) + offset;
268
269 for (p = pass; p->name; p++) {
270 ret = __igt_sync(&tl, ctx, p, "2");
271 if (ret)
272 goto out;
273 }
274 }
275 }
276
277 out:
278 mock_timeline_fini(&tl);
279 return ret;
280 }
281
random_engine(struct rnd_state * rnd)282 static unsigned int random_engine(struct rnd_state *rnd)
283 {
284 return i915_prandom_u32_max_state(I915_NUM_ENGINES, rnd);
285 }
286
bench_sync(void * arg)287 static int bench_sync(void *arg)
288 {
289 struct rnd_state prng;
290 struct intel_timeline tl;
291 unsigned long end_time, count;
292 u64 prng32_1M;
293 ktime_t kt;
294 int order, last_order;
295
296 mock_timeline_init(&tl, 0);
297
298 /* Lookups from cache are very fast and so the random number generation
299 * and the loop itself becomes a significant factor in the per-iteration
300 * timings. We try to compensate the results by measuring the overhead
301 * of the prng and subtract it from the reported results.
302 */
303 prandom_seed_state(&prng, i915_selftest.random_seed);
304 count = 0;
305 kt = ktime_get();
306 end_time = jiffies + HZ/10;
307 do {
308 u32 x;
309
310 /* Make sure the compiler doesn't optimise away the prng call */
311 WRITE_ONCE(x, prandom_u32_state(&prng));
312
313 count++;
314 } while (!time_after(jiffies, end_time));
315 kt = ktime_sub(ktime_get(), kt);
316 pr_debug("%s: %lu random evaluations, %lluns/prng\n",
317 __func__, count, (long long)div64_ul(ktime_to_ns(kt), count));
318 prng32_1M = div64_ul(ktime_to_ns(kt) << 20, count);
319
320 /* Benchmark (only) setting random context ids */
321 prandom_seed_state(&prng, i915_selftest.random_seed);
322 count = 0;
323 kt = ktime_get();
324 end_time = jiffies + HZ/10;
325 do {
326 u64 id = i915_prandom_u64_state(&prng);
327
328 __intel_timeline_sync_set(&tl, id, 0);
329 count++;
330 } while (!time_after(jiffies, end_time));
331 kt = ktime_sub(ktime_get(), kt);
332 kt = ktime_sub_ns(kt, (count * prng32_1M * 2) >> 20);
333 pr_info("%s: %lu random insertions, %lluns/insert\n",
334 __func__, count, (long long)div64_ul(ktime_to_ns(kt), count));
335
336 /* Benchmark looking up the exact same context ids as we just set */
337 prandom_seed_state(&prng, i915_selftest.random_seed);
338 end_time = count;
339 kt = ktime_get();
340 while (end_time--) {
341 u64 id = i915_prandom_u64_state(&prng);
342
343 if (!__intel_timeline_sync_is_later(&tl, id, 0)) {
344 mock_timeline_fini(&tl);
345 pr_err("Lookup of %llu failed\n", id);
346 return -EINVAL;
347 }
348 }
349 kt = ktime_sub(ktime_get(), kt);
350 kt = ktime_sub_ns(kt, (count * prng32_1M * 2) >> 20);
351 pr_info("%s: %lu random lookups, %lluns/lookup\n",
352 __func__, count, (long long)div64_ul(ktime_to_ns(kt), count));
353
354 mock_timeline_fini(&tl);
355 cond_resched();
356
357 mock_timeline_init(&tl, 0);
358
359 /* Benchmark setting the first N (in order) contexts */
360 count = 0;
361 kt = ktime_get();
362 end_time = jiffies + HZ/10;
363 do {
364 __intel_timeline_sync_set(&tl, count++, 0);
365 } while (!time_after(jiffies, end_time));
366 kt = ktime_sub(ktime_get(), kt);
367 pr_info("%s: %lu in-order insertions, %lluns/insert\n",
368 __func__, count, (long long)div64_ul(ktime_to_ns(kt), count));
369
370 /* Benchmark looking up the exact same context ids as we just set */
371 end_time = count;
372 kt = ktime_get();
373 while (end_time--) {
374 if (!__intel_timeline_sync_is_later(&tl, end_time, 0)) {
375 pr_err("Lookup of %lu failed\n", end_time);
376 mock_timeline_fini(&tl);
377 return -EINVAL;
378 }
379 }
380 kt = ktime_sub(ktime_get(), kt);
381 pr_info("%s: %lu in-order lookups, %lluns/lookup\n",
382 __func__, count, (long long)div64_ul(ktime_to_ns(kt), count));
383
384 mock_timeline_fini(&tl);
385 cond_resched();
386
387 mock_timeline_init(&tl, 0);
388
389 /* Benchmark searching for a random context id and maybe changing it */
390 prandom_seed_state(&prng, i915_selftest.random_seed);
391 count = 0;
392 kt = ktime_get();
393 end_time = jiffies + HZ/10;
394 do {
395 u32 id = random_engine(&prng);
396 u32 seqno = prandom_u32_state(&prng);
397
398 if (!__intel_timeline_sync_is_later(&tl, id, seqno))
399 __intel_timeline_sync_set(&tl, id, seqno);
400
401 count++;
402 } while (!time_after(jiffies, end_time));
403 kt = ktime_sub(ktime_get(), kt);
404 kt = ktime_sub_ns(kt, (count * prng32_1M * 2) >> 20);
405 pr_info("%s: %lu repeated insert/lookups, %lluns/op\n",
406 __func__, count, (long long)div64_ul(ktime_to_ns(kt), count));
407 mock_timeline_fini(&tl);
408 cond_resched();
409
410 /* Benchmark searching for a known context id and changing the seqno */
411 for (last_order = 1, order = 1; order < 32;
412 ({ int tmp = last_order; last_order = order; order += tmp; })) {
413 unsigned int mask = BIT(order) - 1;
414
415 mock_timeline_init(&tl, 0);
416
417 count = 0;
418 kt = ktime_get();
419 end_time = jiffies + HZ/10;
420 do {
421 /* Without assuming too many details of the underlying
422 * implementation, try to identify its phase-changes
423 * (if any)!
424 */
425 u64 id = (u64)(count & mask) << order;
426
427 __intel_timeline_sync_is_later(&tl, id, 0);
428 __intel_timeline_sync_set(&tl, id, 0);
429
430 count++;
431 } while (!time_after(jiffies, end_time));
432 kt = ktime_sub(ktime_get(), kt);
433 pr_info("%s: %lu cyclic/%d insert/lookups, %lluns/op\n",
434 __func__, count, order,
435 (long long)div64_ul(ktime_to_ns(kt), count));
436 mock_timeline_fini(&tl);
437 cond_resched();
438 }
439
440 return 0;
441 }
442
intel_timeline_mock_selftests(void)443 int intel_timeline_mock_selftests(void)
444 {
445 static const struct i915_subtest tests[] = {
446 SUBTEST(mock_hwsp_freelist),
447 SUBTEST(igt_sync),
448 SUBTEST(bench_sync),
449 };
450
451 return i915_subtests(tests, NULL);
452 }
453
emit_ggtt_store_dw(struct i915_request * rq,u32 addr,u32 value)454 static int emit_ggtt_store_dw(struct i915_request *rq, u32 addr, u32 value)
455 {
456 u32 *cs;
457
458 cs = intel_ring_begin(rq, 4);
459 if (IS_ERR(cs))
460 return PTR_ERR(cs);
461
462 if (GRAPHICS_VER(rq->i915) >= 8) {
463 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
464 *cs++ = addr;
465 *cs++ = 0;
466 *cs++ = value;
467 } else if (GRAPHICS_VER(rq->i915) >= 4) {
468 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
469 *cs++ = 0;
470 *cs++ = addr;
471 *cs++ = value;
472 } else {
473 *cs++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
474 *cs++ = addr;
475 *cs++ = value;
476 *cs++ = MI_NOOP;
477 }
478
479 intel_ring_advance(rq, cs);
480
481 return 0;
482 }
483
484 static struct i915_request *
checked_tl_write(struct intel_timeline * tl,struct intel_engine_cs * engine,u32 value)485 checked_tl_write(struct intel_timeline *tl, struct intel_engine_cs *engine, u32 value)
486 {
487 struct i915_request *rq;
488 int err;
489
490 err = selftest_tl_pin(tl);
491 if (err) {
492 rq = ERR_PTR(err);
493 goto out;
494 }
495
496 if (READ_ONCE(*tl->hwsp_seqno) != tl->seqno) {
497 pr_err("Timeline created with incorrect breadcrumb, found %x, expected %x\n",
498 *tl->hwsp_seqno, tl->seqno);
499 intel_timeline_unpin(tl);
500 return ERR_PTR(-EINVAL);
501 }
502
503 rq = intel_engine_create_kernel_request(engine);
504 if (IS_ERR(rq))
505 goto out_unpin;
506
507 i915_request_get(rq);
508
509 err = emit_ggtt_store_dw(rq, tl->hwsp_offset, value);
510 i915_request_add(rq);
511 if (err) {
512 i915_request_put(rq);
513 rq = ERR_PTR(err);
514 }
515
516 out_unpin:
517 intel_timeline_unpin(tl);
518 out:
519 if (IS_ERR(rq))
520 pr_err("Failed to write to timeline!\n");
521 return rq;
522 }
523
live_hwsp_engine(void * arg)524 static int live_hwsp_engine(void *arg)
525 {
526 #define NUM_TIMELINES 4096
527 struct intel_gt *gt = arg;
528 struct intel_timeline **timelines;
529 struct intel_engine_cs *engine;
530 enum intel_engine_id id;
531 unsigned long count, n;
532 int err = 0;
533
534 /*
535 * Create a bunch of timelines and check we can write
536 * independently to each of their breadcrumb slots.
537 */
538
539 timelines = kvmalloc_objs(*timelines, NUM_TIMELINES * I915_NUM_ENGINES);
540 if (!timelines)
541 return -ENOMEM;
542
543 count = 0;
544 for_each_engine(engine, gt, id) {
545 if (!intel_engine_can_store_dword(engine))
546 continue;
547
548 intel_engine_pm_get(engine);
549
550 for (n = 0; n < NUM_TIMELINES; n++) {
551 struct intel_timeline *tl;
552 struct i915_request *rq;
553
554 tl = intel_timeline_create(gt);
555 if (IS_ERR(tl)) {
556 err = PTR_ERR(tl);
557 break;
558 }
559
560 rq = checked_tl_write(tl, engine, count);
561 if (IS_ERR(rq)) {
562 intel_timeline_put(tl);
563 err = PTR_ERR(rq);
564 break;
565 }
566
567 timelines[count++] = tl;
568 i915_request_put(rq);
569 }
570
571 intel_engine_pm_put(engine);
572 if (err)
573 break;
574 }
575
576 if (igt_flush_test(gt->i915))
577 err = -EIO;
578
579 for (n = 0; n < count; n++) {
580 struct intel_timeline *tl = timelines[n];
581
582 if (!err && READ_ONCE(*tl->hwsp_seqno) != n) {
583 GEM_TRACE_ERR("Invalid seqno:%lu stored in timeline %llu @ %x, found 0x%x\n",
584 n, tl->fence_context, tl->hwsp_offset, *tl->hwsp_seqno);
585 GEM_TRACE_DUMP();
586 err = -EINVAL;
587 }
588 intel_timeline_put(tl);
589 }
590
591 kvfree(timelines);
592 return err;
593 #undef NUM_TIMELINES
594 }
595
live_hwsp_alternate(void * arg)596 static int live_hwsp_alternate(void *arg)
597 {
598 #define NUM_TIMELINES 4096
599 struct intel_gt *gt = arg;
600 struct intel_timeline **timelines;
601 struct intel_engine_cs *engine;
602 enum intel_engine_id id;
603 unsigned long count, n;
604 int err = 0;
605
606 /*
607 * Create a bunch of timelines and check we can write
608 * independently to each of their breadcrumb slots with adjacent
609 * engines.
610 */
611
612 timelines = kvmalloc_objs(*timelines, NUM_TIMELINES * I915_NUM_ENGINES);
613 if (!timelines)
614 return -ENOMEM;
615
616 count = 0;
617 for (n = 0; n < NUM_TIMELINES; n++) {
618 for_each_engine(engine, gt, id) {
619 struct intel_timeline *tl;
620 struct i915_request *rq;
621
622 if (!intel_engine_can_store_dword(engine))
623 continue;
624
625 tl = intel_timeline_create(gt);
626 if (IS_ERR(tl)) {
627 err = PTR_ERR(tl);
628 goto out;
629 }
630
631 intel_engine_pm_get(engine);
632 rq = checked_tl_write(tl, engine, count);
633 intel_engine_pm_put(engine);
634 if (IS_ERR(rq)) {
635 intel_timeline_put(tl);
636 err = PTR_ERR(rq);
637 goto out;
638 }
639
640 timelines[count++] = tl;
641 i915_request_put(rq);
642 }
643 }
644
645 out:
646 if (igt_flush_test(gt->i915))
647 err = -EIO;
648
649 for (n = 0; n < count; n++) {
650 struct intel_timeline *tl = timelines[n];
651
652 if (!err && READ_ONCE(*tl->hwsp_seqno) != n) {
653 GEM_TRACE_ERR("Invalid seqno:%lu stored in timeline %llu @ %x, found 0x%x\n",
654 n, tl->fence_context, tl->hwsp_offset, *tl->hwsp_seqno);
655 GEM_TRACE_DUMP();
656 err = -EINVAL;
657 }
658 intel_timeline_put(tl);
659 }
660
661 kvfree(timelines);
662 return err;
663 #undef NUM_TIMELINES
664 }
665
live_hwsp_wrap(void * arg)666 static int live_hwsp_wrap(void *arg)
667 {
668 struct intel_gt *gt = arg;
669 struct intel_engine_cs *engine;
670 struct intel_timeline *tl;
671 enum intel_engine_id id;
672 int err = 0;
673
674 /*
675 * Across a seqno wrap, we need to keep the old cacheline alive for
676 * foreign GPU references.
677 */
678
679 tl = intel_timeline_create(gt);
680 if (IS_ERR(tl))
681 return PTR_ERR(tl);
682
683 if (!tl->has_initial_breadcrumb)
684 goto out_free;
685
686 err = selftest_tl_pin(tl);
687 if (err)
688 goto out_free;
689
690 for_each_engine(engine, gt, id) {
691 const u32 *hwsp_seqno[2];
692 struct i915_request *rq;
693 u32 seqno[2];
694
695 if (!intel_engine_can_store_dword(engine))
696 continue;
697
698 rq = intel_engine_create_kernel_request(engine);
699 if (IS_ERR(rq)) {
700 err = PTR_ERR(rq);
701 goto out;
702 }
703
704 tl->seqno = -4u;
705
706 mutex_lock_nested(&tl->mutex, SINGLE_DEPTH_NESTING);
707 err = intel_timeline_get_seqno(tl, rq, &seqno[0]);
708 mutex_unlock(&tl->mutex);
709 if (err) {
710 i915_request_add(rq);
711 goto out;
712 }
713 pr_debug("seqno[0]:%08x, hwsp_offset:%08x\n",
714 seqno[0], tl->hwsp_offset);
715
716 err = emit_ggtt_store_dw(rq, tl->hwsp_offset, seqno[0]);
717 if (err) {
718 i915_request_add(rq);
719 goto out;
720 }
721 hwsp_seqno[0] = tl->hwsp_seqno;
722
723 mutex_lock_nested(&tl->mutex, SINGLE_DEPTH_NESTING);
724 err = intel_timeline_get_seqno(tl, rq, &seqno[1]);
725 mutex_unlock(&tl->mutex);
726 if (err) {
727 i915_request_add(rq);
728 goto out;
729 }
730 pr_debug("seqno[1]:%08x, hwsp_offset:%08x\n",
731 seqno[1], tl->hwsp_offset);
732
733 err = emit_ggtt_store_dw(rq, tl->hwsp_offset, seqno[1]);
734 if (err) {
735 i915_request_add(rq);
736 goto out;
737 }
738 hwsp_seqno[1] = tl->hwsp_seqno;
739
740 /* With wrap should come a new hwsp */
741 GEM_BUG_ON(seqno[1] >= seqno[0]);
742 GEM_BUG_ON(hwsp_seqno[0] == hwsp_seqno[1]);
743
744 i915_request_add(rq);
745
746 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
747 pr_err("Wait for timeline writes timed out!\n");
748 err = -EIO;
749 goto out;
750 }
751
752 if (READ_ONCE(*hwsp_seqno[0]) != seqno[0] ||
753 READ_ONCE(*hwsp_seqno[1]) != seqno[1]) {
754 pr_err("Bad timeline values: found (%x, %x), expected (%x, %x)\n",
755 *hwsp_seqno[0], *hwsp_seqno[1],
756 seqno[0], seqno[1]);
757 err = -EINVAL;
758 goto out;
759 }
760
761 intel_gt_retire_requests(gt); /* recycle HWSP */
762 }
763
764 out:
765 if (igt_flush_test(gt->i915))
766 err = -EIO;
767
768 intel_timeline_unpin(tl);
769 out_free:
770 intel_timeline_put(tl);
771 return err;
772 }
773
emit_read_hwsp(struct i915_request * rq,u32 seqno,u32 hwsp,u32 * addr)774 static int emit_read_hwsp(struct i915_request *rq,
775 u32 seqno, u32 hwsp,
776 u32 *addr)
777 {
778 const u32 gpr = i915_mmio_reg_offset(GEN8_RING_CS_GPR(rq->engine->mmio_base, 0));
779 u32 *cs;
780
781 cs = intel_ring_begin(rq, 12);
782 if (IS_ERR(cs))
783 return PTR_ERR(cs);
784
785 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
786 *cs++ = *addr;
787 *cs++ = 0;
788 *cs++ = seqno;
789 *addr += 4;
790
791 *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | MI_USE_GGTT;
792 *cs++ = gpr;
793 *cs++ = hwsp;
794 *cs++ = 0;
795
796 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
797 *cs++ = gpr;
798 *cs++ = *addr;
799 *cs++ = 0;
800 *addr += 4;
801
802 intel_ring_advance(rq, cs);
803
804 return 0;
805 }
806
807 struct hwsp_watcher {
808 struct i915_vma *vma;
809 struct i915_request *rq;
810 u32 addr;
811 u32 *map;
812 };
813
cmp_lt(u32 a,u32 b)814 static bool cmp_lt(u32 a, u32 b)
815 {
816 return a < b;
817 }
818
cmp_gte(u32 a,u32 b)819 static bool cmp_gte(u32 a, u32 b)
820 {
821 return a >= b;
822 }
823
setup_watcher(struct hwsp_watcher * w,struct intel_gt * gt,struct intel_timeline * tl)824 static int setup_watcher(struct hwsp_watcher *w, struct intel_gt *gt,
825 struct intel_timeline *tl)
826 {
827 struct drm_i915_gem_object *obj;
828 struct i915_vma *vma;
829
830 obj = i915_gem_object_create_internal(gt->i915, SZ_2M);
831 if (IS_ERR(obj))
832 return PTR_ERR(obj);
833
834 /* keep the same cache settings as timeline */
835 i915_gem_object_set_pat_index(obj, tl->hwsp_ggtt->obj->pat_index);
836 w->map = i915_gem_object_pin_map_unlocked(obj,
837 page_unmask_bits(tl->hwsp_ggtt->obj->mm.mapping));
838 if (IS_ERR(w->map)) {
839 i915_gem_object_put(obj);
840 return PTR_ERR(w->map);
841 }
842
843 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);
844 if (IS_ERR(vma)) {
845 i915_gem_object_put(obj);
846 return PTR_ERR(vma);
847 }
848
849 w->vma = vma;
850 w->addr = i915_ggtt_offset(vma);
851 return 0;
852 }
853
switch_tl_lock(struct i915_request * from,struct i915_request * to)854 static void switch_tl_lock(struct i915_request *from, struct i915_request *to)
855 {
856 /* some light mutex juggling required; think co-routines */
857
858 if (from) {
859 lockdep_unpin_lock(&from->context->timeline->mutex, from->cookie);
860 mutex_unlock(&from->context->timeline->mutex);
861 }
862
863 if (to) {
864 mutex_lock(&to->context->timeline->mutex);
865 to->cookie = lockdep_pin_lock(&to->context->timeline->mutex);
866 }
867 }
868
create_watcher(struct hwsp_watcher * w,struct intel_engine_cs * engine,int ringsz)869 static int create_watcher(struct hwsp_watcher *w,
870 struct intel_engine_cs *engine,
871 int ringsz)
872 {
873 struct intel_context *ce;
874
875 ce = intel_context_create(engine);
876 if (IS_ERR(ce))
877 return PTR_ERR(ce);
878
879 ce->ring_size = ringsz;
880 w->rq = intel_context_create_request(ce);
881 intel_context_put(ce);
882 if (IS_ERR(w->rq))
883 return PTR_ERR(w->rq);
884
885 w->addr = i915_ggtt_offset(w->vma);
886
887 switch_tl_lock(w->rq, NULL);
888
889 return 0;
890 }
891
check_watcher(struct hwsp_watcher * w,const char * name,bool (* op)(u32 hwsp,u32 seqno))892 static int check_watcher(struct hwsp_watcher *w, const char *name,
893 bool (*op)(u32 hwsp, u32 seqno))
894 {
895 struct i915_request *rq = fetch_and_zero(&w->rq);
896 u32 offset, end;
897 int err;
898
899 GEM_BUG_ON(w->addr - i915_ggtt_offset(w->vma) > w->vma->size);
900
901 i915_request_get(rq);
902 switch_tl_lock(NULL, rq);
903 i915_request_add(rq);
904
905 if (i915_request_wait(rq, 0, HZ) < 0) {
906 err = -ETIME;
907 goto out;
908 }
909
910 err = 0;
911 offset = 0;
912 end = (w->addr - i915_ggtt_offset(w->vma)) / sizeof(*w->map);
913 while (offset < end) {
914 if (!op(w->map[offset + 1], w->map[offset])) {
915 pr_err("Watcher '%s' found HWSP value %x for seqno %x\n",
916 name, w->map[offset + 1], w->map[offset]);
917 err = -EINVAL;
918 }
919
920 offset += 2;
921 }
922
923 out:
924 i915_request_put(rq);
925 return err;
926 }
927
cleanup_watcher(struct hwsp_watcher * w)928 static void cleanup_watcher(struct hwsp_watcher *w)
929 {
930 if (w->rq) {
931 switch_tl_lock(NULL, w->rq);
932
933 i915_request_add(w->rq);
934 }
935
936 i915_vma_unpin_and_release(&w->vma, I915_VMA_RELEASE_MAP);
937 }
938
retire_requests(struct intel_timeline * tl)939 static bool retire_requests(struct intel_timeline *tl)
940 {
941 struct i915_request *rq, *rn;
942
943 mutex_lock(&tl->mutex);
944 list_for_each_entry_safe(rq, rn, &tl->requests, link)
945 if (!i915_request_retire(rq))
946 break;
947 mutex_unlock(&tl->mutex);
948
949 return !i915_active_fence_isset(&tl->last_request);
950 }
951
wrap_timeline(struct i915_request * rq)952 static struct i915_request *wrap_timeline(struct i915_request *rq)
953 {
954 struct intel_context *ce = rq->context;
955 struct intel_timeline *tl = ce->timeline;
956 u32 seqno = rq->fence.seqno;
957
958 while (tl->seqno >= seqno) { /* Cause a wrap */
959 i915_request_put(rq);
960 rq = intel_context_create_request(ce);
961 if (IS_ERR(rq))
962 return rq;
963
964 i915_request_get(rq);
965 i915_request_add(rq);
966 }
967
968 i915_request_put(rq);
969 rq = i915_request_create(ce);
970 if (IS_ERR(rq))
971 return rq;
972
973 i915_request_get(rq);
974 i915_request_add(rq);
975
976 return rq;
977 }
978
live_hwsp_read(void * arg)979 static int live_hwsp_read(void *arg)
980 {
981 struct intel_gt *gt = arg;
982 struct hwsp_watcher watcher[2] = {};
983 struct intel_engine_cs *engine;
984 struct intel_timeline *tl;
985 enum intel_engine_id id;
986 int err = 0;
987 int i;
988
989 /*
990 * If we take a reference to the HWSP for reading on the GPU, that
991 * read may be arbitrarily delayed (either by foreign fence or
992 * priority saturation) and a wrap can happen within 30 minutes.
993 * When the GPU read is finally submitted it should be correct,
994 * even across multiple wraps.
995 */
996
997 if (GRAPHICS_VER(gt->i915) < 8) /* CS convenience [SRM/LRM] */
998 return 0;
999
1000 tl = intel_timeline_create(gt);
1001 if (IS_ERR(tl))
1002 return PTR_ERR(tl);
1003
1004 if (!tl->has_initial_breadcrumb)
1005 goto out_free;
1006
1007 selftest_tl_pin(tl);
1008
1009 for (i = 0; i < ARRAY_SIZE(watcher); i++) {
1010 err = setup_watcher(&watcher[i], gt, tl);
1011 if (err)
1012 goto out;
1013 }
1014
1015 for_each_engine(engine, gt, id) {
1016 struct intel_context *ce;
1017 unsigned long count = 0;
1018 IGT_TIMEOUT(end_time);
1019
1020 /* Create a request we can use for remote reading of the HWSP */
1021 err = create_watcher(&watcher[1], engine, SZ_512K);
1022 if (err)
1023 goto out;
1024
1025 do {
1026 struct i915_sw_fence *submit;
1027 struct i915_request *rq;
1028 u32 hwsp, dummy;
1029
1030 submit = heap_fence_create(GFP_KERNEL);
1031 if (!submit) {
1032 err = -ENOMEM;
1033 goto out;
1034 }
1035
1036 err = create_watcher(&watcher[0], engine, SZ_4K);
1037 if (err)
1038 goto out;
1039
1040 ce = intel_context_create(engine);
1041 if (IS_ERR(ce)) {
1042 err = PTR_ERR(ce);
1043 goto out;
1044 }
1045
1046 ce->timeline = intel_timeline_get(tl);
1047
1048 /* Ensure timeline is mapped, done during first pin */
1049 err = intel_context_pin(ce);
1050 if (err) {
1051 intel_context_put(ce);
1052 goto out;
1053 }
1054
1055 /*
1056 * Start at a new wrap, and set seqno right before another wrap,
1057 * saving 30 minutes of nops
1058 */
1059 tl->seqno = -12u + 2 * (count & 3);
1060 __intel_timeline_get_seqno(tl, &dummy);
1061
1062 rq = i915_request_create(ce);
1063 if (IS_ERR(rq)) {
1064 err = PTR_ERR(rq);
1065 intel_context_unpin(ce);
1066 intel_context_put(ce);
1067 goto out;
1068 }
1069
1070 err = i915_sw_fence_await_dma_fence(&rq->submit,
1071 &watcher[0].rq->fence, 0,
1072 GFP_KERNEL);
1073 if (err < 0) {
1074 i915_request_add(rq);
1075 intel_context_unpin(ce);
1076 intel_context_put(ce);
1077 goto out;
1078 }
1079
1080 switch_tl_lock(rq, watcher[0].rq);
1081 err = intel_timeline_read_hwsp(rq, watcher[0].rq, &hwsp);
1082 if (err == 0)
1083 err = emit_read_hwsp(watcher[0].rq, /* before */
1084 rq->fence.seqno, hwsp,
1085 &watcher[0].addr);
1086 switch_tl_lock(watcher[0].rq, rq);
1087 if (err) {
1088 i915_request_add(rq);
1089 intel_context_unpin(ce);
1090 intel_context_put(ce);
1091 goto out;
1092 }
1093
1094 switch_tl_lock(rq, watcher[1].rq);
1095 err = intel_timeline_read_hwsp(rq, watcher[1].rq, &hwsp);
1096 if (err == 0)
1097 err = emit_read_hwsp(watcher[1].rq, /* after */
1098 rq->fence.seqno, hwsp,
1099 &watcher[1].addr);
1100 switch_tl_lock(watcher[1].rq, rq);
1101 if (err) {
1102 i915_request_add(rq);
1103 intel_context_unpin(ce);
1104 intel_context_put(ce);
1105 goto out;
1106 }
1107
1108 i915_request_get(rq);
1109 i915_request_add(rq);
1110
1111 rq = wrap_timeline(rq);
1112 intel_context_unpin(ce);
1113 intel_context_put(ce);
1114 if (IS_ERR(rq)) {
1115 err = PTR_ERR(rq);
1116 goto out;
1117 }
1118
1119 err = i915_sw_fence_await_dma_fence(&watcher[1].rq->submit,
1120 &rq->fence, 0,
1121 GFP_KERNEL);
1122 if (err < 0) {
1123 i915_request_put(rq);
1124 goto out;
1125 }
1126
1127 err = check_watcher(&watcher[0], "before", cmp_lt);
1128 i915_sw_fence_commit(submit);
1129 heap_fence_put(submit);
1130 if (err) {
1131 i915_request_put(rq);
1132 goto out;
1133 }
1134 count++;
1135
1136 /* Flush the timeline before manually wrapping again */
1137 if (i915_request_wait(rq,
1138 I915_WAIT_INTERRUPTIBLE,
1139 HZ) < 0) {
1140 err = -ETIME;
1141 i915_request_put(rq);
1142 goto out;
1143 }
1144 retire_requests(tl);
1145 i915_request_put(rq);
1146
1147 /* Single requests are limited to half a ring at most */
1148 if (8 * watcher[1].rq->ring->emit >
1149 3 * watcher[1].rq->ring->size)
1150 break;
1151
1152 } while (!__igt_timeout(end_time, NULL) &&
1153 count < (PAGE_SIZE / TIMELINE_SEQNO_BYTES - 1) / 2);
1154
1155 pr_info("%s: simulated %lu wraps\n", engine->name, count);
1156 err = check_watcher(&watcher[1], "after", cmp_gte);
1157 if (err)
1158 goto out;
1159 }
1160
1161 out:
1162 for (i = 0; i < ARRAY_SIZE(watcher); i++)
1163 cleanup_watcher(&watcher[i]);
1164
1165 intel_timeline_unpin(tl);
1166
1167 if (igt_flush_test(gt->i915))
1168 err = -EIO;
1169
1170 out_free:
1171 intel_timeline_put(tl);
1172 return err;
1173 }
1174
live_hwsp_rollover_kernel(void * arg)1175 static int live_hwsp_rollover_kernel(void *arg)
1176 {
1177 struct intel_gt *gt = arg;
1178 struct intel_engine_cs *engine;
1179 enum intel_engine_id id;
1180 int err = 0;
1181
1182 /*
1183 * Run the host for long enough, and even the kernel context will
1184 * see a seqno rollover.
1185 */
1186
1187 for_each_engine(engine, gt, id) {
1188 struct intel_context *ce = engine->kernel_context;
1189 struct intel_timeline *tl = ce->timeline;
1190 struct i915_request *rq[3] = {};
1191 int i;
1192
1193 st_engine_heartbeat_disable(engine);
1194 if (intel_gt_wait_for_idle(gt, HZ / 2)) {
1195 err = -EIO;
1196 goto out;
1197 }
1198
1199 GEM_BUG_ON(i915_active_fence_isset(&tl->last_request));
1200 tl->seqno = -2u;
1201 WRITE_ONCE(*(u32 *)tl->hwsp_seqno, tl->seqno);
1202
1203 for (i = 0; i < ARRAY_SIZE(rq); i++) {
1204 struct i915_request *this;
1205
1206 this = i915_request_create(ce);
1207 if (IS_ERR(this)) {
1208 err = PTR_ERR(this);
1209 goto out;
1210 }
1211
1212 pr_debug("%s: create fence.seqnp:%d\n",
1213 engine->name,
1214 lower_32_bits(this->fence.seqno));
1215
1216 GEM_BUG_ON(rcu_access_pointer(this->timeline) != tl);
1217
1218 rq[i] = i915_request_get(this);
1219 i915_request_add(this);
1220 }
1221
1222 /* We expected a wrap! */
1223 GEM_BUG_ON(rq[2]->fence.seqno > rq[0]->fence.seqno);
1224
1225 if (i915_request_wait(rq[2], 0, HZ / 5) < 0) {
1226 pr_err("Wait for timeline wrap timed out!\n");
1227 err = -EIO;
1228 goto out;
1229 }
1230
1231 for (i = 0; i < ARRAY_SIZE(rq); i++) {
1232 if (!i915_request_completed(rq[i])) {
1233 pr_err("Pre-wrap request not completed!\n");
1234 err = -EINVAL;
1235 goto out;
1236 }
1237 }
1238
1239 out:
1240 for (i = 0; i < ARRAY_SIZE(rq); i++)
1241 i915_request_put(rq[i]);
1242 st_engine_heartbeat_enable(engine);
1243 if (err)
1244 break;
1245 }
1246
1247 if (igt_flush_test(gt->i915))
1248 err = -EIO;
1249
1250 return err;
1251 }
1252
live_hwsp_rollover_user(void * arg)1253 static int live_hwsp_rollover_user(void *arg)
1254 {
1255 struct intel_gt *gt = arg;
1256 struct intel_engine_cs *engine;
1257 enum intel_engine_id id;
1258 int err = 0;
1259
1260 /*
1261 * Simulate a long running user context, and force the seqno wrap
1262 * on the user's timeline.
1263 */
1264
1265 for_each_engine(engine, gt, id) {
1266 struct i915_request *rq[3] = {};
1267 struct intel_timeline *tl;
1268 struct intel_context *ce;
1269 int i;
1270
1271 ce = intel_context_create(engine);
1272 if (IS_ERR(ce))
1273 return PTR_ERR(ce);
1274
1275 err = intel_context_alloc_state(ce);
1276 if (err)
1277 goto out;
1278
1279 tl = ce->timeline;
1280 if (!tl->has_initial_breadcrumb)
1281 goto out;
1282
1283 err = intel_context_pin(ce);
1284 if (err)
1285 goto out;
1286
1287 tl->seqno = -4u;
1288 WRITE_ONCE(*(u32 *)tl->hwsp_seqno, tl->seqno);
1289
1290 for (i = 0; i < ARRAY_SIZE(rq); i++) {
1291 struct i915_request *this;
1292
1293 this = intel_context_create_request(ce);
1294 if (IS_ERR(this)) {
1295 err = PTR_ERR(this);
1296 goto out_unpin;
1297 }
1298
1299 pr_debug("%s: create fence.seqnp:%d\n",
1300 engine->name,
1301 lower_32_bits(this->fence.seqno));
1302
1303 GEM_BUG_ON(rcu_access_pointer(this->timeline) != tl);
1304
1305 rq[i] = i915_request_get(this);
1306 i915_request_add(this);
1307 }
1308
1309 /* We expected a wrap! */
1310 GEM_BUG_ON(rq[2]->fence.seqno > rq[0]->fence.seqno);
1311
1312 if (i915_request_wait(rq[2], 0, HZ / 5) < 0) {
1313 pr_err("Wait for timeline wrap timed out!\n");
1314 err = -EIO;
1315 goto out_unpin;
1316 }
1317
1318 for (i = 0; i < ARRAY_SIZE(rq); i++) {
1319 if (!i915_request_completed(rq[i])) {
1320 pr_err("Pre-wrap request not completed!\n");
1321 err = -EINVAL;
1322 goto out_unpin;
1323 }
1324 }
1325 out_unpin:
1326 intel_context_unpin(ce);
1327 out:
1328 for (i = 0; i < ARRAY_SIZE(rq); i++)
1329 i915_request_put(rq[i]);
1330 intel_context_put(ce);
1331 if (err)
1332 break;
1333 }
1334
1335 if (igt_flush_test(gt->i915))
1336 err = -EIO;
1337
1338 return err;
1339 }
1340
live_hwsp_recycle(void * arg)1341 static int live_hwsp_recycle(void *arg)
1342 {
1343 struct intel_gt *gt = arg;
1344 struct intel_engine_cs *engine;
1345 enum intel_engine_id id;
1346 unsigned long count;
1347 int err = 0;
1348
1349 /*
1350 * Check seqno writes into one timeline at a time. We expect to
1351 * recycle the breadcrumb slot between iterations and neither
1352 * want to confuse ourselves or the GPU.
1353 */
1354
1355 count = 0;
1356 for_each_engine(engine, gt, id) {
1357 IGT_TIMEOUT(end_time);
1358
1359 if (!intel_engine_can_store_dword(engine))
1360 continue;
1361
1362 intel_engine_pm_get(engine);
1363
1364 do {
1365 struct intel_timeline *tl;
1366 struct i915_request *rq;
1367
1368 tl = intel_timeline_create(gt);
1369 if (IS_ERR(tl)) {
1370 err = PTR_ERR(tl);
1371 break;
1372 }
1373
1374 rq = checked_tl_write(tl, engine, count);
1375 if (IS_ERR(rq)) {
1376 intel_timeline_put(tl);
1377 err = PTR_ERR(rq);
1378 break;
1379 }
1380
1381 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
1382 pr_err("Wait for timeline writes timed out!\n");
1383 i915_request_put(rq);
1384 intel_timeline_put(tl);
1385 err = -EIO;
1386 break;
1387 }
1388
1389 if (READ_ONCE(*tl->hwsp_seqno) != count) {
1390 GEM_TRACE_ERR("Invalid seqno:%lu stored in timeline %llu @ %x found 0x%x\n",
1391 count, tl->fence_context,
1392 tl->hwsp_offset, *tl->hwsp_seqno);
1393 GEM_TRACE_DUMP();
1394 err = -EINVAL;
1395 }
1396
1397 i915_request_put(rq);
1398 intel_timeline_put(tl);
1399 count++;
1400
1401 if (err)
1402 break;
1403 } while (!__igt_timeout(end_time, NULL));
1404
1405 intel_engine_pm_put(engine);
1406 if (err)
1407 break;
1408 }
1409
1410 return err;
1411 }
1412
intel_timeline_live_selftests(struct drm_i915_private * i915)1413 int intel_timeline_live_selftests(struct drm_i915_private *i915)
1414 {
1415 static const struct i915_subtest tests[] = {
1416 SUBTEST(live_hwsp_recycle),
1417 SUBTEST(live_hwsp_engine),
1418 SUBTEST(live_hwsp_alternate),
1419 SUBTEST(live_hwsp_wrap),
1420 SUBTEST(live_hwsp_read),
1421 SUBTEST(live_hwsp_rollover_kernel),
1422 SUBTEST(live_hwsp_rollover_user),
1423 };
1424
1425 if (intel_gt_is_wedged(to_gt(i915)))
1426 return 0;
1427
1428 return intel_gt_live_subtests(tests, to_gt(i915));
1429 }
1430