1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2019 Intel Corporation
4 */
5
6 #include <drm/drm_print.h>
7
8 #include "i915_selftest.h"
9
10 #include "display/intel_display_device.h"
11 #include "gt/intel_context.h"
12 #include "gt/intel_engine_regs.h"
13 #include "gt/intel_engine_user.h"
14 #include "gt/intel_gpu_commands.h"
15 #include "gt/intel_gt.h"
16 #include "gt/intel_gt_regs.h"
17 #include "gem/i915_gem_lmem.h"
18
19 #include "gem/selftests/igt_gem_utils.h"
20 #include "selftests/igt_flush_test.h"
21 #include "selftests/mock_drm.h"
22 #include "selftests/i915_random.h"
23 #include "huge_gem_object.h"
24 #include "mock_context.h"
25
26 #define OW_SIZE 16 /* in bytes */
27 #define F_SUBTILE_SIZE 64 /* in bytes */
28 #define F_TILE_WIDTH 128 /* in bytes */
29 #define F_TILE_HEIGHT 32 /* in pixels */
30 #define F_SUBTILE_WIDTH OW_SIZE /* in bytes */
31 #define F_SUBTILE_HEIGHT 4 /* in pixels */
32
linear_x_y_to_ftiled_pos(int x,int y,u32 stride,int bpp)33 static int linear_x_y_to_ftiled_pos(int x, int y, u32 stride, int bpp)
34 {
35 int tile_base;
36 int tile_x, tile_y;
37 int swizzle, subtile;
38 int pixel_size = bpp / 8;
39 int pos;
40
41 /*
42 * Subtile remapping for F tile. Note that map[a]==b implies map[b]==a
43 * so we can use the same table to tile and until.
44 */
45 static const u8 f_subtile_map[] = {
46 0, 1, 2, 3, 8, 9, 10, 11,
47 4, 5, 6, 7, 12, 13, 14, 15,
48 16, 17, 18, 19, 24, 25, 26, 27,
49 20, 21, 22, 23, 28, 29, 30, 31,
50 32, 33, 34, 35, 40, 41, 42, 43,
51 36, 37, 38, 39, 44, 45, 46, 47,
52 48, 49, 50, 51, 56, 57, 58, 59,
53 52, 53, 54, 55, 60, 61, 62, 63
54 };
55
56 x *= pixel_size;
57 /*
58 * Where does the 4k tile start (in bytes)? This is the same for Y and
59 * F so we can use the Y-tile algorithm to get to that point.
60 */
61 tile_base =
62 y / F_TILE_HEIGHT * stride * F_TILE_HEIGHT +
63 x / F_TILE_WIDTH * 4096;
64
65 /* Find pixel within tile */
66 tile_x = x % F_TILE_WIDTH;
67 tile_y = y % F_TILE_HEIGHT;
68
69 /* And figure out the subtile within the 4k tile */
70 subtile = tile_y / F_SUBTILE_HEIGHT * 8 + tile_x / F_SUBTILE_WIDTH;
71
72 /* Swizzle the subtile number according to the bspec diagram */
73 swizzle = f_subtile_map[subtile];
74
75 /* Calculate new position */
76 pos = tile_base +
77 swizzle * F_SUBTILE_SIZE +
78 tile_y % F_SUBTILE_HEIGHT * OW_SIZE +
79 tile_x % F_SUBTILE_WIDTH;
80
81 GEM_BUG_ON(!IS_ALIGNED(pos, pixel_size));
82
83 return pos / pixel_size * 4;
84 }
85
86 enum client_tiling {
87 CLIENT_TILING_LINEAR,
88 CLIENT_TILING_X,
89 CLIENT_TILING_Y, /* Y-major, either Tile4 (Xe_HP and beyond) or legacy TileY */
90 CLIENT_NUM_TILING_TYPES
91 };
92
93 #define WIDTH 512
94 #define HEIGHT 32
95
96 struct blit_buffer {
97 struct i915_vma *vma;
98 u32 start_val;
99 enum client_tiling tiling;
100 };
101
102 struct tiled_blits {
103 struct intel_context *ce;
104 struct blit_buffer buffers[3];
105 struct blit_buffer scratch;
106 struct i915_vma *batch;
107 u64 hole;
108 u64 align;
109 u32 width;
110 u32 height;
111 };
112
fastblit_supports_x_tiling(const struct drm_i915_private * i915)113 static bool fastblit_supports_x_tiling(const struct drm_i915_private *i915)
114 {
115 struct intel_display *display = i915->display;
116 int gen = GRAPHICS_VER(i915);
117
118 /* XY_FAST_COPY_BLT does not exist on pre-gen9 platforms */
119 drm_WARN_ON(&i915->drm, gen < 9);
120
121 if (gen < 12)
122 return true;
123
124 if (GRAPHICS_VER_FULL(i915) < IP_VER(12, 55))
125 return false;
126
127 return intel_display_device_present(display);
128 }
129
fast_blit_ok(const struct blit_buffer * buf)130 static bool fast_blit_ok(const struct blit_buffer *buf)
131 {
132 /* XY_FAST_COPY_BLT does not exist on pre-gen9 platforms */
133 if (GRAPHICS_VER(buf->vma->vm->i915) < 9)
134 return false;
135
136 /* filter out platforms with unsupported X-tile support in fastblit */
137 if (buf->tiling == CLIENT_TILING_X && !fastblit_supports_x_tiling(buf->vma->vm->i915))
138 return false;
139
140 return true;
141 }
142
prepare_blit(const struct tiled_blits * t,struct blit_buffer * dst,struct blit_buffer * src,struct drm_i915_gem_object * batch)143 static int prepare_blit(const struct tiled_blits *t,
144 struct blit_buffer *dst,
145 struct blit_buffer *src,
146 struct drm_i915_gem_object *batch)
147 {
148 const int ver = GRAPHICS_VER(to_i915(batch->base.dev));
149 bool use_64b_reloc = ver >= 8;
150 u32 src_pitch, dst_pitch;
151 u32 cmd, *cs;
152
153 cs = i915_gem_object_pin_map_unlocked(batch, I915_MAP_WC);
154 if (IS_ERR(cs))
155 return PTR_ERR(cs);
156
157 if (fast_blit_ok(dst) && fast_blit_ok(src)) {
158 struct intel_gt *gt = t->ce->engine->gt;
159 u32 src_tiles = 0, dst_tiles = 0;
160 u32 src_4t = 0, dst_4t = 0;
161
162 /* Need to program BLIT_CCTL if it is not done previously
163 * before using XY_FAST_COPY_BLT
164 */
165 *cs++ = MI_LOAD_REGISTER_IMM(1);
166 *cs++ = i915_mmio_reg_offset(BLIT_CCTL(t->ce->engine->mmio_base));
167 *cs++ = (BLIT_CCTL_SRC_MOCS(gt->mocs.uc_index) |
168 BLIT_CCTL_DST_MOCS(gt->mocs.uc_index));
169
170 src_pitch = t->width; /* in dwords */
171 if (src->tiling == CLIENT_TILING_Y) {
172 src_tiles = XY_FAST_COPY_BLT_D0_SRC_TILE_MODE(YMAJOR);
173 if (GRAPHICS_VER_FULL(to_i915(batch->base.dev)) >= IP_VER(12, 55))
174 src_4t = XY_FAST_COPY_BLT_D1_SRC_TILE4;
175 } else if (src->tiling == CLIENT_TILING_X) {
176 src_tiles = XY_FAST_COPY_BLT_D0_SRC_TILE_MODE(TILE_X);
177 } else {
178 src_pitch *= 4; /* in bytes */
179 }
180
181 dst_pitch = t->width; /* in dwords */
182 if (dst->tiling == CLIENT_TILING_Y) {
183 dst_tiles = XY_FAST_COPY_BLT_D0_DST_TILE_MODE(YMAJOR);
184 if (GRAPHICS_VER_FULL(to_i915(batch->base.dev)) >= IP_VER(12, 55))
185 dst_4t = XY_FAST_COPY_BLT_D1_DST_TILE4;
186 } else if (dst->tiling == CLIENT_TILING_X) {
187 dst_tiles = XY_FAST_COPY_BLT_D0_DST_TILE_MODE(TILE_X);
188 } else {
189 dst_pitch *= 4; /* in bytes */
190 }
191
192 *cs++ = GEN9_XY_FAST_COPY_BLT_CMD | (10 - 2) |
193 src_tiles | dst_tiles;
194 *cs++ = src_4t | dst_4t | BLT_DEPTH_32 | dst_pitch;
195 *cs++ = 0;
196 *cs++ = t->height << 16 | t->width;
197 *cs++ = lower_32_bits(i915_vma_offset(dst->vma));
198 *cs++ = upper_32_bits(i915_vma_offset(dst->vma));
199 *cs++ = 0;
200 *cs++ = src_pitch;
201 *cs++ = lower_32_bits(i915_vma_offset(src->vma));
202 *cs++ = upper_32_bits(i915_vma_offset(src->vma));
203 } else {
204 if (ver >= 6) {
205 *cs++ = MI_LOAD_REGISTER_IMM(1);
206 *cs++ = i915_mmio_reg_offset(BCS_SWCTRL);
207 cmd = (BCS_SRC_Y | BCS_DST_Y) << 16;
208 if (src->tiling == CLIENT_TILING_Y)
209 cmd |= BCS_SRC_Y;
210 if (dst->tiling == CLIENT_TILING_Y)
211 cmd |= BCS_DST_Y;
212 *cs++ = cmd;
213
214 cmd = MI_FLUSH_DW;
215 if (ver >= 8)
216 cmd++;
217 *cs++ = cmd;
218 *cs++ = 0;
219 *cs++ = 0;
220 *cs++ = 0;
221 }
222
223 cmd = XY_SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (8 - 2);
224 if (ver >= 8)
225 cmd += 2;
226
227 src_pitch = t->width * 4;
228 if (src->tiling) {
229 cmd |= XY_SRC_COPY_BLT_SRC_TILED;
230 src_pitch /= 4;
231 }
232
233 dst_pitch = t->width * 4;
234 if (dst->tiling) {
235 cmd |= XY_SRC_COPY_BLT_DST_TILED;
236 dst_pitch /= 4;
237 }
238
239 *cs++ = cmd;
240 *cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | dst_pitch;
241 *cs++ = 0;
242 *cs++ = t->height << 16 | t->width;
243 *cs++ = lower_32_bits(i915_vma_offset(dst->vma));
244 if (use_64b_reloc)
245 *cs++ = upper_32_bits(i915_vma_offset(dst->vma));
246 *cs++ = 0;
247 *cs++ = src_pitch;
248 *cs++ = lower_32_bits(i915_vma_offset(src->vma));
249 if (use_64b_reloc)
250 *cs++ = upper_32_bits(i915_vma_offset(src->vma));
251 }
252
253 *cs++ = MI_BATCH_BUFFER_END;
254
255 i915_gem_object_flush_map(batch);
256 i915_gem_object_unpin_map(batch);
257
258 return 0;
259 }
260
tiled_blits_destroy_buffers(struct tiled_blits * t)261 static void tiled_blits_destroy_buffers(struct tiled_blits *t)
262 {
263 int i;
264
265 for (i = 0; i < ARRAY_SIZE(t->buffers); i++)
266 i915_vma_put(t->buffers[i].vma);
267
268 i915_vma_put(t->scratch.vma);
269 i915_vma_put(t->batch);
270 }
271
272 static struct i915_vma *
__create_vma(struct tiled_blits * t,size_t size,bool lmem)273 __create_vma(struct tiled_blits *t, size_t size, bool lmem)
274 {
275 struct drm_i915_private *i915 = t->ce->vm->i915;
276 struct drm_i915_gem_object *obj;
277 struct i915_vma *vma;
278
279 if (lmem)
280 obj = i915_gem_object_create_lmem(i915, size, 0);
281 else
282 obj = i915_gem_object_create_shmem(i915, size);
283 if (IS_ERR(obj))
284 return ERR_CAST(obj);
285
286 vma = i915_vma_instance(obj, t->ce->vm, NULL);
287 if (IS_ERR(vma))
288 i915_gem_object_put(obj);
289
290 return vma;
291 }
292
create_vma(struct tiled_blits * t,bool lmem)293 static struct i915_vma *create_vma(struct tiled_blits *t, bool lmem)
294 {
295 return __create_vma(t, PAGE_ALIGN(t->width * t->height * 4), lmem);
296 }
297
tiled_blits_create_buffers(struct tiled_blits * t,int width,int height,struct rnd_state * prng)298 static int tiled_blits_create_buffers(struct tiled_blits *t,
299 int width, int height,
300 struct rnd_state *prng)
301 {
302 struct drm_i915_private *i915 = t->ce->engine->i915;
303 int i;
304
305 t->width = width;
306 t->height = height;
307
308 t->batch = __create_vma(t, PAGE_SIZE, false);
309 if (IS_ERR(t->batch))
310 return PTR_ERR(t->batch);
311
312 t->scratch.vma = create_vma(t, false);
313 if (IS_ERR(t->scratch.vma)) {
314 i915_vma_put(t->batch);
315 return PTR_ERR(t->scratch.vma);
316 }
317
318 for (i = 0; i < ARRAY_SIZE(t->buffers); i++) {
319 struct i915_vma *vma;
320
321 vma = create_vma(t, HAS_LMEM(i915) && i % 2);
322 if (IS_ERR(vma)) {
323 tiled_blits_destroy_buffers(t);
324 return PTR_ERR(vma);
325 }
326
327 t->buffers[i].vma = vma;
328 t->buffers[i].tiling =
329 i915_prandom_u32_max_state(CLIENT_NUM_TILING_TYPES, prng);
330 }
331
332 return 0;
333 }
334
fill_scratch(struct tiled_blits * t,u32 * vaddr,u32 val)335 static void fill_scratch(struct tiled_blits *t, u32 *vaddr, u32 val)
336 {
337 int i;
338
339 t->scratch.start_val = val;
340 for (i = 0; i < t->width * t->height; i++)
341 vaddr[i] = val++;
342
343 i915_gem_object_flush_map(t->scratch.vma->obj);
344 }
345
swizzle_bit(unsigned int bit,u64 offset)346 static u64 swizzle_bit(unsigned int bit, u64 offset)
347 {
348 return (offset & BIT_ULL(bit)) >> (bit - 6);
349 }
350
tiled_offset(const struct intel_gt * gt,u64 v,unsigned int stride,enum client_tiling tiling,int x_pos,int y_pos)351 static u64 tiled_offset(const struct intel_gt *gt,
352 u64 v,
353 unsigned int stride,
354 enum client_tiling tiling,
355 int x_pos, int y_pos)
356 {
357 unsigned int swizzle;
358 u64 x, y;
359
360 if (tiling == CLIENT_TILING_LINEAR)
361 return v;
362
363 y = div64_u64_rem(v, stride, &x);
364
365 if (tiling == CLIENT_TILING_X) {
366 v = div64_u64_rem(y, 8, &y) * stride * 8;
367 v += y * 512;
368 v += div64_u64_rem(x, 512, &x) << 12;
369 v += x;
370
371 swizzle = gt->ggtt->bit_6_swizzle_x;
372 } else if (GRAPHICS_VER_FULL(gt->i915) >= IP_VER(12, 55)) {
373 /* Y-major tiling layout is Tile4 for Xe_HP and beyond */
374 v = linear_x_y_to_ftiled_pos(x_pos, y_pos, stride, 32);
375
376 /* no swizzling for f-tiling */
377 swizzle = I915_BIT_6_SWIZZLE_NONE;
378 } else {
379 const unsigned int ytile_span = 16;
380 const unsigned int ytile_height = 512;
381
382 v = div64_u64_rem(y, 32, &y) * stride * 32;
383 v += y * ytile_span;
384 v += div64_u64_rem(x, ytile_span, &x) * ytile_height;
385 v += x;
386
387 swizzle = gt->ggtt->bit_6_swizzle_y;
388 }
389
390 switch (swizzle) {
391 case I915_BIT_6_SWIZZLE_9:
392 v ^= swizzle_bit(9, v);
393 break;
394 case I915_BIT_6_SWIZZLE_9_10:
395 v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v);
396 break;
397 case I915_BIT_6_SWIZZLE_9_11:
398 v ^= swizzle_bit(9, v) ^ swizzle_bit(11, v);
399 break;
400 case I915_BIT_6_SWIZZLE_9_10_11:
401 v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v) ^ swizzle_bit(11, v);
402 break;
403 }
404
405 return v;
406 }
407
repr_tiling(enum client_tiling tiling)408 static const char *repr_tiling(enum client_tiling tiling)
409 {
410 switch (tiling) {
411 case CLIENT_TILING_LINEAR: return "linear";
412 case CLIENT_TILING_X: return "X";
413 case CLIENT_TILING_Y: return "Y / 4";
414 default: return "unknown";
415 }
416 }
417
verify_buffer(const struct tiled_blits * t,struct blit_buffer * buf,struct rnd_state * prng)418 static int verify_buffer(const struct tiled_blits *t,
419 struct blit_buffer *buf,
420 struct rnd_state *prng)
421 {
422 const u32 *vaddr;
423 int ret = 0;
424 int x, y, p;
425
426 x = i915_prandom_u32_max_state(t->width, prng);
427 y = i915_prandom_u32_max_state(t->height, prng);
428 p = y * t->width + x;
429
430 vaddr = i915_gem_object_pin_map_unlocked(buf->vma->obj, I915_MAP_WC);
431 if (IS_ERR(vaddr))
432 return PTR_ERR(vaddr);
433
434 if (vaddr[0] != buf->start_val) {
435 ret = -EINVAL;
436 } else {
437 u64 v = tiled_offset(buf->vma->vm->gt,
438 p * 4, t->width * 4,
439 buf->tiling, x, y);
440
441 if (vaddr[v / sizeof(*vaddr)] != buf->start_val + p)
442 ret = -EINVAL;
443 }
444 if (ret) {
445 pr_err("Invalid %s tiling detected at (%d, %d), start_val %x\n",
446 repr_tiling(buf->tiling),
447 x, y, buf->start_val);
448 igt_hexdump(vaddr, 4096);
449 }
450
451 i915_gem_object_unpin_map(buf->vma->obj);
452 return ret;
453 }
454
pin_buffer(struct i915_vma * vma,u64 addr)455 static int pin_buffer(struct i915_vma *vma, u64 addr)
456 {
457 int err;
458
459 if (drm_mm_node_allocated(&vma->node) && i915_vma_offset(vma) != addr) {
460 err = i915_vma_unbind_unlocked(vma);
461 if (err)
462 return err;
463 }
464
465 err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_OFFSET_FIXED | addr);
466 if (err)
467 return err;
468
469 GEM_BUG_ON(i915_vma_offset(vma) != addr);
470 return 0;
471 }
472
473 static int
tiled_blit(struct tiled_blits * t,struct blit_buffer * dst,u64 dst_addr,struct blit_buffer * src,u64 src_addr)474 tiled_blit(struct tiled_blits *t,
475 struct blit_buffer *dst, u64 dst_addr,
476 struct blit_buffer *src, u64 src_addr)
477 {
478 struct i915_request *rq;
479 int err;
480
481 err = pin_buffer(src->vma, src_addr);
482 if (err) {
483 pr_err("Cannot pin src @ %llx\n", src_addr);
484 return err;
485 }
486
487 err = pin_buffer(dst->vma, dst_addr);
488 if (err) {
489 pr_err("Cannot pin dst @ %llx\n", dst_addr);
490 goto err_src;
491 }
492
493 err = i915_vma_pin(t->batch, 0, 0, PIN_USER | PIN_HIGH);
494 if (err) {
495 pr_err("cannot pin batch\n");
496 goto err_dst;
497 }
498
499 err = prepare_blit(t, dst, src, t->batch->obj);
500 if (err)
501 goto err_bb;
502
503 rq = intel_context_create_request(t->ce);
504 if (IS_ERR(rq)) {
505 err = PTR_ERR(rq);
506 goto err_bb;
507 }
508
509 err = igt_vma_move_to_active_unlocked(t->batch, rq, 0);
510 if (!err)
511 err = igt_vma_move_to_active_unlocked(src->vma, rq, 0);
512 if (!err)
513 err = igt_vma_move_to_active_unlocked(dst->vma, rq, 0);
514 if (!err)
515 err = rq->engine->emit_bb_start(rq,
516 i915_vma_offset(t->batch),
517 i915_vma_size(t->batch),
518 0);
519 i915_request_get(rq);
520 i915_request_add(rq);
521 if (i915_request_wait(rq, 0, HZ / 2) < 0)
522 err = -ETIME;
523 i915_request_put(rq);
524
525 dst->start_val = src->start_val;
526 err_bb:
527 i915_vma_unpin(t->batch);
528 err_dst:
529 i915_vma_unpin(dst->vma);
530 err_src:
531 i915_vma_unpin(src->vma);
532 return err;
533 }
534
535 static struct tiled_blits *
tiled_blits_create(struct intel_engine_cs * engine,struct rnd_state * prng)536 tiled_blits_create(struct intel_engine_cs *engine, struct rnd_state *prng)
537 {
538 struct drm_mm_node hole;
539 struct tiled_blits *t;
540 u64 hole_size;
541 int err;
542
543 t = kzalloc_obj(*t);
544 if (!t)
545 return ERR_PTR(-ENOMEM);
546
547 t->ce = intel_context_create(engine);
548 if (IS_ERR(t->ce)) {
549 err = PTR_ERR(t->ce);
550 goto err_free;
551 }
552
553 t->align = i915_vm_min_alignment(t->ce->vm, INTEL_MEMORY_LOCAL);
554 t->align = max(t->align,
555 i915_vm_min_alignment(t->ce->vm, INTEL_MEMORY_SYSTEM));
556
557 hole_size = 2 * round_up(WIDTH * HEIGHT * 4, t->align);
558 hole_size *= 2; /* room to maneuver */
559 hole_size += 2 * t->align; /* padding on either side */
560
561 mutex_lock(&t->ce->vm->mutex);
562 memset(&hole, 0, sizeof(hole));
563 err = drm_mm_insert_node_in_range(&t->ce->vm->mm, &hole,
564 hole_size, t->align,
565 I915_COLOR_UNEVICTABLE,
566 0, U64_MAX,
567 DRM_MM_INSERT_BEST);
568 if (!err)
569 drm_mm_remove_node(&hole);
570 mutex_unlock(&t->ce->vm->mutex);
571 if (err) {
572 err = -ENODEV;
573 goto err_put;
574 }
575
576 t->hole = hole.start + t->align;
577 pr_info("Using hole at %llx\n", t->hole);
578
579 err = tiled_blits_create_buffers(t, WIDTH, HEIGHT, prng);
580 if (err)
581 goto err_put;
582
583 return t;
584
585 err_put:
586 intel_context_put(t->ce);
587 err_free:
588 kfree(t);
589 return ERR_PTR(err);
590 }
591
tiled_blits_destroy(struct tiled_blits * t)592 static void tiled_blits_destroy(struct tiled_blits *t)
593 {
594 tiled_blits_destroy_buffers(t);
595
596 intel_context_put(t->ce);
597 kfree(t);
598 }
599
tiled_blits_prepare(struct tiled_blits * t,struct rnd_state * prng)600 static int tiled_blits_prepare(struct tiled_blits *t,
601 struct rnd_state *prng)
602 {
603 u64 offset = round_up(t->width * t->height * 4, t->align);
604 u32 *map;
605 int err;
606 int i;
607
608 map = i915_gem_object_pin_map_unlocked(t->scratch.vma->obj, I915_MAP_WC);
609 if (IS_ERR(map))
610 return PTR_ERR(map);
611
612 /* Use scratch to fill objects */
613 for (i = 0; i < ARRAY_SIZE(t->buffers); i++) {
614 fill_scratch(t, map, prandom_u32_state(prng));
615 GEM_BUG_ON(verify_buffer(t, &t->scratch, prng));
616
617 err = tiled_blit(t,
618 &t->buffers[i], t->hole + offset,
619 &t->scratch, t->hole);
620 if (err == 0)
621 err = verify_buffer(t, &t->buffers[i], prng);
622 if (err) {
623 pr_err("Failed to create buffer %d\n", i);
624 break;
625 }
626 }
627
628 i915_gem_object_unpin_map(t->scratch.vma->obj);
629 return err;
630 }
631
tiled_blits_bounce(struct tiled_blits * t,struct rnd_state * prng)632 static int tiled_blits_bounce(struct tiled_blits *t, struct rnd_state *prng)
633 {
634 u64 offset = round_up(t->width * t->height * 4, 2 * t->align);
635 int err;
636
637 /* We want to check position invariant tiling across GTT eviction */
638
639 err = tiled_blit(t,
640 &t->buffers[1], t->hole + offset / 2,
641 &t->buffers[0], t->hole + 2 * offset);
642 if (err)
643 return err;
644
645 /* Simulating GTT eviction of the same buffer / layout */
646 t->buffers[2].tiling = t->buffers[0].tiling;
647
648 /* Reposition so that we overlap the old addresses, and slightly off */
649 err = tiled_blit(t,
650 &t->buffers[2], t->hole + t->align,
651 &t->buffers[1], t->hole + 3 * offset / 2);
652 if (err)
653 return err;
654
655 err = verify_buffer(t, &t->buffers[2], prng);
656 if (err)
657 return err;
658
659 return 0;
660 }
661
__igt_client_tiled_blits(struct intel_engine_cs * engine,struct rnd_state * prng)662 static int __igt_client_tiled_blits(struct intel_engine_cs *engine,
663 struct rnd_state *prng)
664 {
665 struct tiled_blits *t;
666 int err;
667
668 t = tiled_blits_create(engine, prng);
669 if (IS_ERR(t))
670 return PTR_ERR(t);
671
672 err = tiled_blits_prepare(t, prng);
673 if (err)
674 goto out;
675
676 err = tiled_blits_bounce(t, prng);
677 if (err)
678 goto out;
679
680 out:
681 tiled_blits_destroy(t);
682 return err;
683 }
684
has_bit17_swizzle(int sw)685 static bool has_bit17_swizzle(int sw)
686 {
687 return (sw == I915_BIT_6_SWIZZLE_9_10_17 ||
688 sw == I915_BIT_6_SWIZZLE_9_17);
689 }
690
bad_swizzling(struct drm_i915_private * i915)691 static bool bad_swizzling(struct drm_i915_private *i915)
692 {
693 struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
694
695 if (i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES)
696 return true;
697
698 if (has_bit17_swizzle(ggtt->bit_6_swizzle_x) ||
699 has_bit17_swizzle(ggtt->bit_6_swizzle_y))
700 return true;
701
702 return false;
703 }
704
igt_client_tiled_blits(void * arg)705 static int igt_client_tiled_blits(void *arg)
706 {
707 struct drm_i915_private *i915 = arg;
708 I915_RND_STATE(prng);
709 int inst = 0;
710
711 /* Test requires explicit BLT tiling controls */
712 if (GRAPHICS_VER(i915) < 4)
713 return 0;
714
715 if (bad_swizzling(i915)) /* Requires sane (sub-page) swizzling */
716 return 0;
717
718 do {
719 struct intel_engine_cs *engine;
720 int err;
721
722 engine = intel_engine_lookup_user(i915,
723 I915_ENGINE_CLASS_COPY,
724 inst++);
725 if (!engine)
726 return 0;
727
728 err = __igt_client_tiled_blits(engine, &prng);
729 if (err == -ENODEV)
730 err = 0;
731 if (err)
732 return err;
733 } while (1);
734 }
735
i915_gem_client_blt_live_selftests(struct drm_i915_private * i915)736 int i915_gem_client_blt_live_selftests(struct drm_i915_private *i915)
737 {
738 static const struct i915_subtest tests[] = {
739 SUBTEST(igt_client_tiled_blits),
740 };
741
742 if (intel_gt_is_wedged(to_gt(i915)))
743 return 0;
744
745 return i915_live_subtests(tests, i915);
746 }
747