1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2021 Intel Corporation
4 */
5
6 #include <drm/ttm/ttm_bo.h>
7
8 #include "i915_vma.h"
9 #include "intel_display_types.h"
10 #include "intel_dpt.h"
11 #include "intel_fb.h"
12 #include "intel_fb_pin.h"
13 #include "intel_fbdev.h"
14 #include "xe_bo.h"
15 #include "xe_device.h"
16 #include "xe_ggtt.h"
17 #include "xe_pm.h"
18
19 static void
write_dpt_rotated(struct xe_bo * bo,struct iosys_map * map,u32 * dpt_ofs,u32 bo_ofs,u32 width,u32 height,u32 src_stride,u32 dst_stride)20 write_dpt_rotated(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, u32 bo_ofs,
21 u32 width, u32 height, u32 src_stride, u32 dst_stride)
22 {
23 struct xe_device *xe = xe_bo_device(bo);
24 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt;
25 u32 column, row;
26
27 /* TODO: Maybe rewrite so we can traverse the bo addresses sequentially,
28 * by writing dpt/ggtt in a different order?
29 */
30
31 for (column = 0; column < width; column++) {
32 u32 src_idx = src_stride * (height - 1) + column + bo_ofs;
33
34 for (row = 0; row < height; row++) {
35 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, src_idx * XE_PAGE_SIZE,
36 xe->pat.idx[XE_CACHE_NONE]);
37
38 iosys_map_wr(map, *dpt_ofs, u64, pte);
39 *dpt_ofs += 8;
40 src_idx -= src_stride;
41 }
42
43 /* The DE ignores the PTEs for the padding tiles */
44 *dpt_ofs += (dst_stride - height) * 8;
45 }
46
47 /* Align to next page */
48 *dpt_ofs = ALIGN(*dpt_ofs, 4096);
49 }
50
51 static void
write_dpt_remapped(struct xe_bo * bo,struct iosys_map * map,u32 * dpt_ofs,u32 bo_ofs,u32 width,u32 height,u32 src_stride,u32 dst_stride)52 write_dpt_remapped(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs,
53 u32 bo_ofs, u32 width, u32 height, u32 src_stride,
54 u32 dst_stride)
55 {
56 struct xe_device *xe = xe_bo_device(bo);
57 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt;
58 u64 (*pte_encode_bo)(struct xe_bo *bo, u64 bo_offset, u16 pat_index)
59 = ggtt->pt_ops->pte_encode_bo;
60 u32 column, row;
61
62 for (row = 0; row < height; row++) {
63 u32 src_idx = src_stride * row + bo_ofs;
64
65 for (column = 0; column < width; column++) {
66 iosys_map_wr(map, *dpt_ofs, u64,
67 pte_encode_bo(bo, src_idx * XE_PAGE_SIZE,
68 xe->pat.idx[XE_CACHE_NONE]));
69
70 *dpt_ofs += 8;
71 src_idx++;
72 }
73
74 /* The DE ignores the PTEs for the padding tiles */
75 *dpt_ofs += (dst_stride - width) * 8;
76 }
77
78 /* Align to next page */
79 *dpt_ofs = ALIGN(*dpt_ofs, 4096);
80 }
81
__xe_pin_fb_vma_dpt(const struct intel_framebuffer * fb,const struct i915_gtt_view * view,struct i915_vma * vma,unsigned int alignment)82 static int __xe_pin_fb_vma_dpt(const struct intel_framebuffer *fb,
83 const struct i915_gtt_view *view,
84 struct i915_vma *vma,
85 unsigned int alignment)
86 {
87 struct xe_device *xe = to_xe_device(fb->base.dev);
88 struct xe_tile *tile0 = xe_device_get_root_tile(xe);
89 struct xe_ggtt *ggtt = tile0->mem.ggtt;
90 struct drm_gem_object *obj = intel_fb_bo(&fb->base);
91 struct xe_bo *bo = gem_to_xe_bo(obj), *dpt;
92 u32 dpt_size, size = bo->ttm.base.size;
93
94 if (view->type == I915_GTT_VIEW_NORMAL)
95 dpt_size = ALIGN(size / XE_PAGE_SIZE * 8, XE_PAGE_SIZE);
96 else if (view->type == I915_GTT_VIEW_REMAPPED)
97 dpt_size = ALIGN(intel_remapped_info_size(&fb->remapped_view.gtt.remapped) * 8,
98 XE_PAGE_SIZE);
99 else
100 /* display uses 4K tiles instead of bytes here, convert to entries.. */
101 dpt_size = ALIGN(intel_rotation_info_size(&view->rotated) * 8,
102 XE_PAGE_SIZE);
103
104 if (IS_DGFX(xe))
105 dpt = xe_bo_create_pin_map_at_aligned(xe, tile0, NULL,
106 dpt_size, ~0ull,
107 ttm_bo_type_kernel,
108 XE_BO_FLAG_VRAM0 |
109 XE_BO_FLAG_GGTT |
110 XE_BO_FLAG_PAGETABLE,
111 alignment);
112 else
113 dpt = xe_bo_create_pin_map_at_aligned(xe, tile0, NULL,
114 dpt_size, ~0ull,
115 ttm_bo_type_kernel,
116 XE_BO_FLAG_STOLEN |
117 XE_BO_FLAG_GGTT |
118 XE_BO_FLAG_PAGETABLE,
119 alignment);
120 if (IS_ERR(dpt))
121 dpt = xe_bo_create_pin_map_at_aligned(xe, tile0, NULL,
122 dpt_size, ~0ull,
123 ttm_bo_type_kernel,
124 XE_BO_FLAG_SYSTEM |
125 XE_BO_FLAG_GGTT |
126 XE_BO_FLAG_PAGETABLE,
127 alignment);
128 if (IS_ERR(dpt))
129 return PTR_ERR(dpt);
130
131 if (view->type == I915_GTT_VIEW_NORMAL) {
132 u32 x;
133
134 for (x = 0; x < size / XE_PAGE_SIZE; x++) {
135 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, x * XE_PAGE_SIZE,
136 xe->pat.idx[XE_CACHE_NONE]);
137
138 iosys_map_wr(&dpt->vmap, x * 8, u64, pte);
139 }
140 } else if (view->type == I915_GTT_VIEW_REMAPPED) {
141 const struct intel_remapped_info *remap_info = &view->remapped;
142 u32 i, dpt_ofs = 0;
143
144 for (i = 0; i < ARRAY_SIZE(remap_info->plane); i++)
145 write_dpt_remapped(bo, &dpt->vmap, &dpt_ofs,
146 remap_info->plane[i].offset,
147 remap_info->plane[i].width,
148 remap_info->plane[i].height,
149 remap_info->plane[i].src_stride,
150 remap_info->plane[i].dst_stride);
151
152 } else {
153 const struct intel_rotation_info *rot_info = &view->rotated;
154 u32 i, dpt_ofs = 0;
155
156 for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++)
157 write_dpt_rotated(bo, &dpt->vmap, &dpt_ofs,
158 rot_info->plane[i].offset,
159 rot_info->plane[i].width,
160 rot_info->plane[i].height,
161 rot_info->plane[i].src_stride,
162 rot_info->plane[i].dst_stride);
163 }
164
165 vma->dpt = dpt;
166 vma->node = dpt->ggtt_node[tile0->id];
167
168 /* Ensure DPT writes are flushed */
169 xe_device_l2_flush(xe);
170 return 0;
171 }
172
173 static void
write_ggtt_rotated(struct xe_bo * bo,struct xe_ggtt * ggtt,u32 * ggtt_ofs,u32 bo_ofs,u32 width,u32 height,u32 src_stride,u32 dst_stride)174 write_ggtt_rotated(struct xe_bo *bo, struct xe_ggtt *ggtt, u32 *ggtt_ofs, u32 bo_ofs,
175 u32 width, u32 height, u32 src_stride, u32 dst_stride)
176 {
177 struct xe_device *xe = xe_bo_device(bo);
178 u32 column, row;
179
180 for (column = 0; column < width; column++) {
181 u32 src_idx = src_stride * (height - 1) + column + bo_ofs;
182
183 for (row = 0; row < height; row++) {
184 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, src_idx * XE_PAGE_SIZE,
185 xe->pat.idx[XE_CACHE_NONE]);
186
187 ggtt->pt_ops->ggtt_set_pte(ggtt, *ggtt_ofs, pte);
188 *ggtt_ofs += XE_PAGE_SIZE;
189 src_idx -= src_stride;
190 }
191
192 /* The DE ignores the PTEs for the padding tiles */
193 *ggtt_ofs += (dst_stride - height) * XE_PAGE_SIZE;
194 }
195 }
196
__xe_pin_fb_vma_ggtt(const struct intel_framebuffer * fb,const struct i915_gtt_view * view,struct i915_vma * vma,unsigned int alignment)197 static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb,
198 const struct i915_gtt_view *view,
199 struct i915_vma *vma,
200 unsigned int alignment)
201 {
202 struct drm_gem_object *obj = intel_fb_bo(&fb->base);
203 struct xe_bo *bo = gem_to_xe_bo(obj);
204 struct xe_device *xe = to_xe_device(fb->base.dev);
205 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt;
206 u32 align;
207 int ret;
208
209 /* TODO: Consider sharing framebuffer mapping?
210 * embed i915_vma inside intel_framebuffer
211 */
212 xe_pm_runtime_get_noresume(tile_to_xe(ggtt->tile));
213 ret = mutex_lock_interruptible(&ggtt->lock);
214 if (ret)
215 goto out;
216
217 align = XE_PAGE_SIZE;
218 if (xe_bo_is_vram(bo) && ggtt->flags & XE_GGTT_FLAGS_64K)
219 align = max_t(u32, align, SZ_64K);
220
221 if (bo->ggtt_node[ggtt->tile->id] && view->type == I915_GTT_VIEW_NORMAL) {
222 vma->node = bo->ggtt_node[ggtt->tile->id];
223 } else if (view->type == I915_GTT_VIEW_NORMAL) {
224 u32 x, size = bo->ttm.base.size;
225
226 vma->node = xe_ggtt_node_init(ggtt);
227 if (IS_ERR(vma->node)) {
228 ret = PTR_ERR(vma->node);
229 goto out_unlock;
230 }
231
232 ret = xe_ggtt_node_insert_locked(vma->node, size, align, 0);
233 if (ret) {
234 xe_ggtt_node_fini(vma->node);
235 goto out_unlock;
236 }
237
238 for (x = 0; x < size; x += XE_PAGE_SIZE) {
239 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, x,
240 xe->pat.idx[XE_CACHE_NONE]);
241
242 ggtt->pt_ops->ggtt_set_pte(ggtt, vma->node->base.start + x, pte);
243 }
244 } else {
245 u32 i, ggtt_ofs;
246 const struct intel_rotation_info *rot_info = &view->rotated;
247
248 /* display seems to use tiles instead of bytes here, so convert it back.. */
249 u32 size = intel_rotation_info_size(rot_info) * XE_PAGE_SIZE;
250
251 vma->node = xe_ggtt_node_init(ggtt);
252 if (IS_ERR(vma->node)) {
253 ret = PTR_ERR(vma->node);
254 goto out_unlock;
255 }
256
257 ret = xe_ggtt_node_insert_locked(vma->node, size, align, 0);
258 if (ret) {
259 xe_ggtt_node_fini(vma->node);
260 goto out_unlock;
261 }
262
263 ggtt_ofs = vma->node->base.start;
264
265 for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++)
266 write_ggtt_rotated(bo, ggtt, &ggtt_ofs,
267 rot_info->plane[i].offset,
268 rot_info->plane[i].width,
269 rot_info->plane[i].height,
270 rot_info->plane[i].src_stride,
271 rot_info->plane[i].dst_stride);
272 }
273
274 out_unlock:
275 mutex_unlock(&ggtt->lock);
276 out:
277 xe_pm_runtime_put(tile_to_xe(ggtt->tile));
278 return ret;
279 }
280
__xe_pin_fb_vma(const struct intel_framebuffer * fb,const struct i915_gtt_view * view,unsigned int alignment)281 static struct i915_vma *__xe_pin_fb_vma(const struct intel_framebuffer *fb,
282 const struct i915_gtt_view *view,
283 unsigned int alignment)
284 {
285 struct drm_device *dev = fb->base.dev;
286 struct xe_device *xe = to_xe_device(dev);
287 struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
288 struct drm_gem_object *obj = intel_fb_bo(&fb->base);
289 struct xe_bo *bo = gem_to_xe_bo(obj);
290 int ret;
291
292 if (!vma)
293 return ERR_PTR(-ENODEV);
294
295 refcount_set(&vma->ref, 1);
296 if (IS_DGFX(to_xe_device(bo->ttm.base.dev)) &&
297 intel_fb_rc_ccs_cc_plane(&fb->base) >= 0 &&
298 !(bo->flags & XE_BO_FLAG_NEEDS_CPU_ACCESS)) {
299 struct xe_tile *tile = xe_device_get_root_tile(xe);
300
301 /*
302 * If we need to able to access the clear-color value stored in
303 * the buffer, then we require that such buffers are also CPU
304 * accessible. This is important on small-bar systems where
305 * only some subset of VRAM is CPU accessible.
306 */
307 if (tile->mem.vram.io_size < tile->mem.vram.usable_size) {
308 ret = -EINVAL;
309 goto err;
310 }
311 }
312
313 /*
314 * Pin the framebuffer, we can't use xe_bo_(un)pin functions as the
315 * assumptions are incorrect for framebuffers
316 */
317 ret = ttm_bo_reserve(&bo->ttm, false, false, NULL);
318 if (ret)
319 goto err;
320
321 if (IS_DGFX(xe))
322 ret = xe_bo_migrate(bo, XE_PL_VRAM0);
323 else
324 ret = xe_bo_validate(bo, NULL, true);
325 if (!ret)
326 ttm_bo_pin(&bo->ttm);
327 ttm_bo_unreserve(&bo->ttm);
328 if (ret)
329 goto err;
330
331 vma->bo = bo;
332 if (intel_fb_uses_dpt(&fb->base))
333 ret = __xe_pin_fb_vma_dpt(fb, view, vma, alignment);
334 else
335 ret = __xe_pin_fb_vma_ggtt(fb, view, vma, alignment);
336 if (ret)
337 goto err_unpin;
338
339 return vma;
340
341 err_unpin:
342 ttm_bo_reserve(&bo->ttm, false, false, NULL);
343 ttm_bo_unpin(&bo->ttm);
344 ttm_bo_unreserve(&bo->ttm);
345 err:
346 kfree(vma);
347 return ERR_PTR(ret);
348 }
349
__xe_unpin_fb_vma(struct i915_vma * vma)350 static void __xe_unpin_fb_vma(struct i915_vma *vma)
351 {
352 u8 tile_id = vma->node->ggtt->tile->id;
353
354 if (!refcount_dec_and_test(&vma->ref))
355 return;
356
357 if (vma->dpt)
358 xe_bo_unpin_map_no_vm(vma->dpt);
359 else if (!xe_ggtt_node_allocated(vma->bo->ggtt_node[tile_id]) ||
360 vma->bo->ggtt_node[tile_id]->base.start != vma->node->base.start)
361 xe_ggtt_node_remove(vma->node, false);
362
363 ttm_bo_reserve(&vma->bo->ttm, false, false, NULL);
364 ttm_bo_unpin(&vma->bo->ttm);
365 ttm_bo_unreserve(&vma->bo->ttm);
366 kfree(vma);
367 }
368
369 struct i915_vma *
intel_fb_pin_to_ggtt(const struct drm_framebuffer * fb,const struct i915_gtt_view * view,unsigned int alignment,unsigned int phys_alignment,unsigned int vtd_guard,bool uses_fence,unsigned long * out_flags)370 intel_fb_pin_to_ggtt(const struct drm_framebuffer *fb,
371 const struct i915_gtt_view *view,
372 unsigned int alignment,
373 unsigned int phys_alignment,
374 unsigned int vtd_guard,
375 bool uses_fence,
376 unsigned long *out_flags)
377 {
378 *out_flags = 0;
379
380 return __xe_pin_fb_vma(to_intel_framebuffer(fb), view, phys_alignment);
381 }
382
intel_fb_unpin_vma(struct i915_vma * vma,unsigned long flags)383 void intel_fb_unpin_vma(struct i915_vma *vma, unsigned long flags)
384 {
385 __xe_unpin_fb_vma(vma);
386 }
387
reuse_vma(struct intel_plane_state * new_plane_state,const struct intel_plane_state * old_plane_state)388 static bool reuse_vma(struct intel_plane_state *new_plane_state,
389 const struct intel_plane_state *old_plane_state)
390 {
391 struct intel_framebuffer *fb = to_intel_framebuffer(new_plane_state->hw.fb);
392 struct xe_device *xe = to_xe_device(fb->base.dev);
393 struct i915_vma *vma;
394
395 if (old_plane_state->hw.fb == new_plane_state->hw.fb &&
396 !memcmp(&old_plane_state->view.gtt,
397 &new_plane_state->view.gtt,
398 sizeof(new_plane_state->view.gtt))) {
399 vma = old_plane_state->ggtt_vma;
400 goto found;
401 }
402
403 if (fb == intel_fbdev_framebuffer(xe->display.fbdev.fbdev)) {
404 vma = intel_fbdev_vma_pointer(xe->display.fbdev.fbdev);
405 if (vma)
406 goto found;
407 }
408
409 return false;
410
411 found:
412 refcount_inc(&vma->ref);
413 new_plane_state->ggtt_vma = vma;
414 return true;
415 }
416
intel_plane_pin_fb(struct intel_plane_state * new_plane_state,const struct intel_plane_state * old_plane_state)417 int intel_plane_pin_fb(struct intel_plane_state *new_plane_state,
418 const struct intel_plane_state *old_plane_state)
419 {
420 struct drm_framebuffer *fb = new_plane_state->hw.fb;
421 struct drm_gem_object *obj = intel_fb_bo(fb);
422 struct xe_bo *bo = gem_to_xe_bo(obj);
423 struct i915_vma *vma;
424 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
425 struct intel_plane *plane = to_intel_plane(new_plane_state->uapi.plane);
426 unsigned int alignment = plane->min_alignment(plane, fb, 0);
427
428 if (reuse_vma(new_plane_state, old_plane_state))
429 return 0;
430
431 /* We reject creating !SCANOUT fb's, so this is weird.. */
432 drm_WARN_ON(bo->ttm.base.dev, !(bo->flags & XE_BO_FLAG_SCANOUT));
433
434 vma = __xe_pin_fb_vma(intel_fb, &new_plane_state->view.gtt, alignment);
435
436 if (IS_ERR(vma))
437 return PTR_ERR(vma);
438
439 new_plane_state->ggtt_vma = vma;
440 return 0;
441 }
442
intel_plane_unpin_fb(struct intel_plane_state * old_plane_state)443 void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
444 {
445 __xe_unpin_fb_vma(old_plane_state->ggtt_vma);
446 old_plane_state->ggtt_vma = NULL;
447 }
448
449 /*
450 * For Xe introduce dummy intel_dpt_create which just return NULL,
451 * intel_dpt_destroy which does nothing, and fake intel_dpt_ofsset returning 0;
452 */
intel_dpt_create(struct intel_framebuffer * fb)453 struct i915_address_space *intel_dpt_create(struct intel_framebuffer *fb)
454 {
455 return NULL;
456 }
457
intel_dpt_destroy(struct i915_address_space * vm)458 void intel_dpt_destroy(struct i915_address_space *vm)
459 {
460 return;
461 }
462
intel_dpt_offset(struct i915_vma * dpt_vma)463 u64 intel_dpt_offset(struct i915_vma *dpt_vma)
464 {
465 return 0;
466 }
467