1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2021 Intel Corporation
4 */
5
6 #include <drm/ttm/ttm_bo.h>
7
8 #include "i915_vma.h"
9 #include "intel_display_core.h"
10 #include "intel_display_types.h"
11 #include "intel_dpt.h"
12 #include "intel_fb.h"
13 #include "intel_fb_pin.h"
14 #include "intel_fbdev.h"
15 #include "xe_bo.h"
16 #include "xe_device.h"
17 #include "xe_ggtt.h"
18 #include "xe_pm.h"
19
20 static void
write_dpt_rotated(struct xe_bo * bo,struct iosys_map * map,u32 * dpt_ofs,u32 bo_ofs,u32 width,u32 height,u32 src_stride,u32 dst_stride)21 write_dpt_rotated(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, u32 bo_ofs,
22 u32 width, u32 height, u32 src_stride, u32 dst_stride)
23 {
24 struct xe_device *xe = xe_bo_device(bo);
25 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt;
26 u32 column, row;
27 u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe->pat.idx[XE_CACHE_NONE]);
28
29 /* TODO: Maybe rewrite so we can traverse the bo addresses sequentially,
30 * by writing dpt/ggtt in a different order?
31 */
32
33 for (column = 0; column < width; column++) {
34 u32 src_idx = src_stride * (height - 1) + column + bo_ofs;
35
36 for (row = 0; row < height; row++) {
37 u64 addr = xe_bo_addr(bo, src_idx * XE_PAGE_SIZE, XE_PAGE_SIZE);
38
39 iosys_map_wr(map, *dpt_ofs, u64, pte | addr);
40 *dpt_ofs += 8;
41 src_idx -= src_stride;
42 }
43
44 /* The DE ignores the PTEs for the padding tiles */
45 *dpt_ofs += (dst_stride - height) * 8;
46 }
47
48 /* Align to next page */
49 *dpt_ofs = ALIGN(*dpt_ofs, 4096);
50 }
51
52 static void
write_dpt_remapped(struct xe_bo * bo,struct iosys_map * map,u32 * dpt_ofs,u32 bo_ofs,u32 width,u32 height,u32 src_stride,u32 dst_stride)53 write_dpt_remapped(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs,
54 u32 bo_ofs, u32 width, u32 height, u32 src_stride,
55 u32 dst_stride)
56 {
57 struct xe_device *xe = xe_bo_device(bo);
58 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt;
59 u32 column, row;
60 u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe->pat.idx[XE_CACHE_NONE]);
61
62 for (row = 0; row < height; row++) {
63 u32 src_idx = src_stride * row + bo_ofs;
64
65 for (column = 0; column < width; column++) {
66 u64 addr = xe_bo_addr(bo, src_idx * XE_PAGE_SIZE, XE_PAGE_SIZE);
67 iosys_map_wr(map, *dpt_ofs, u64, pte | addr);
68
69 *dpt_ofs += 8;
70 src_idx++;
71 }
72
73 /* The DE ignores the PTEs for the padding tiles */
74 *dpt_ofs += (dst_stride - width) * 8;
75 }
76
77 /* Align to next page */
78 *dpt_ofs = ALIGN(*dpt_ofs, 4096);
79 }
80
__xe_pin_fb_vma_dpt(const struct intel_framebuffer * fb,const struct i915_gtt_view * view,struct i915_vma * vma,unsigned int alignment)81 static int __xe_pin_fb_vma_dpt(const struct intel_framebuffer *fb,
82 const struct i915_gtt_view *view,
83 struct i915_vma *vma,
84 unsigned int alignment)
85 {
86 struct xe_device *xe = to_xe_device(fb->base.dev);
87 struct xe_tile *tile0 = xe_device_get_root_tile(xe);
88 struct xe_ggtt *ggtt = tile0->mem.ggtt;
89 struct drm_gem_object *obj = intel_fb_bo(&fb->base);
90 struct xe_bo *bo = gem_to_xe_bo(obj), *dpt;
91 u32 dpt_size, size = bo->ttm.base.size;
92
93 if (view->type == I915_GTT_VIEW_NORMAL)
94 dpt_size = ALIGN(size / XE_PAGE_SIZE * 8, XE_PAGE_SIZE);
95 else if (view->type == I915_GTT_VIEW_REMAPPED)
96 dpt_size = ALIGN(intel_remapped_info_size(&fb->remapped_view.gtt.remapped) * 8,
97 XE_PAGE_SIZE);
98 else
99 /* display uses 4K tiles instead of bytes here, convert to entries.. */
100 dpt_size = ALIGN(intel_rotation_info_size(&view->rotated) * 8,
101 XE_PAGE_SIZE);
102
103 if (IS_DGFX(xe))
104 dpt = xe_bo_create_pin_map_at_aligned(xe, tile0, NULL,
105 dpt_size, ~0ull,
106 ttm_bo_type_kernel,
107 XE_BO_FLAG_VRAM0 |
108 XE_BO_FLAG_GGTT |
109 XE_BO_FLAG_PAGETABLE,
110 alignment);
111 else
112 dpt = xe_bo_create_pin_map_at_aligned(xe, tile0, NULL,
113 dpt_size, ~0ull,
114 ttm_bo_type_kernel,
115 XE_BO_FLAG_STOLEN |
116 XE_BO_FLAG_GGTT |
117 XE_BO_FLAG_PAGETABLE,
118 alignment);
119 if (IS_ERR(dpt))
120 dpt = xe_bo_create_pin_map_at_aligned(xe, tile0, NULL,
121 dpt_size, ~0ull,
122 ttm_bo_type_kernel,
123 XE_BO_FLAG_SYSTEM |
124 XE_BO_FLAG_GGTT |
125 XE_BO_FLAG_PAGETABLE,
126 alignment);
127 if (IS_ERR(dpt))
128 return PTR_ERR(dpt);
129
130 if (view->type == I915_GTT_VIEW_NORMAL) {
131 u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe->pat.idx[XE_CACHE_NONE]);
132 u32 x;
133
134 for (x = 0; x < size / XE_PAGE_SIZE; x++) {
135 u64 addr = xe_bo_addr(bo, x * XE_PAGE_SIZE, XE_PAGE_SIZE);
136
137 iosys_map_wr(&dpt->vmap, x * 8, u64, pte | addr);
138 }
139 } else if (view->type == I915_GTT_VIEW_REMAPPED) {
140 const struct intel_remapped_info *remap_info = &view->remapped;
141 u32 i, dpt_ofs = 0;
142
143 for (i = 0; i < ARRAY_SIZE(remap_info->plane); i++)
144 write_dpt_remapped(bo, &dpt->vmap, &dpt_ofs,
145 remap_info->plane[i].offset,
146 remap_info->plane[i].width,
147 remap_info->plane[i].height,
148 remap_info->plane[i].src_stride,
149 remap_info->plane[i].dst_stride);
150
151 } else {
152 const struct intel_rotation_info *rot_info = &view->rotated;
153 u32 i, dpt_ofs = 0;
154
155 for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++)
156 write_dpt_rotated(bo, &dpt->vmap, &dpt_ofs,
157 rot_info->plane[i].offset,
158 rot_info->plane[i].width,
159 rot_info->plane[i].height,
160 rot_info->plane[i].src_stride,
161 rot_info->plane[i].dst_stride);
162 }
163
164 vma->dpt = dpt;
165 vma->node = dpt->ggtt_node[tile0->id];
166
167 /* Ensure DPT writes are flushed */
168 xe_device_l2_flush(xe);
169 return 0;
170 }
171
172 static void
write_ggtt_rotated(struct xe_bo * bo,struct xe_ggtt * ggtt,u32 * ggtt_ofs,u32 bo_ofs,u32 width,u32 height,u32 src_stride,u32 dst_stride)173 write_ggtt_rotated(struct xe_bo *bo, struct xe_ggtt *ggtt, u32 *ggtt_ofs, u32 bo_ofs,
174 u32 width, u32 height, u32 src_stride, u32 dst_stride)
175 {
176 struct xe_device *xe = xe_bo_device(bo);
177 u32 column, row;
178 u64 pte = ggtt->pt_ops->pte_encode_flags(bo, xe->pat.idx[XE_CACHE_NONE]);
179
180 for (column = 0; column < width; column++) {
181 u32 src_idx = src_stride * (height - 1) + column + bo_ofs;
182
183 for (row = 0; row < height; row++) {
184 u64 addr = xe_bo_addr(bo, src_idx * XE_PAGE_SIZE, XE_PAGE_SIZE);
185
186 ggtt->pt_ops->ggtt_set_pte(ggtt, *ggtt_ofs, pte | addr);
187 *ggtt_ofs += XE_PAGE_SIZE;
188 src_idx -= src_stride;
189 }
190
191 /* The DE ignores the PTEs for the padding tiles */
192 *ggtt_ofs += (dst_stride - height) * XE_PAGE_SIZE;
193 }
194 }
195
__xe_pin_fb_vma_ggtt(const struct intel_framebuffer * fb,const struct i915_gtt_view * view,struct i915_vma * vma,unsigned int alignment)196 static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb,
197 const struct i915_gtt_view *view,
198 struct i915_vma *vma,
199 unsigned int alignment)
200 {
201 struct drm_gem_object *obj = intel_fb_bo(&fb->base);
202 struct xe_bo *bo = gem_to_xe_bo(obj);
203 struct xe_device *xe = to_xe_device(fb->base.dev);
204 struct xe_tile *tile0 = xe_device_get_root_tile(xe);
205 struct xe_ggtt *ggtt = tile0->mem.ggtt;
206 u32 align;
207 int ret;
208
209 /* TODO: Consider sharing framebuffer mapping?
210 * embed i915_vma inside intel_framebuffer
211 */
212 xe_pm_runtime_get_noresume(xe);
213 ret = mutex_lock_interruptible(&ggtt->lock);
214 if (ret)
215 goto out;
216
217 align = XE_PAGE_SIZE;
218 if (xe_bo_is_vram(bo) && ggtt->flags & XE_GGTT_FLAGS_64K)
219 align = max_t(u32, align, SZ_64K);
220
221 if (bo->ggtt_node[tile0->id] && view->type == I915_GTT_VIEW_NORMAL) {
222 vma->node = bo->ggtt_node[tile0->id];
223 } else if (view->type == I915_GTT_VIEW_NORMAL) {
224 vma->node = xe_ggtt_node_init(ggtt);
225 if (IS_ERR(vma->node)) {
226 ret = PTR_ERR(vma->node);
227 goto out_unlock;
228 }
229
230 ret = xe_ggtt_node_insert_locked(vma->node, xe_bo_size(bo), align, 0);
231 if (ret) {
232 xe_ggtt_node_fini(vma->node);
233 goto out_unlock;
234 }
235
236 xe_ggtt_map_bo(ggtt, vma->node, bo, xe->pat.idx[XE_CACHE_NONE]);
237 } else {
238 u32 i, ggtt_ofs;
239 const struct intel_rotation_info *rot_info = &view->rotated;
240
241 /* display seems to use tiles instead of bytes here, so convert it back.. */
242 u32 size = intel_rotation_info_size(rot_info) * XE_PAGE_SIZE;
243
244 vma->node = xe_ggtt_node_init(ggtt);
245 if (IS_ERR(vma->node)) {
246 ret = PTR_ERR(vma->node);
247 goto out_unlock;
248 }
249
250 ret = xe_ggtt_node_insert_locked(vma->node, size, align, 0);
251 if (ret) {
252 xe_ggtt_node_fini(vma->node);
253 goto out_unlock;
254 }
255
256 ggtt_ofs = vma->node->base.start;
257
258 for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++)
259 write_ggtt_rotated(bo, ggtt, &ggtt_ofs,
260 rot_info->plane[i].offset,
261 rot_info->plane[i].width,
262 rot_info->plane[i].height,
263 rot_info->plane[i].src_stride,
264 rot_info->plane[i].dst_stride);
265 }
266
267 out_unlock:
268 mutex_unlock(&ggtt->lock);
269 out:
270 xe_pm_runtime_put(xe);
271 return ret;
272 }
273
__xe_pin_fb_vma(const struct intel_framebuffer * fb,const struct i915_gtt_view * view,unsigned int alignment)274 static struct i915_vma *__xe_pin_fb_vma(const struct intel_framebuffer *fb,
275 const struct i915_gtt_view *view,
276 unsigned int alignment)
277 {
278 struct drm_device *dev = fb->base.dev;
279 struct xe_device *xe = to_xe_device(dev);
280 struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
281 struct drm_gem_object *obj = intel_fb_bo(&fb->base);
282 struct xe_bo *bo = gem_to_xe_bo(obj);
283 int ret;
284
285 if (!vma)
286 return ERR_PTR(-ENODEV);
287
288 refcount_set(&vma->ref, 1);
289 if (IS_DGFX(to_xe_device(bo->ttm.base.dev)) &&
290 intel_fb_rc_ccs_cc_plane(&fb->base) >= 0 &&
291 !(bo->flags & XE_BO_FLAG_NEEDS_CPU_ACCESS)) {
292 struct xe_tile *tile = xe_device_get_root_tile(xe);
293
294 /*
295 * If we need to able to access the clear-color value stored in
296 * the buffer, then we require that such buffers are also CPU
297 * accessible. This is important on small-bar systems where
298 * only some subset of VRAM is CPU accessible.
299 */
300 if (tile->mem.vram.io_size < tile->mem.vram.usable_size) {
301 ret = -EINVAL;
302 goto err;
303 }
304 }
305
306 /*
307 * Pin the framebuffer, we can't use xe_bo_(un)pin functions as the
308 * assumptions are incorrect for framebuffers
309 */
310 ret = ttm_bo_reserve(&bo->ttm, false, false, NULL);
311 if (ret)
312 goto err;
313
314 if (IS_DGFX(xe))
315 ret = xe_bo_migrate(bo, XE_PL_VRAM0);
316 else
317 ret = xe_bo_validate(bo, NULL, true);
318 if (!ret)
319 ttm_bo_pin(&bo->ttm);
320 ttm_bo_unreserve(&bo->ttm);
321 if (ret)
322 goto err;
323
324 vma->bo = bo;
325 if (intel_fb_uses_dpt(&fb->base))
326 ret = __xe_pin_fb_vma_dpt(fb, view, vma, alignment);
327 else
328 ret = __xe_pin_fb_vma_ggtt(fb, view, vma, alignment);
329 if (ret)
330 goto err_unpin;
331
332 return vma;
333
334 err_unpin:
335 ttm_bo_reserve(&bo->ttm, false, false, NULL);
336 ttm_bo_unpin(&bo->ttm);
337 ttm_bo_unreserve(&bo->ttm);
338 err:
339 kfree(vma);
340 return ERR_PTR(ret);
341 }
342
__xe_unpin_fb_vma(struct i915_vma * vma)343 static void __xe_unpin_fb_vma(struct i915_vma *vma)
344 {
345 u8 tile_id = xe_device_get_root_tile(xe_bo_device(vma->bo))->id;
346
347 if (!refcount_dec_and_test(&vma->ref))
348 return;
349
350 if (vma->dpt)
351 xe_bo_unpin_map_no_vm(vma->dpt);
352 else if (!xe_ggtt_node_allocated(vma->bo->ggtt_node[tile_id]) ||
353 vma->bo->ggtt_node[tile_id]->base.start != vma->node->base.start)
354 xe_ggtt_node_remove(vma->node, false);
355
356 ttm_bo_reserve(&vma->bo->ttm, false, false, NULL);
357 ttm_bo_unpin(&vma->bo->ttm);
358 ttm_bo_unreserve(&vma->bo->ttm);
359 kfree(vma);
360 }
361
362 struct i915_vma *
intel_fb_pin_to_ggtt(const struct drm_framebuffer * fb,const struct i915_gtt_view * view,unsigned int alignment,unsigned int phys_alignment,unsigned int vtd_guard,bool uses_fence,unsigned long * out_flags)363 intel_fb_pin_to_ggtt(const struct drm_framebuffer *fb,
364 const struct i915_gtt_view *view,
365 unsigned int alignment,
366 unsigned int phys_alignment,
367 unsigned int vtd_guard,
368 bool uses_fence,
369 unsigned long *out_flags)
370 {
371 *out_flags = 0;
372
373 return __xe_pin_fb_vma(to_intel_framebuffer(fb), view, phys_alignment);
374 }
375
intel_fb_unpin_vma(struct i915_vma * vma,unsigned long flags)376 void intel_fb_unpin_vma(struct i915_vma *vma, unsigned long flags)
377 {
378 __xe_unpin_fb_vma(vma);
379 }
380
reuse_vma(struct intel_plane_state * new_plane_state,const struct intel_plane_state * old_plane_state)381 static bool reuse_vma(struct intel_plane_state *new_plane_state,
382 const struct intel_plane_state *old_plane_state)
383 {
384 struct intel_framebuffer *fb = to_intel_framebuffer(new_plane_state->hw.fb);
385 struct xe_device *xe = to_xe_device(fb->base.dev);
386 struct intel_display *display = xe->display;
387 struct i915_vma *vma;
388
389 if (old_plane_state->hw.fb == new_plane_state->hw.fb &&
390 !memcmp(&old_plane_state->view.gtt,
391 &new_plane_state->view.gtt,
392 sizeof(new_plane_state->view.gtt))) {
393 vma = old_plane_state->ggtt_vma;
394 goto found;
395 }
396
397 if (fb == intel_fbdev_framebuffer(display->fbdev.fbdev)) {
398 vma = intel_fbdev_vma_pointer(display->fbdev.fbdev);
399 if (vma)
400 goto found;
401 }
402
403 return false;
404
405 found:
406 refcount_inc(&vma->ref);
407 new_plane_state->ggtt_vma = vma;
408 return true;
409 }
410
intel_plane_pin_fb(struct intel_plane_state * new_plane_state,const struct intel_plane_state * old_plane_state)411 int intel_plane_pin_fb(struct intel_plane_state *new_plane_state,
412 const struct intel_plane_state *old_plane_state)
413 {
414 struct drm_framebuffer *fb = new_plane_state->hw.fb;
415 struct drm_gem_object *obj = intel_fb_bo(fb);
416 struct xe_bo *bo = gem_to_xe_bo(obj);
417 struct i915_vma *vma;
418 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
419 struct intel_plane *plane = to_intel_plane(new_plane_state->uapi.plane);
420 unsigned int alignment = plane->min_alignment(plane, fb, 0);
421
422 if (reuse_vma(new_plane_state, old_plane_state))
423 return 0;
424
425 /* We reject creating !SCANOUT fb's, so this is weird.. */
426 drm_WARN_ON(bo->ttm.base.dev, !(bo->flags & XE_BO_FLAG_SCANOUT));
427
428 vma = __xe_pin_fb_vma(intel_fb, &new_plane_state->view.gtt, alignment);
429
430 if (IS_ERR(vma))
431 return PTR_ERR(vma);
432
433 new_plane_state->ggtt_vma = vma;
434 return 0;
435 }
436
intel_plane_unpin_fb(struct intel_plane_state * old_plane_state)437 void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
438 {
439 __xe_unpin_fb_vma(old_plane_state->ggtt_vma);
440 old_plane_state->ggtt_vma = NULL;
441 }
442
443 /*
444 * For Xe introduce dummy intel_dpt_create which just return NULL,
445 * intel_dpt_destroy which does nothing, and fake intel_dpt_ofsset returning 0;
446 */
intel_dpt_create(struct intel_framebuffer * fb)447 struct i915_address_space *intel_dpt_create(struct intel_framebuffer *fb)
448 {
449 return NULL;
450 }
451
intel_dpt_destroy(struct i915_address_space * vm)452 void intel_dpt_destroy(struct i915_address_space *vm)
453 {
454 return;
455 }
456
intel_dpt_offset(struct i915_vma * dpt_vma)457 u64 intel_dpt_offset(struct i915_vma *dpt_vma)
458 {
459 return 0;
460 }
461
intel_fb_get_map(struct i915_vma * vma,struct iosys_map * map)462 void intel_fb_get_map(struct i915_vma *vma, struct iosys_map *map)
463 {
464 *map = vma->bo->vmap;
465 }
466