1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2021 Intel Corporation
4 */
5
6 #include <drm/ttm/ttm_bo.h>
7
8 #include "intel_display_types.h"
9 #include "intel_dpt.h"
10 #include "intel_fb.h"
11 #include "intel_fb_pin.h"
12 #include "xe_bo.h"
13 #include "xe_device.h"
14 #include "xe_ggtt.h"
15 #include "xe_pm.h"
16
17 static void
write_dpt_rotated(struct xe_bo * bo,struct iosys_map * map,u32 * dpt_ofs,u32 bo_ofs,u32 width,u32 height,u32 src_stride,u32 dst_stride)18 write_dpt_rotated(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, u32 bo_ofs,
19 u32 width, u32 height, u32 src_stride, u32 dst_stride)
20 {
21 struct xe_device *xe = xe_bo_device(bo);
22 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt;
23 u32 column, row;
24
25 /* TODO: Maybe rewrite so we can traverse the bo addresses sequentially,
26 * by writing dpt/ggtt in a different order?
27 */
28
29 for (column = 0; column < width; column++) {
30 u32 src_idx = src_stride * (height - 1) + column + bo_ofs;
31
32 for (row = 0; row < height; row++) {
33 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, src_idx * XE_PAGE_SIZE,
34 xe->pat.idx[XE_CACHE_NONE]);
35
36 iosys_map_wr(map, *dpt_ofs, u64, pte);
37 *dpt_ofs += 8;
38 src_idx -= src_stride;
39 }
40
41 /* The DE ignores the PTEs for the padding tiles */
42 *dpt_ofs += (dst_stride - height) * 8;
43 }
44
45 /* Align to next page */
46 *dpt_ofs = ALIGN(*dpt_ofs, 4096);
47 }
48
49 static void
write_dpt_remapped(struct xe_bo * bo,struct iosys_map * map,u32 * dpt_ofs,u32 bo_ofs,u32 width,u32 height,u32 src_stride,u32 dst_stride)50 write_dpt_remapped(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs,
51 u32 bo_ofs, u32 width, u32 height, u32 src_stride,
52 u32 dst_stride)
53 {
54 struct xe_device *xe = xe_bo_device(bo);
55 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt;
56 u64 (*pte_encode_bo)(struct xe_bo *bo, u64 bo_offset, u16 pat_index)
57 = ggtt->pt_ops->pte_encode_bo;
58 u32 column, row;
59
60 for (row = 0; row < height; row++) {
61 u32 src_idx = src_stride * row + bo_ofs;
62
63 for (column = 0; column < width; column++) {
64 iosys_map_wr(map, *dpt_ofs, u64,
65 pte_encode_bo(bo, src_idx * XE_PAGE_SIZE,
66 xe->pat.idx[XE_CACHE_NONE]));
67
68 *dpt_ofs += 8;
69 src_idx++;
70 }
71
72 /* The DE ignores the PTEs for the padding tiles */
73 *dpt_ofs += (dst_stride - width) * 8;
74 }
75
76 /* Align to next page */
77 *dpt_ofs = ALIGN(*dpt_ofs, 4096);
78 }
79
__xe_pin_fb_vma_dpt(const struct intel_framebuffer * fb,const struct i915_gtt_view * view,struct i915_vma * vma,u64 physical_alignment)80 static int __xe_pin_fb_vma_dpt(const struct intel_framebuffer *fb,
81 const struct i915_gtt_view *view,
82 struct i915_vma *vma,
83 u64 physical_alignment)
84 {
85 struct xe_device *xe = to_xe_device(fb->base.dev);
86 struct xe_tile *tile0 = xe_device_get_root_tile(xe);
87 struct xe_ggtt *ggtt = tile0->mem.ggtt;
88 struct drm_gem_object *obj = intel_fb_bo(&fb->base);
89 struct xe_bo *bo = gem_to_xe_bo(obj), *dpt;
90 u32 dpt_size, size = bo->ttm.base.size;
91
92 if (view->type == I915_GTT_VIEW_NORMAL)
93 dpt_size = ALIGN(size / XE_PAGE_SIZE * 8, XE_PAGE_SIZE);
94 else if (view->type == I915_GTT_VIEW_REMAPPED)
95 dpt_size = ALIGN(intel_remapped_info_size(&fb->remapped_view.gtt.remapped) * 8,
96 XE_PAGE_SIZE);
97 else
98 /* display uses 4K tiles instead of bytes here, convert to entries.. */
99 dpt_size = ALIGN(intel_rotation_info_size(&view->rotated) * 8,
100 XE_PAGE_SIZE);
101
102 if (IS_DGFX(xe))
103 dpt = xe_bo_create_pin_map_at_aligned(xe, tile0, NULL,
104 dpt_size, ~0ull,
105 ttm_bo_type_kernel,
106 XE_BO_FLAG_VRAM0 |
107 XE_BO_FLAG_GGTT |
108 XE_BO_FLAG_PAGETABLE,
109 physical_alignment);
110 else
111 dpt = xe_bo_create_pin_map_at_aligned(xe, tile0, NULL,
112 dpt_size, ~0ull,
113 ttm_bo_type_kernel,
114 XE_BO_FLAG_STOLEN |
115 XE_BO_FLAG_GGTT |
116 XE_BO_FLAG_PAGETABLE,
117 physical_alignment);
118 if (IS_ERR(dpt))
119 dpt = xe_bo_create_pin_map_at_aligned(xe, tile0, NULL,
120 dpt_size, ~0ull,
121 ttm_bo_type_kernel,
122 XE_BO_FLAG_SYSTEM |
123 XE_BO_FLAG_GGTT |
124 XE_BO_FLAG_PAGETABLE,
125 physical_alignment);
126 if (IS_ERR(dpt))
127 return PTR_ERR(dpt);
128
129 if (view->type == I915_GTT_VIEW_NORMAL) {
130 u32 x;
131
132 for (x = 0; x < size / XE_PAGE_SIZE; x++) {
133 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, x * XE_PAGE_SIZE,
134 xe->pat.idx[XE_CACHE_NONE]);
135
136 iosys_map_wr(&dpt->vmap, x * 8, u64, pte);
137 }
138 } else if (view->type == I915_GTT_VIEW_REMAPPED) {
139 const struct intel_remapped_info *remap_info = &view->remapped;
140 u32 i, dpt_ofs = 0;
141
142 for (i = 0; i < ARRAY_SIZE(remap_info->plane); i++)
143 write_dpt_remapped(bo, &dpt->vmap, &dpt_ofs,
144 remap_info->plane[i].offset,
145 remap_info->plane[i].width,
146 remap_info->plane[i].height,
147 remap_info->plane[i].src_stride,
148 remap_info->plane[i].dst_stride);
149
150 } else {
151 const struct intel_rotation_info *rot_info = &view->rotated;
152 u32 i, dpt_ofs = 0;
153
154 for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++)
155 write_dpt_rotated(bo, &dpt->vmap, &dpt_ofs,
156 rot_info->plane[i].offset,
157 rot_info->plane[i].width,
158 rot_info->plane[i].height,
159 rot_info->plane[i].src_stride,
160 rot_info->plane[i].dst_stride);
161 }
162
163 vma->dpt = dpt;
164 vma->node = dpt->ggtt_node;
165 return 0;
166 }
167
168 static void
write_ggtt_rotated(struct xe_bo * bo,struct xe_ggtt * ggtt,u32 * ggtt_ofs,u32 bo_ofs,u32 width,u32 height,u32 src_stride,u32 dst_stride)169 write_ggtt_rotated(struct xe_bo *bo, struct xe_ggtt *ggtt, u32 *ggtt_ofs, u32 bo_ofs,
170 u32 width, u32 height, u32 src_stride, u32 dst_stride)
171 {
172 struct xe_device *xe = xe_bo_device(bo);
173 u32 column, row;
174
175 for (column = 0; column < width; column++) {
176 u32 src_idx = src_stride * (height - 1) + column + bo_ofs;
177
178 for (row = 0; row < height; row++) {
179 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, src_idx * XE_PAGE_SIZE,
180 xe->pat.idx[XE_CACHE_NONE]);
181
182 ggtt->pt_ops->ggtt_set_pte(ggtt, *ggtt_ofs, pte);
183 *ggtt_ofs += XE_PAGE_SIZE;
184 src_idx -= src_stride;
185 }
186
187 /* The DE ignores the PTEs for the padding tiles */
188 *ggtt_ofs += (dst_stride - height) * XE_PAGE_SIZE;
189 }
190 }
191
__xe_pin_fb_vma_ggtt(const struct intel_framebuffer * fb,const struct i915_gtt_view * view,struct i915_vma * vma,u64 physical_alignment)192 static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb,
193 const struct i915_gtt_view *view,
194 struct i915_vma *vma,
195 u64 physical_alignment)
196 {
197 struct drm_gem_object *obj = intel_fb_bo(&fb->base);
198 struct xe_bo *bo = gem_to_xe_bo(obj);
199 struct xe_device *xe = to_xe_device(fb->base.dev);
200 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt;
201 u32 align;
202 int ret;
203
204 /* TODO: Consider sharing framebuffer mapping?
205 * embed i915_vma inside intel_framebuffer
206 */
207 xe_pm_runtime_get_noresume(tile_to_xe(ggtt->tile));
208 ret = mutex_lock_interruptible(&ggtt->lock);
209 if (ret)
210 goto out;
211
212 align = XE_PAGE_SIZE;
213 if (xe_bo_is_vram(bo) && ggtt->flags & XE_GGTT_FLAGS_64K)
214 align = max_t(u32, align, SZ_64K);
215
216 if (bo->ggtt_node && view->type == I915_GTT_VIEW_NORMAL) {
217 vma->node = bo->ggtt_node;
218 } else if (view->type == I915_GTT_VIEW_NORMAL) {
219 u32 x, size = bo->ttm.base.size;
220
221 vma->node = xe_ggtt_node_init(ggtt);
222 if (IS_ERR(vma->node)) {
223 ret = PTR_ERR(vma->node);
224 goto out_unlock;
225 }
226
227 ret = xe_ggtt_node_insert_locked(vma->node, size, align, 0);
228 if (ret) {
229 xe_ggtt_node_fini(vma->node);
230 goto out_unlock;
231 }
232
233 for (x = 0; x < size; x += XE_PAGE_SIZE) {
234 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, x,
235 xe->pat.idx[XE_CACHE_NONE]);
236
237 ggtt->pt_ops->ggtt_set_pte(ggtt, vma->node->base.start + x, pte);
238 }
239 } else {
240 u32 i, ggtt_ofs;
241 const struct intel_rotation_info *rot_info = &view->rotated;
242
243 /* display seems to use tiles instead of bytes here, so convert it back.. */
244 u32 size = intel_rotation_info_size(rot_info) * XE_PAGE_SIZE;
245
246 vma->node = xe_ggtt_node_init(ggtt);
247 if (IS_ERR(vma->node)) {
248 ret = PTR_ERR(vma->node);
249 goto out_unlock;
250 }
251
252 ret = xe_ggtt_node_insert_locked(vma->node, size, align, 0);
253 if (ret) {
254 xe_ggtt_node_fini(vma->node);
255 goto out_unlock;
256 }
257
258 ggtt_ofs = vma->node->base.start;
259
260 for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++)
261 write_ggtt_rotated(bo, ggtt, &ggtt_ofs,
262 rot_info->plane[i].offset,
263 rot_info->plane[i].width,
264 rot_info->plane[i].height,
265 rot_info->plane[i].src_stride,
266 rot_info->plane[i].dst_stride);
267 }
268
269 out_unlock:
270 mutex_unlock(&ggtt->lock);
271 out:
272 xe_pm_runtime_put(tile_to_xe(ggtt->tile));
273 return ret;
274 }
275
__xe_pin_fb_vma(const struct intel_framebuffer * fb,const struct i915_gtt_view * view,u64 physical_alignment)276 static struct i915_vma *__xe_pin_fb_vma(const struct intel_framebuffer *fb,
277 const struct i915_gtt_view *view,
278 u64 physical_alignment)
279 {
280 struct drm_device *dev = fb->base.dev;
281 struct xe_device *xe = to_xe_device(dev);
282 struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
283 struct drm_gem_object *obj = intel_fb_bo(&fb->base);
284 struct xe_bo *bo = gem_to_xe_bo(obj);
285 int ret;
286
287 if (!vma)
288 return ERR_PTR(-ENODEV);
289
290 if (IS_DGFX(to_xe_device(bo->ttm.base.dev)) &&
291 intel_fb_rc_ccs_cc_plane(&fb->base) >= 0 &&
292 !(bo->flags & XE_BO_FLAG_NEEDS_CPU_ACCESS)) {
293 struct xe_tile *tile = xe_device_get_root_tile(xe);
294
295 /*
296 * If we need to able to access the clear-color value stored in
297 * the buffer, then we require that such buffers are also CPU
298 * accessible. This is important on small-bar systems where
299 * only some subset of VRAM is CPU accessible.
300 */
301 if (tile->mem.vram.io_size < tile->mem.vram.usable_size) {
302 ret = -EINVAL;
303 goto err;
304 }
305 }
306
307 /*
308 * Pin the framebuffer, we can't use xe_bo_(un)pin functions as the
309 * assumptions are incorrect for framebuffers
310 */
311 ret = ttm_bo_reserve(&bo->ttm, false, false, NULL);
312 if (ret)
313 goto err;
314
315 if (IS_DGFX(xe))
316 ret = xe_bo_migrate(bo, XE_PL_VRAM0);
317 else
318 ret = xe_bo_validate(bo, NULL, true);
319 if (!ret)
320 ttm_bo_pin(&bo->ttm);
321 ttm_bo_unreserve(&bo->ttm);
322 if (ret)
323 goto err;
324
325 vma->bo = bo;
326 if (intel_fb_uses_dpt(&fb->base))
327 ret = __xe_pin_fb_vma_dpt(fb, view, vma, physical_alignment);
328 else
329 ret = __xe_pin_fb_vma_ggtt(fb, view, vma, physical_alignment);
330 if (ret)
331 goto err_unpin;
332
333 /* Ensure DPT writes are flushed */
334 xe_device_l2_flush(xe);
335 return vma;
336
337 err_unpin:
338 ttm_bo_reserve(&bo->ttm, false, false, NULL);
339 ttm_bo_unpin(&bo->ttm);
340 ttm_bo_unreserve(&bo->ttm);
341 err:
342 kfree(vma);
343 return ERR_PTR(ret);
344 }
345
__xe_unpin_fb_vma(struct i915_vma * vma)346 static void __xe_unpin_fb_vma(struct i915_vma *vma)
347 {
348 if (vma->dpt)
349 xe_bo_unpin_map_no_vm(vma->dpt);
350 else if (!xe_ggtt_node_allocated(vma->bo->ggtt_node) ||
351 vma->bo->ggtt_node->base.start != vma->node->base.start)
352 xe_ggtt_node_remove(vma->node, false);
353
354 ttm_bo_reserve(&vma->bo->ttm, false, false, NULL);
355 ttm_bo_unpin(&vma->bo->ttm);
356 ttm_bo_unreserve(&vma->bo->ttm);
357 kfree(vma);
358 }
359
360 struct i915_vma *
intel_fb_pin_to_ggtt(const struct drm_framebuffer * fb,const struct i915_gtt_view * view,unsigned int alignment,unsigned int phys_alignment,bool uses_fence,unsigned long * out_flags)361 intel_fb_pin_to_ggtt(const struct drm_framebuffer *fb,
362 const struct i915_gtt_view *view,
363 unsigned int alignment,
364 unsigned int phys_alignment,
365 bool uses_fence,
366 unsigned long *out_flags)
367 {
368 *out_flags = 0;
369
370 return __xe_pin_fb_vma(to_intel_framebuffer(fb), view, phys_alignment);
371 }
372
intel_fb_unpin_vma(struct i915_vma * vma,unsigned long flags)373 void intel_fb_unpin_vma(struct i915_vma *vma, unsigned long flags)
374 {
375 __xe_unpin_fb_vma(vma);
376 }
377
intel_plane_pin_fb(struct intel_plane_state * plane_state)378 int intel_plane_pin_fb(struct intel_plane_state *plane_state)
379 {
380 struct drm_framebuffer *fb = plane_state->hw.fb;
381 struct drm_gem_object *obj = intel_fb_bo(fb);
382 struct xe_bo *bo = gem_to_xe_bo(obj);
383 struct i915_vma *vma;
384 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
385 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
386 u64 phys_alignment = plane->min_alignment(plane, fb, 0);
387
388 /* We reject creating !SCANOUT fb's, so this is weird.. */
389 drm_WARN_ON(bo->ttm.base.dev, !(bo->flags & XE_BO_FLAG_SCANOUT));
390
391 vma = __xe_pin_fb_vma(intel_fb, &plane_state->view.gtt, phys_alignment);
392
393 if (IS_ERR(vma))
394 return PTR_ERR(vma);
395
396 plane_state->ggtt_vma = vma;
397 return 0;
398 }
399
intel_plane_unpin_fb(struct intel_plane_state * old_plane_state)400 void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
401 {
402 __xe_unpin_fb_vma(old_plane_state->ggtt_vma);
403 old_plane_state->ggtt_vma = NULL;
404 }
405
406 /*
407 * For Xe introduce dummy intel_dpt_create which just return NULL,
408 * intel_dpt_destroy which does nothing, and fake intel_dpt_ofsset returning 0;
409 */
intel_dpt_create(struct intel_framebuffer * fb)410 struct i915_address_space *intel_dpt_create(struct intel_framebuffer *fb)
411 {
412 return NULL;
413 }
414
intel_dpt_destroy(struct i915_address_space * vm)415 void intel_dpt_destroy(struct i915_address_space *vm)
416 {
417 return;
418 }
419
intel_dpt_offset(struct i915_vma * dpt_vma)420 u64 intel_dpt_offset(struct i915_vma *dpt_vma)
421 {
422 return 0;
423 }
424