1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include <drm/ttm/ttm_bo.h> 7 8 #include "intel_display_core.h" 9 #include "intel_display_types.h" 10 #include "intel_fb.h" 11 #include "intel_fb_pin.h" 12 #include "intel_fbdev.h" 13 #include "xe_bo.h" 14 #include "xe_device.h" 15 #include "xe_display_vma.h" 16 #include "xe_ggtt.h" 17 #include "xe_pm.h" 18 #include "xe_vram_types.h" 19 20 static void 21 write_dpt_rotated(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, u32 bo_ofs, 22 u32 width, u32 height, u32 src_stride, u32 dst_stride) 23 { 24 struct xe_device *xe = xe_bo_device(bo); 25 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 26 u32 column, row; 27 u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe->pat.idx[XE_CACHE_NONE]); 28 29 /* TODO: Maybe rewrite so we can traverse the bo addresses sequentially, 30 * by writing dpt/ggtt in a different order? 31 */ 32 33 for (column = 0; column < width; column++) { 34 u32 src_idx = src_stride * (height - 1) + column + bo_ofs; 35 36 for (row = 0; row < height; row++) { 37 u64 addr = xe_bo_addr(bo, src_idx * XE_PAGE_SIZE, XE_PAGE_SIZE); 38 39 iosys_map_wr(map, *dpt_ofs, u64, pte | addr); 40 *dpt_ofs += 8; 41 src_idx -= src_stride; 42 } 43 44 /* The DE ignores the PTEs for the padding tiles */ 45 *dpt_ofs += (dst_stride - height) * 8; 46 } 47 48 /* Align to next page */ 49 *dpt_ofs = ALIGN(*dpt_ofs, 4096); 50 } 51 52 static void 53 write_dpt_remapped(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, 54 u32 bo_ofs, u32 width, u32 height, u32 src_stride, 55 u32 dst_stride) 56 { 57 struct xe_device *xe = xe_bo_device(bo); 58 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 59 u32 column, row; 60 u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe->pat.idx[XE_CACHE_NONE]); 61 62 for (row = 0; row < height; row++) { 63 u32 src_idx = src_stride * row + bo_ofs; 64 65 for (column = 0; column < width; column++) { 66 u64 addr = xe_bo_addr(bo, src_idx * XE_PAGE_SIZE, XE_PAGE_SIZE); 67 iosys_map_wr(map, *dpt_ofs, u64, pte | addr); 68 69 *dpt_ofs += 8; 70 src_idx++; 71 } 72 73 /* The DE ignores the PTEs for the padding tiles */ 74 *dpt_ofs += (dst_stride - width) * 8; 75 } 76 77 /* Align to next page */ 78 *dpt_ofs = ALIGN(*dpt_ofs, 4096); 79 } 80 81 static int __xe_pin_fb_vma_dpt(const struct intel_framebuffer *fb, 82 const struct i915_gtt_view *view, 83 struct i915_vma *vma, 84 unsigned int alignment) 85 { 86 struct xe_device *xe = to_xe_device(fb->base.dev); 87 struct xe_tile *tile0 = xe_device_get_root_tile(xe); 88 struct xe_ggtt *ggtt = tile0->mem.ggtt; 89 struct drm_gem_object *obj = intel_fb_bo(&fb->base); 90 struct xe_bo *bo = gem_to_xe_bo(obj), *dpt; 91 u32 dpt_size, size = bo->ttm.base.size; 92 93 if (view->type == I915_GTT_VIEW_NORMAL) 94 dpt_size = ALIGN(size / XE_PAGE_SIZE * 8, XE_PAGE_SIZE); 95 else if (view->type == I915_GTT_VIEW_REMAPPED) 96 dpt_size = ALIGN(intel_remapped_info_size(&fb->remapped_view.gtt.remapped) * 8, 97 XE_PAGE_SIZE); 98 else 99 /* display uses 4K tiles instead of bytes here, convert to entries.. */ 100 dpt_size = ALIGN(intel_rotation_info_size(&view->rotated) * 8, 101 XE_PAGE_SIZE); 102 103 if (IS_DGFX(xe)) 104 dpt = xe_bo_create_pin_map_at_novm(xe, tile0, 105 dpt_size, ~0ull, 106 ttm_bo_type_kernel, 107 XE_BO_FLAG_VRAM0 | 108 XE_BO_FLAG_GGTT | 109 XE_BO_FLAG_PAGETABLE, 110 alignment, false); 111 else 112 dpt = xe_bo_create_pin_map_at_novm(xe, tile0, 113 dpt_size, ~0ull, 114 ttm_bo_type_kernel, 115 XE_BO_FLAG_STOLEN | 116 XE_BO_FLAG_GGTT | 117 XE_BO_FLAG_PAGETABLE, 118 alignment, false); 119 if (IS_ERR(dpt)) 120 dpt = xe_bo_create_pin_map_at_novm(xe, tile0, 121 dpt_size, ~0ull, 122 ttm_bo_type_kernel, 123 XE_BO_FLAG_SYSTEM | 124 XE_BO_FLAG_GGTT | 125 XE_BO_FLAG_PAGETABLE, 126 alignment, false); 127 if (IS_ERR(dpt)) 128 return PTR_ERR(dpt); 129 130 if (view->type == I915_GTT_VIEW_NORMAL) { 131 u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe->pat.idx[XE_CACHE_NONE]); 132 u32 x; 133 134 for (x = 0; x < size / XE_PAGE_SIZE; x++) { 135 u64 addr = xe_bo_addr(bo, x * XE_PAGE_SIZE, XE_PAGE_SIZE); 136 137 iosys_map_wr(&dpt->vmap, x * 8, u64, pte | addr); 138 } 139 } else if (view->type == I915_GTT_VIEW_REMAPPED) { 140 const struct intel_remapped_info *remap_info = &view->remapped; 141 u32 i, dpt_ofs = 0; 142 143 for (i = 0; i < ARRAY_SIZE(remap_info->plane); i++) 144 write_dpt_remapped(bo, &dpt->vmap, &dpt_ofs, 145 remap_info->plane[i].offset, 146 remap_info->plane[i].width, 147 remap_info->plane[i].height, 148 remap_info->plane[i].src_stride, 149 remap_info->plane[i].dst_stride); 150 151 } else { 152 const struct intel_rotation_info *rot_info = &view->rotated; 153 u32 i, dpt_ofs = 0; 154 155 for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++) 156 write_dpt_rotated(bo, &dpt->vmap, &dpt_ofs, 157 rot_info->plane[i].offset, 158 rot_info->plane[i].width, 159 rot_info->plane[i].height, 160 rot_info->plane[i].src_stride, 161 rot_info->plane[i].dst_stride); 162 } 163 164 vma->dpt = dpt; 165 vma->node = dpt->ggtt_node[tile0->id]; 166 167 /* Ensure DPT writes are flushed */ 168 xe_device_l2_flush(xe); 169 return 0; 170 } 171 172 static void 173 write_ggtt_rotated(struct xe_ggtt *ggtt, u32 *ggtt_ofs, 174 u64 pte_flags, 175 xe_ggtt_set_pte_fn write_pte, 176 struct xe_bo *bo, u32 bo_ofs, 177 u32 width, u32 height, u32 src_stride, u32 dst_stride) 178 { 179 u32 column, row; 180 181 for (column = 0; column < width; column++) { 182 u32 src_idx = src_stride * (height - 1) + column + bo_ofs; 183 184 for (row = 0; row < height; row++) { 185 u64 addr = xe_bo_addr(bo, src_idx * XE_PAGE_SIZE, XE_PAGE_SIZE); 186 187 write_pte(ggtt, *ggtt_ofs, pte_flags | addr); 188 *ggtt_ofs += XE_PAGE_SIZE; 189 src_idx -= src_stride; 190 } 191 192 /* The DE ignores the PTEs for the padding tiles */ 193 *ggtt_ofs += (dst_stride - height) * XE_PAGE_SIZE; 194 } 195 } 196 197 struct fb_rotate_args { 198 const struct i915_gtt_view *view; 199 struct xe_bo *bo; 200 }; 201 202 static void write_ggtt_rotated_node(struct xe_ggtt *ggtt, struct xe_ggtt_node *node, 203 u64 pte_flags, xe_ggtt_set_pte_fn write_pte, void *data) 204 { 205 struct fb_rotate_args *args = data; 206 struct xe_bo *bo = args->bo; 207 const struct intel_rotation_info *rot_info = &args->view->rotated; 208 u32 ggtt_ofs = xe_ggtt_node_addr(node); 209 210 for (u32 i = 0; i < ARRAY_SIZE(rot_info->plane); i++) 211 write_ggtt_rotated(ggtt, &ggtt_ofs, pte_flags, write_pte, 212 bo, rot_info->plane[i].offset, 213 rot_info->plane[i].width, 214 rot_info->plane[i].height, 215 rot_info->plane[i].src_stride, 216 rot_info->plane[i].dst_stride); 217 } 218 219 static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb, 220 const struct i915_gtt_view *view, 221 struct i915_vma *vma, 222 unsigned int alignment) 223 { 224 struct drm_gem_object *obj = intel_fb_bo(&fb->base); 225 struct xe_bo *bo = gem_to_xe_bo(obj); 226 struct xe_device *xe = to_xe_device(fb->base.dev); 227 struct xe_tile *tile0 = xe_device_get_root_tile(xe); 228 struct xe_ggtt *ggtt = tile0->mem.ggtt; 229 u64 pte, size; 230 u32 align; 231 int ret = 0; 232 233 /* TODO: Consider sharing framebuffer mapping? 234 * embed i915_vma inside intel_framebuffer 235 */ 236 guard(xe_pm_runtime_noresume)(xe); 237 238 align = XE_PAGE_SIZE; 239 if (xe_bo_is_vram(bo) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) 240 align = max(align, SZ_64K); 241 242 /* Fast case, preallocated GGTT view? */ 243 if (bo->ggtt_node[tile0->id] && view->type == I915_GTT_VIEW_NORMAL) { 244 vma->node = bo->ggtt_node[tile0->id]; 245 return 0; 246 } 247 248 /* TODO: Consider sharing framebuffer mapping? 249 * embed i915_vma inside intel_framebuffer 250 */ 251 if (view->type == I915_GTT_VIEW_NORMAL) 252 size = xe_bo_size(bo); 253 else 254 /* display uses tiles instead of bytes here, so convert it back.. */ 255 size = intel_rotation_info_size(&view->rotated) * XE_PAGE_SIZE; 256 257 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe->pat.idx[XE_CACHE_NONE]); 258 vma->node = xe_ggtt_insert_node_transform(ggtt, bo, pte, 259 ALIGN(size, align), align, 260 view->type == I915_GTT_VIEW_NORMAL ? 261 NULL : write_ggtt_rotated_node, 262 &(struct fb_rotate_args){view, bo}); 263 if (IS_ERR(vma->node)) 264 ret = PTR_ERR(vma->node); 265 266 return ret; 267 } 268 269 static struct i915_vma *__xe_pin_fb_vma(const struct intel_framebuffer *fb, 270 const struct i915_gtt_view *view, 271 unsigned int alignment) 272 { 273 struct drm_device *dev = fb->base.dev; 274 struct xe_device *xe = to_xe_device(dev); 275 struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL); 276 struct drm_gem_object *obj = intel_fb_bo(&fb->base); 277 struct xe_bo *bo = gem_to_xe_bo(obj); 278 struct xe_validation_ctx ctx; 279 struct drm_exec exec; 280 int ret = 0; 281 282 if (!vma) 283 return ERR_PTR(-ENODEV); 284 285 refcount_set(&vma->ref, 1); 286 if (IS_DGFX(to_xe_device(bo->ttm.base.dev)) && 287 intel_fb_rc_ccs_cc_plane(&fb->base) >= 0 && 288 !(bo->flags & XE_BO_FLAG_NEEDS_CPU_ACCESS)) { 289 struct xe_vram_region *vram = xe_device_get_root_tile(xe)->mem.vram; 290 291 /* 292 * If we need to able to access the clear-color value stored in 293 * the buffer, then we require that such buffers are also CPU 294 * accessible. This is important on small-bar systems where 295 * only some subset of VRAM is CPU accessible. 296 */ 297 if (xe_vram_region_io_size(vram) < xe_vram_region_usable_size(vram)) { 298 ret = -EINVAL; 299 goto err; 300 } 301 } 302 303 /* 304 * Pin the framebuffer, we can't use xe_bo_(un)pin functions as the 305 * assumptions are incorrect for framebuffers 306 */ 307 xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.interruptible = true}, 308 ret) { 309 ret = drm_exec_lock_obj(&exec, &bo->ttm.base); 310 drm_exec_retry_on_contention(&exec); 311 if (ret) 312 break; 313 314 if (IS_DGFX(xe)) 315 ret = xe_bo_migrate(bo, XE_PL_VRAM0, NULL, &exec); 316 else 317 ret = xe_bo_validate(bo, NULL, true, &exec); 318 drm_exec_retry_on_contention(&exec); 319 xe_validation_retry_on_oom(&ctx, &ret); 320 if (!ret) 321 ttm_bo_pin(&bo->ttm); 322 } 323 if (ret) 324 goto err; 325 326 vma->bo = bo; 327 if (intel_fb_uses_dpt(&fb->base)) 328 ret = __xe_pin_fb_vma_dpt(fb, view, vma, alignment); 329 else 330 ret = __xe_pin_fb_vma_ggtt(fb, view, vma, alignment); 331 if (ret) 332 goto err_unpin; 333 334 return vma; 335 336 err_unpin: 337 ttm_bo_reserve(&bo->ttm, false, false, NULL); 338 ttm_bo_unpin(&bo->ttm); 339 ttm_bo_unreserve(&bo->ttm); 340 err: 341 kfree(vma); 342 return ERR_PTR(ret); 343 } 344 345 static void __xe_unpin_fb_vma(struct i915_vma *vma) 346 { 347 u8 tile_id = xe_device_get_root_tile(xe_bo_device(vma->bo))->id; 348 349 if (!refcount_dec_and_test(&vma->ref)) 350 return; 351 352 if (vma->dpt) 353 xe_bo_unpin_map_no_vm(vma->dpt); 354 else if (vma->bo->ggtt_node[tile_id] != vma->node) 355 xe_ggtt_node_remove(vma->node, false); 356 357 ttm_bo_reserve(&vma->bo->ttm, false, false, NULL); 358 ttm_bo_unpin(&vma->bo->ttm); 359 ttm_bo_unreserve(&vma->bo->ttm); 360 kfree(vma); 361 } 362 363 struct i915_vma * 364 intel_fb_pin_to_ggtt(const struct drm_framebuffer *fb, 365 const struct i915_gtt_view *view, 366 unsigned int alignment, 367 unsigned int phys_alignment, 368 unsigned int vtd_guard, 369 bool uses_fence, 370 unsigned long *out_flags) 371 { 372 *out_flags = 0; 373 374 return __xe_pin_fb_vma(to_intel_framebuffer(fb), view, alignment); 375 } 376 377 void intel_fb_unpin_vma(struct i915_vma *vma, unsigned long flags) 378 { 379 __xe_unpin_fb_vma(vma); 380 } 381 382 static bool reuse_vma(struct intel_plane_state *new_plane_state, 383 const struct intel_plane_state *old_plane_state) 384 { 385 struct intel_framebuffer *fb = to_intel_framebuffer(new_plane_state->hw.fb); 386 struct intel_plane *plane = to_intel_plane(new_plane_state->uapi.plane); 387 struct xe_device *xe = to_xe_device(fb->base.dev); 388 struct intel_display *display = xe->display; 389 struct i915_vma *vma; 390 391 if (old_plane_state->hw.fb == new_plane_state->hw.fb && 392 !memcmp(&old_plane_state->view.gtt, 393 &new_plane_state->view.gtt, 394 sizeof(new_plane_state->view.gtt))) { 395 vma = old_plane_state->ggtt_vma; 396 goto found; 397 } 398 399 if (fb == intel_fbdev_framebuffer(display->fbdev.fbdev)) { 400 vma = intel_fbdev_vma_pointer(display->fbdev.fbdev); 401 if (vma) 402 goto found; 403 } 404 405 return false; 406 407 found: 408 refcount_inc(&vma->ref); 409 new_plane_state->ggtt_vma = vma; 410 411 new_plane_state->surf = xe_ggtt_node_addr(new_plane_state->ggtt_vma->node) + 412 plane->surf_offset(new_plane_state); 413 414 return true; 415 } 416 417 int intel_plane_pin_fb(struct intel_plane_state *new_plane_state, 418 const struct intel_plane_state *old_plane_state) 419 { 420 struct drm_framebuffer *fb = new_plane_state->hw.fb; 421 struct drm_gem_object *obj = intel_fb_bo(fb); 422 struct xe_bo *bo = gem_to_xe_bo(obj); 423 struct i915_vma *vma; 424 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); 425 struct intel_plane *plane = to_intel_plane(new_plane_state->uapi.plane); 426 unsigned int alignment = plane->min_alignment(plane, fb, 0); 427 428 if (reuse_vma(new_plane_state, old_plane_state)) 429 return 0; 430 431 /* We reject creating !SCANOUT fb's, so this is weird.. */ 432 drm_WARN_ON(bo->ttm.base.dev, !(bo->flags & XE_BO_FLAG_SCANOUT)); 433 434 vma = __xe_pin_fb_vma(intel_fb, &new_plane_state->view.gtt, alignment); 435 436 if (IS_ERR(vma)) 437 return PTR_ERR(vma); 438 439 new_plane_state->ggtt_vma = vma; 440 441 new_plane_state->surf = xe_ggtt_node_addr(new_plane_state->ggtt_vma->node) + 442 plane->surf_offset(new_plane_state); 443 444 return 0; 445 } 446 447 void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state) 448 { 449 __xe_unpin_fb_vma(old_plane_state->ggtt_vma); 450 old_plane_state->ggtt_vma = NULL; 451 } 452 453 void intel_fb_get_map(struct i915_vma *vma, struct iosys_map *map) 454 { 455 *map = vma->bo->vmap; 456 } 457