1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include <drm/intel/display_parent_interface.h> 7 #include <drm/ttm/ttm_bo.h> 8 9 /* FIXME move the types to parent interface? */ 10 #include "i915_gtt_view_types.h" 11 12 /* FIXME move intel_remapped_info_size() & co. to parent interface? */ 13 #include "intel_fb.h" 14 15 #include "xe_bo.h" 16 #include "xe_device.h" 17 #include "xe_display_vma.h" 18 #include "xe_fb_pin.h" 19 #include "xe_ggtt.h" 20 #include "xe_pat.h" 21 #include "xe_pm.h" 22 #include "xe_vram_types.h" 23 24 static void 25 write_dpt_rotated(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, u32 bo_ofs, 26 u32 width, u32 height, u32 src_stride, u32 dst_stride) 27 { 28 struct xe_device *xe = xe_bo_device(bo); 29 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 30 u32 column, row; 31 u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe_cache_pat_idx(xe, XE_CACHE_NONE)); 32 33 /* TODO: Maybe rewrite so we can traverse the bo addresses sequentially, 34 * by writing dpt/ggtt in a different order? 35 */ 36 37 for (column = 0; column < width; column++) { 38 u32 src_idx = src_stride * (height - 1) + column + bo_ofs; 39 40 for (row = 0; row < height; row++) { 41 u64 addr = xe_bo_addr(bo, src_idx * XE_PAGE_SIZE, XE_PAGE_SIZE); 42 43 iosys_map_wr(map, *dpt_ofs, u64, pte | addr); 44 *dpt_ofs += 8; 45 src_idx -= src_stride; 46 } 47 48 /* The DE ignores the PTEs for the padding tiles */ 49 *dpt_ofs += (dst_stride - height) * 8; 50 } 51 52 /* Align to next page */ 53 *dpt_ofs = ALIGN(*dpt_ofs, 4096); 54 } 55 56 static unsigned int 57 write_dpt_padding(struct iosys_map *map, unsigned int dest, unsigned int pad) 58 { 59 /* The DE ignores the PTEs for the padding tiles */ 60 return dest + pad * sizeof(u64); 61 } 62 63 static unsigned int 64 write_dpt_remapped_linear(struct xe_bo *bo, struct iosys_map *map, 65 unsigned int dest, 66 const struct intel_remapped_plane_info *plane) 67 { 68 struct xe_device *xe = xe_bo_device(bo); 69 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 70 const u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, 71 xe_cache_pat_idx(xe, XE_CACHE_NONE)); 72 unsigned int offset = plane->offset * XE_PAGE_SIZE; 73 unsigned int size = plane->size; 74 75 while (size--) { 76 u64 addr = xe_bo_addr(bo, offset, XE_PAGE_SIZE); 77 78 iosys_map_wr(map, dest, u64, addr | pte); 79 dest += sizeof(u64); 80 offset += XE_PAGE_SIZE; 81 } 82 83 return dest; 84 } 85 86 static unsigned int 87 write_dpt_remapped_tiled(struct xe_bo *bo, struct iosys_map *map, 88 unsigned int dest, 89 const struct intel_remapped_plane_info *plane) 90 { 91 struct xe_device *xe = xe_bo_device(bo); 92 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 93 const u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, 94 xe_cache_pat_idx(xe, XE_CACHE_NONE)); 95 unsigned int offset, column, row; 96 97 for (row = 0; row < plane->height; row++) { 98 offset = (plane->offset + plane->src_stride * row) * 99 XE_PAGE_SIZE; 100 101 for (column = 0; column < plane->width; column++) { 102 u64 addr = xe_bo_addr(bo, offset, XE_PAGE_SIZE); 103 104 iosys_map_wr(map, dest, u64, addr | pte); 105 dest += sizeof(u64); 106 offset += XE_PAGE_SIZE; 107 } 108 109 dest = write_dpt_padding(map, dest, 110 plane->dst_stride - plane->width); 111 } 112 113 return dest; 114 } 115 116 static void 117 write_dpt_remapped(struct xe_bo *bo, 118 const struct intel_remapped_info *remap_info, 119 struct iosys_map *map) 120 { 121 unsigned int i, dest = 0; 122 123 for (i = 0; i < ARRAY_SIZE(remap_info->plane); i++) { 124 const struct intel_remapped_plane_info *plane = 125 &remap_info->plane[i]; 126 127 if (!plane->linear && !plane->width && !plane->height) 128 continue; 129 130 if (dest && remap_info->plane_alignment) { 131 const unsigned int index = dest / sizeof(u64); 132 const unsigned int pad = 133 ALIGN(index, remap_info->plane_alignment) - 134 index; 135 136 dest = write_dpt_padding(map, dest, pad); 137 } 138 139 if (plane->linear) 140 dest = write_dpt_remapped_linear(bo, map, dest, plane); 141 else 142 dest = write_dpt_remapped_tiled(bo, map, dest, plane); 143 } 144 } 145 146 static int __xe_pin_fb_vma_dpt(struct drm_gem_object *obj, 147 const struct intel_fb_pin_params *pin_params, 148 struct i915_vma *vma) 149 { 150 struct xe_device *xe = to_xe_device(obj->dev); 151 struct xe_tile *tile0 = xe_device_get_root_tile(xe); 152 struct xe_ggtt *ggtt = tile0->mem.ggtt; 153 const struct i915_gtt_view *view = pin_params->view; 154 struct xe_bo *bo = gem_to_xe_bo(obj), *dpt; 155 u32 dpt_size, size = bo->ttm.base.size; 156 157 if (view->type == I915_GTT_VIEW_NORMAL) 158 dpt_size = ALIGN(size / XE_PAGE_SIZE * 8, XE_PAGE_SIZE); 159 else if (view->type == I915_GTT_VIEW_REMAPPED) 160 dpt_size = ALIGN(intel_remapped_info_size(&view->remapped) * 8, 161 XE_PAGE_SIZE); 162 else 163 /* display uses 4K tiles instead of bytes here, convert to entries.. */ 164 dpt_size = ALIGN(intel_rotation_info_size(&view->rotated) * 8, 165 XE_PAGE_SIZE); 166 167 if (IS_DGFX(xe)) 168 dpt = xe_bo_create_pin_map_at_novm(xe, tile0, 169 dpt_size, ~0ull, 170 ttm_bo_type_kernel, 171 XE_BO_FLAG_VRAM0 | 172 XE_BO_FLAG_GGTT | 173 XE_BO_FLAG_PAGETABLE, 174 pin_params->alignment, false); 175 else 176 dpt = xe_bo_create_pin_map_at_novm(xe, tile0, 177 dpt_size, ~0ull, 178 ttm_bo_type_kernel, 179 XE_BO_FLAG_STOLEN | 180 XE_BO_FLAG_GGTT | 181 XE_BO_FLAG_PAGETABLE, 182 pin_params->alignment, false); 183 if (IS_ERR(dpt)) 184 dpt = xe_bo_create_pin_map_at_novm(xe, tile0, 185 dpt_size, ~0ull, 186 ttm_bo_type_kernel, 187 XE_BO_FLAG_SYSTEM | 188 XE_BO_FLAG_GGTT | 189 XE_BO_FLAG_PAGETABLE | 190 XE_BO_FLAG_FORCE_WC, 191 pin_params->alignment, false); 192 if (IS_ERR(dpt)) 193 return PTR_ERR(dpt); 194 195 if (view->type == I915_GTT_VIEW_NORMAL) { 196 u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe_cache_pat_idx(xe, XE_CACHE_NONE)); 197 u32 x; 198 199 for (x = 0; x < size / XE_PAGE_SIZE; x++) { 200 u64 addr = xe_bo_addr(bo, x * XE_PAGE_SIZE, XE_PAGE_SIZE); 201 202 iosys_map_wr(&dpt->vmap, x * 8, u64, pte | addr); 203 } 204 } else if (view->type == I915_GTT_VIEW_REMAPPED) { 205 write_dpt_remapped(bo, &view->remapped, &dpt->vmap); 206 } else { 207 const struct intel_rotation_info *rot_info = &view->rotated; 208 u32 i, dpt_ofs = 0; 209 210 for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++) 211 write_dpt_rotated(bo, &dpt->vmap, &dpt_ofs, 212 rot_info->plane[i].offset, 213 rot_info->plane[i].width, 214 rot_info->plane[i].height, 215 rot_info->plane[i].src_stride, 216 rot_info->plane[i].dst_stride); 217 } 218 219 vma->dpt = dpt; 220 vma->node = dpt->ggtt_node[tile0->id]; 221 222 /* Ensure DPT writes are flushed */ 223 xe_device_l2_flush(xe); 224 return 0; 225 } 226 227 static void 228 write_ggtt_rotated(struct xe_ggtt *ggtt, u32 *ggtt_ofs, 229 u64 pte_flags, 230 xe_ggtt_set_pte_fn write_pte, 231 struct xe_bo *bo, u32 bo_ofs, 232 u32 width, u32 height, u32 src_stride, u32 dst_stride) 233 { 234 u32 column, row; 235 236 for (column = 0; column < width; column++) { 237 u32 src_idx = src_stride * (height - 1) + column + bo_ofs; 238 239 for (row = 0; row < height; row++) { 240 u64 addr = xe_bo_addr(bo, src_idx * XE_PAGE_SIZE, XE_PAGE_SIZE); 241 242 write_pte(ggtt, *ggtt_ofs, pte_flags | addr); 243 *ggtt_ofs += XE_PAGE_SIZE; 244 src_idx -= src_stride; 245 } 246 247 /* The DE ignores the PTEs for the padding tiles */ 248 *ggtt_ofs += (dst_stride - height) * XE_PAGE_SIZE; 249 } 250 } 251 252 struct fb_rotate_args { 253 const struct i915_gtt_view *view; 254 struct xe_bo *bo; 255 }; 256 257 static void write_ggtt_rotated_node(struct xe_ggtt *ggtt, struct xe_ggtt_node *node, 258 u64 pte_flags, xe_ggtt_set_pte_fn write_pte, void *data) 259 { 260 struct fb_rotate_args *args = data; 261 struct xe_bo *bo = args->bo; 262 const struct intel_rotation_info *rot_info = &args->view->rotated; 263 u32 ggtt_ofs = xe_ggtt_node_addr(node); 264 265 for (u32 i = 0; i < ARRAY_SIZE(rot_info->plane); i++) 266 write_ggtt_rotated(ggtt, &ggtt_ofs, pte_flags, write_pte, 267 bo, rot_info->plane[i].offset, 268 rot_info->plane[i].width, 269 rot_info->plane[i].height, 270 rot_info->plane[i].src_stride, 271 rot_info->plane[i].dst_stride); 272 } 273 274 static int __xe_pin_fb_vma_ggtt(struct drm_gem_object *obj, 275 const struct intel_fb_pin_params *pin_params, 276 struct i915_vma *vma) 277 { 278 const struct i915_gtt_view *view = pin_params->view; 279 struct xe_bo *bo = gem_to_xe_bo(obj); 280 struct xe_device *xe = to_xe_device(obj->dev); 281 struct xe_tile *tile0 = xe_device_get_root_tile(xe); 282 struct xe_ggtt *ggtt = tile0->mem.ggtt; 283 u64 pte, size; 284 u32 align; 285 int ret = 0; 286 287 /* TODO: Consider sharing framebuffer mapping? 288 * embed i915_vma inside intel_framebuffer 289 */ 290 guard(xe_pm_runtime_noresume)(xe); 291 292 align = max(XE_PAGE_SIZE, pin_params->alignment); 293 if (xe_bo_is_vram(bo) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) 294 align = max(align, SZ_64K); 295 296 /* Fast case, preallocated GGTT view? */ 297 if (bo->ggtt_node[tile0->id] && view->type == I915_GTT_VIEW_NORMAL) { 298 vma->node = bo->ggtt_node[tile0->id]; 299 return 0; 300 } 301 302 /* TODO: Consider sharing framebuffer mapping? 303 * embed i915_vma inside intel_framebuffer 304 */ 305 if (view->type == I915_GTT_VIEW_NORMAL) 306 size = xe_bo_size(bo); 307 else 308 /* display uses tiles instead of bytes here, so convert it back.. */ 309 size = intel_rotation_info_size(&view->rotated) * XE_PAGE_SIZE; 310 311 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe_cache_pat_idx(xe, XE_CACHE_NONE)); 312 vma->node = xe_ggtt_insert_node_transform(ggtt, bo, pte, 313 ALIGN(size, align), align, 314 view->type == I915_GTT_VIEW_NORMAL ? 315 NULL : write_ggtt_rotated_node, 316 &(struct fb_rotate_args){view, bo}); 317 if (IS_ERR(vma->node)) 318 ret = PTR_ERR(vma->node); 319 320 return ret; 321 } 322 323 static struct i915_vma *__xe_pin_fb_vma(struct drm_gem_object *obj, bool is_dpt, 324 const struct intel_fb_pin_params *pin_params) 325 { 326 struct xe_device *xe = to_xe_device(obj->dev); 327 struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL); 328 struct xe_bo *bo = gem_to_xe_bo(obj); 329 struct xe_validation_ctx ctx; 330 struct drm_exec exec; 331 int ret = 0; 332 333 /* We reject creating !SCANOUT fb's, so this is weird.. */ 334 drm_WARN_ON(bo->ttm.base.dev, !(bo->flags & XE_BO_FLAG_FORCE_WC) && 335 bo->ttm.type != ttm_bo_type_sg); 336 337 if (!vma) 338 return ERR_PTR(-ENODEV); 339 340 refcount_set(&vma->ref, 1); 341 if (IS_DGFX(to_xe_device(bo->ttm.base.dev)) && 342 pin_params->needs_cpu_lmem_access && 343 !(bo->flags & XE_BO_FLAG_NEEDS_CPU_ACCESS)) { 344 struct xe_vram_region *vram = xe_device_get_root_tile(xe)->mem.vram; 345 346 /* 347 * If we need to able to access the clear-color value stored in 348 * the buffer, then we require that such buffers are also CPU 349 * accessible. This is important on small-bar systems where 350 * only some subset of VRAM is CPU accessible. 351 */ 352 if (xe_vram_region_io_size(vram) < xe_vram_region_usable_size(vram)) { 353 ret = -EINVAL; 354 goto err; 355 } 356 } 357 358 /* 359 * Pin the framebuffer, we can't use xe_bo_(un)pin functions as the 360 * assumptions are incorrect for framebuffers 361 */ 362 xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.interruptible = true}, 363 ret) { 364 ret = drm_exec_lock_obj(&exec, &bo->ttm.base); 365 drm_exec_retry_on_contention(&exec); 366 if (ret) 367 break; 368 369 if (IS_DGFX(xe)) 370 ret = xe_bo_migrate(bo, XE_PL_VRAM0, NULL, &exec); 371 else 372 ret = xe_bo_validate(bo, NULL, true, &exec); 373 drm_exec_retry_on_contention(&exec); 374 xe_validation_retry_on_oom(&ctx, &ret); 375 if (!ret) 376 ttm_bo_pin(&bo->ttm); 377 } 378 if (ret) 379 goto err; 380 381 vma->bo = bo; 382 if (is_dpt) 383 ret = __xe_pin_fb_vma_dpt(obj, pin_params, vma); 384 else 385 ret = __xe_pin_fb_vma_ggtt(obj, pin_params, vma); 386 if (ret) 387 goto err_unpin; 388 389 return vma; 390 391 err_unpin: 392 ttm_bo_reserve(&bo->ttm, false, false, NULL); 393 ttm_bo_unpin(&bo->ttm); 394 ttm_bo_unreserve(&bo->ttm); 395 err: 396 kfree(vma); 397 return ERR_PTR(ret); 398 } 399 400 static void __xe_unpin_fb_vma(struct i915_vma *vma) 401 { 402 u8 tile_id = xe_device_get_root_tile(xe_bo_device(vma->bo))->id; 403 404 if (!refcount_dec_and_test(&vma->ref)) 405 return; 406 407 if (vma->dpt) 408 xe_bo_unpin_map_no_vm(vma->dpt); 409 else if (vma->bo->ggtt_node[tile_id] != vma->node) 410 xe_ggtt_node_remove(vma->node, false); 411 412 ttm_bo_reserve(&vma->bo->ttm, false, false, NULL); 413 ttm_bo_unpin(&vma->bo->ttm); 414 ttm_bo_unreserve(&vma->bo->ttm); 415 kfree(vma); 416 } 417 418 int xe_fb_pin_ggtt_pin(struct drm_gem_object *obj, 419 const struct intel_fb_pin_params *pin_params, 420 struct i915_vma **out_ggtt_vma, 421 u32 *out_offset, 422 int *out_fence_id) 423 { 424 struct i915_vma *ggtt_vma; 425 426 ggtt_vma = __xe_pin_fb_vma(obj, false, pin_params); 427 if (IS_ERR(ggtt_vma)) 428 return PTR_ERR(ggtt_vma); 429 430 *out_ggtt_vma = ggtt_vma; 431 *out_offset = xe_ggtt_node_addr(ggtt_vma->node); 432 if (out_fence_id) 433 *out_fence_id = -1; 434 435 return 0; 436 } 437 438 static void xe_fb_pin_ggtt_unpin(struct i915_vma *ggtt_vma, 439 int fence_id) 440 { 441 WARN_ON(fence_id >= 0); 442 443 __xe_unpin_fb_vma(ggtt_vma); 444 } 445 446 static int xe_fb_pin_dpt_pin(struct drm_gem_object *obj, struct intel_dpt *dpt, 447 const struct intel_fb_pin_params *pin_params, 448 struct i915_vma **out_dpt_vma, 449 struct i915_vma **out_ggtt_vma, 450 u32 *out_offset) 451 { 452 struct i915_vma *ggtt_vma; 453 454 WARN_ON(dpt); 455 456 ggtt_vma = __xe_pin_fb_vma(obj, true, pin_params); 457 if (IS_ERR(ggtt_vma)) 458 return PTR_ERR(ggtt_vma); 459 460 *out_dpt_vma = NULL; /* not used on xe */ 461 *out_ggtt_vma = ggtt_vma; 462 *out_offset = xe_ggtt_node_addr(ggtt_vma->node); 463 464 return 0; 465 } 466 467 static void xe_fb_pin_dpt_unpin(struct intel_dpt *dpt, 468 struct i915_vma *dpt_vma, 469 struct i915_vma *ggtt_vma) 470 { 471 WARN_ON(dpt || dpt_vma); 472 473 __xe_unpin_fb_vma(ggtt_vma); 474 } 475 476 static struct i915_vma * 477 xe_fb_pin_reuse_vma(struct i915_vma *old_ggtt_vma, 478 struct drm_gem_object *old_obj, 479 const struct i915_gtt_view *old_view, 480 struct drm_gem_object *new_obj, 481 const struct i915_gtt_view *new_view, 482 u32 *out_offset) 483 { 484 if (old_ggtt_vma && old_obj == new_obj && 485 !memcmp(old_view, new_view, sizeof(*new_view))) { 486 refcount_inc(&old_ggtt_vma->ref); 487 488 *out_offset = xe_ggtt_node_addr(old_ggtt_vma->node); 489 490 return old_ggtt_vma; 491 } 492 493 return NULL; 494 } 495 496 static void xe_fb_pin_get_map(struct i915_vma *vma, struct iosys_map *map) 497 { 498 *map = vma->bo->vmap; 499 } 500 501 const struct intel_display_fb_pin_interface xe_display_fb_pin_interface = { 502 .ggtt_pin = xe_fb_pin_ggtt_pin, 503 .ggtt_unpin = xe_fb_pin_ggtt_unpin, 504 .dpt_pin = xe_fb_pin_dpt_pin, 505 .dpt_unpin = xe_fb_pin_dpt_unpin, 506 .reuse_vma = xe_fb_pin_reuse_vma, 507 .get_map = xe_fb_pin_get_map, 508 }; 509