1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include <drm/intel/display_parent_interface.h> 7 #include <drm/intel/gtt_view_types.h> 8 #include <drm/ttm/ttm_bo.h> 9 10 /* FIXME move intel_remapped_info_size() & co. to parent interface? */ 11 #include "intel_fb.h" 12 13 #include "xe_bo.h" 14 #include "xe_device.h" 15 #include "xe_display_vma.h" 16 #include "xe_fb_pin.h" 17 #include "xe_ggtt.h" 18 #include "xe_pat.h" 19 #include "xe_pm.h" 20 #include "xe_vram_types.h" 21 22 static void 23 write_dpt_rotated(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, u32 bo_ofs, 24 u32 width, u32 height, u32 src_stride, u32 dst_stride) 25 { 26 struct xe_device *xe = xe_bo_device(bo); 27 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 28 u32 column, row; 29 u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe_cache_pat_idx(xe, XE_CACHE_NONE)); 30 31 /* TODO: Maybe rewrite so we can traverse the bo addresses sequentially, 32 * by writing dpt/ggtt in a different order? 33 */ 34 35 for (column = 0; column < width; column++) { 36 u32 src_idx = src_stride * (height - 1) + column + bo_ofs; 37 38 for (row = 0; row < height; row++) { 39 u64 addr = xe_bo_addr(bo, src_idx * XE_PAGE_SIZE, XE_PAGE_SIZE); 40 41 iosys_map_wr(map, *dpt_ofs, u64, pte | addr); 42 *dpt_ofs += 8; 43 src_idx -= src_stride; 44 } 45 46 /* The DE ignores the PTEs for the padding tiles */ 47 *dpt_ofs += (dst_stride - height) * 8; 48 } 49 50 /* Align to next page */ 51 *dpt_ofs = ALIGN(*dpt_ofs, 4096); 52 } 53 54 static unsigned int 55 write_dpt_padding(struct iosys_map *map, unsigned int dest, unsigned int pad) 56 { 57 /* The DE ignores the PTEs for the padding tiles */ 58 return dest + pad * sizeof(u64); 59 } 60 61 static unsigned int 62 write_dpt_remapped_linear(struct xe_bo *bo, struct iosys_map *map, 63 unsigned int dest, 64 const struct intel_remapped_plane_info *plane) 65 { 66 struct xe_device *xe = xe_bo_device(bo); 67 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 68 const u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, 69 xe_cache_pat_idx(xe, XE_CACHE_NONE)); 70 unsigned int offset = plane->offset * XE_PAGE_SIZE; 71 unsigned int size = plane->size; 72 73 while (size--) { 74 u64 addr = xe_bo_addr(bo, offset, XE_PAGE_SIZE); 75 76 iosys_map_wr(map, dest, u64, addr | pte); 77 dest += sizeof(u64); 78 offset += XE_PAGE_SIZE; 79 } 80 81 return dest; 82 } 83 84 static unsigned int 85 write_dpt_remapped_tiled(struct xe_bo *bo, struct iosys_map *map, 86 unsigned int dest, 87 const struct intel_remapped_plane_info *plane) 88 { 89 struct xe_device *xe = xe_bo_device(bo); 90 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 91 const u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, 92 xe_cache_pat_idx(xe, XE_CACHE_NONE)); 93 unsigned int offset, column, row; 94 95 for (row = 0; row < plane->height; row++) { 96 offset = (plane->offset + plane->src_stride * row) * 97 XE_PAGE_SIZE; 98 99 for (column = 0; column < plane->width; column++) { 100 u64 addr = xe_bo_addr(bo, offset, XE_PAGE_SIZE); 101 102 iosys_map_wr(map, dest, u64, addr | pte); 103 dest += sizeof(u64); 104 offset += XE_PAGE_SIZE; 105 } 106 107 dest = write_dpt_padding(map, dest, 108 plane->dst_stride - plane->width); 109 } 110 111 return dest; 112 } 113 114 static void 115 write_dpt_remapped(struct xe_bo *bo, 116 const struct intel_remapped_info *remap_info, 117 struct iosys_map *map) 118 { 119 unsigned int i, dest = 0; 120 121 for (i = 0; i < ARRAY_SIZE(remap_info->plane); i++) { 122 const struct intel_remapped_plane_info *plane = 123 &remap_info->plane[i]; 124 125 if (!plane->linear && !plane->width && !plane->height) 126 continue; 127 128 if (dest && remap_info->plane_alignment) { 129 const unsigned int index = dest / sizeof(u64); 130 const unsigned int pad = 131 ALIGN(index, remap_info->plane_alignment) - 132 index; 133 134 dest = write_dpt_padding(map, dest, pad); 135 } 136 137 if (plane->linear) 138 dest = write_dpt_remapped_linear(bo, map, dest, plane); 139 else 140 dest = write_dpt_remapped_tiled(bo, map, dest, plane); 141 } 142 } 143 144 static int __xe_pin_fb_vma_dpt(struct drm_gem_object *obj, 145 const struct intel_fb_pin_params *pin_params, 146 struct i915_vma *vma) 147 { 148 struct xe_device *xe = to_xe_device(obj->dev); 149 struct xe_tile *tile0 = xe_device_get_root_tile(xe); 150 struct xe_ggtt *ggtt = tile0->mem.ggtt; 151 const struct intel_gtt_view *view = pin_params->view; 152 struct xe_bo *bo = gem_to_xe_bo(obj), *dpt; 153 u32 dpt_size, size = bo->ttm.base.size; 154 155 if (intel_gtt_view_is_normal(view)) 156 dpt_size = ALIGN(size / XE_PAGE_SIZE * 8, XE_PAGE_SIZE); 157 else if (intel_gtt_view_is_remapped(view)) 158 dpt_size = ALIGN(intel_remapped_info_size(&view->remapped) * 8, 159 XE_PAGE_SIZE); 160 else 161 /* display uses 4K tiles instead of bytes here, convert to entries.. */ 162 dpt_size = ALIGN(intel_rotation_info_size(&view->rotated) * 8, 163 XE_PAGE_SIZE); 164 165 dpt = xe_bo_create_pin_map_at_novm(xe, tile0, 166 dpt_size, ~0ull, 167 ttm_bo_type_kernel, 168 XE_BO_FLAG_VRAM_IF_DGFX(tile0) | 169 XE_BO_FLAG_GGTT | 170 XE_BO_FLAG_PAGETABLE | 171 XE_BO_FLAG_FORCE_WC, 172 pin_params->alignment, false); 173 if (IS_ERR(dpt)) 174 return PTR_ERR(dpt); 175 176 if (intel_gtt_view_is_normal(view)) { 177 u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe_cache_pat_idx(xe, XE_CACHE_NONE)); 178 u32 x; 179 180 for (x = 0; x < size / XE_PAGE_SIZE; x++) { 181 u64 addr = xe_bo_addr(bo, x * XE_PAGE_SIZE, XE_PAGE_SIZE); 182 183 iosys_map_wr(&dpt->vmap, x * 8, u64, pte | addr); 184 } 185 } else if (intel_gtt_view_is_remapped(view)) { 186 write_dpt_remapped(bo, &view->remapped, &dpt->vmap); 187 } else { 188 const struct intel_rotation_info *rot_info = &view->rotated; 189 u32 i, dpt_ofs = 0; 190 191 for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++) 192 write_dpt_rotated(bo, &dpt->vmap, &dpt_ofs, 193 rot_info->plane[i].offset, 194 rot_info->plane[i].width, 195 rot_info->plane[i].height, 196 rot_info->plane[i].src_stride, 197 rot_info->plane[i].dst_stride); 198 } 199 200 vma->dpt = dpt; 201 vma->node = dpt->ggtt_node[tile0->id]; 202 203 /* Ensure DPT writes are flushed */ 204 xe_device_l2_flush(xe, false); 205 return 0; 206 } 207 208 static void 209 write_ggtt_rotated(struct xe_ggtt *ggtt, u32 *ggtt_ofs, 210 u64 pte_flags, 211 xe_ggtt_set_pte_fn write_pte, 212 struct xe_bo *bo, u32 bo_ofs, 213 u32 width, u32 height, u32 src_stride, u32 dst_stride) 214 { 215 u32 column, row; 216 217 for (column = 0; column < width; column++) { 218 u32 src_idx = src_stride * (height - 1) + column + bo_ofs; 219 220 for (row = 0; row < height; row++) { 221 u64 addr = xe_bo_addr(bo, src_idx * XE_PAGE_SIZE, XE_PAGE_SIZE); 222 223 write_pte(ggtt, *ggtt_ofs, pte_flags | addr); 224 *ggtt_ofs += XE_PAGE_SIZE; 225 src_idx -= src_stride; 226 } 227 228 /* The DE ignores the PTEs for the padding tiles */ 229 *ggtt_ofs += (dst_stride - height) * XE_PAGE_SIZE; 230 } 231 } 232 233 struct fb_rotate_args { 234 const struct intel_gtt_view *view; 235 struct xe_bo *bo; 236 }; 237 238 static void write_ggtt_rotated_node(struct xe_ggtt *ggtt, struct xe_ggtt_node *node, 239 u64 pte_flags, xe_ggtt_set_pte_fn write_pte, void *data) 240 { 241 struct fb_rotate_args *args = data; 242 struct xe_bo *bo = args->bo; 243 const struct intel_rotation_info *rot_info = &args->view->rotated; 244 u32 ggtt_ofs = xe_ggtt_node_addr(node); 245 246 for (u32 i = 0; i < ARRAY_SIZE(rot_info->plane); i++) 247 write_ggtt_rotated(ggtt, &ggtt_ofs, pte_flags, write_pte, 248 bo, rot_info->plane[i].offset, 249 rot_info->plane[i].width, 250 rot_info->plane[i].height, 251 rot_info->plane[i].src_stride, 252 rot_info->plane[i].dst_stride); 253 } 254 255 static int __xe_pin_fb_vma_ggtt(struct drm_gem_object *obj, 256 const struct intel_fb_pin_params *pin_params, 257 struct i915_vma *vma) 258 { 259 const struct intel_gtt_view *view = pin_params->view; 260 struct xe_bo *bo = gem_to_xe_bo(obj); 261 struct xe_device *xe = to_xe_device(obj->dev); 262 struct xe_tile *tile0 = xe_device_get_root_tile(xe); 263 struct xe_ggtt *ggtt = tile0->mem.ggtt; 264 u64 pte, size; 265 u32 align; 266 int ret = 0; 267 268 /* TODO: Consider sharing framebuffer mapping? 269 * embed i915_vma inside intel_framebuffer 270 */ 271 guard(xe_pm_runtime_noresume)(xe); 272 273 align = max(XE_PAGE_SIZE, pin_params->alignment); 274 if (xe_bo_is_vram(bo) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) 275 align = max(align, SZ_64K); 276 277 /* Fast case, preallocated GGTT view? */ 278 if (bo->ggtt_node[tile0->id] && intel_gtt_view_is_normal(view)) { 279 vma->node = bo->ggtt_node[tile0->id]; 280 return 0; 281 } 282 283 /* TODO: Consider sharing framebuffer mapping? 284 * embed i915_vma inside intel_framebuffer 285 */ 286 if (intel_gtt_view_is_normal(view)) 287 size = xe_bo_size(bo); 288 else 289 /* display uses tiles instead of bytes here, so convert it back.. */ 290 size = intel_rotation_info_size(&view->rotated) * XE_PAGE_SIZE; 291 292 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe_cache_pat_idx(xe, XE_CACHE_NONE)); 293 vma->node = xe_ggtt_insert_node_transform(ggtt, bo, pte, 294 ALIGN(size, align), align, 295 intel_gtt_view_is_normal(view) ? 296 NULL : write_ggtt_rotated_node, 297 &(struct fb_rotate_args){view, bo}); 298 if (IS_ERR(vma->node)) 299 ret = PTR_ERR(vma->node); 300 301 return ret; 302 } 303 304 static struct i915_vma *__xe_pin_fb_vma(struct drm_gem_object *obj, bool is_dpt, 305 const struct intel_fb_pin_params *pin_params) 306 { 307 struct xe_device *xe = to_xe_device(obj->dev); 308 struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL); 309 struct xe_bo *bo = gem_to_xe_bo(obj); 310 struct xe_validation_ctx ctx; 311 struct drm_exec exec; 312 int ret = 0; 313 314 /* We reject creating !SCANOUT fb's, so this is weird.. */ 315 drm_WARN_ON(bo->ttm.base.dev, !(bo->flags & XE_BO_FLAG_FORCE_WC) && 316 bo->ttm.type != ttm_bo_type_sg); 317 318 if (!vma) 319 return ERR_PTR(-ENODEV); 320 321 refcount_set(&vma->ref, 1); 322 if (IS_DGFX(to_xe_device(bo->ttm.base.dev)) && 323 pin_params->needs_cpu_lmem_access && 324 !(bo->flags & XE_BO_FLAG_NEEDS_CPU_ACCESS)) { 325 struct xe_vram_region *vram = xe_device_get_root_tile(xe)->mem.vram; 326 327 /* 328 * If we need to able to access the clear-color value stored in 329 * the buffer, then we require that such buffers are also CPU 330 * accessible. This is important on small-bar systems where 331 * only some subset of VRAM is CPU accessible. 332 */ 333 if (xe_vram_region_io_size(vram) < xe_vram_region_usable_size(vram)) { 334 ret = -EINVAL; 335 goto err; 336 } 337 } 338 339 /* 340 * Pin the framebuffer, we can't use xe_bo_(un)pin functions as the 341 * assumptions are incorrect for framebuffers 342 */ 343 xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {.interruptible = true}, 344 ret) { 345 ret = drm_exec_lock_obj(&exec, &bo->ttm.base); 346 drm_exec_retry_on_contention(&exec); 347 if (ret) 348 break; 349 350 if (IS_DGFX(xe)) 351 ret = xe_bo_migrate(bo, XE_PL_VRAM0, NULL, &exec); 352 else 353 ret = xe_bo_validate(bo, NULL, true, &exec); 354 drm_exec_retry_on_contention(&exec); 355 xe_validation_retry_on_oom(&ctx, &ret); 356 if (!ret) 357 xe_bo_pin_account(bo); 358 } 359 if (ret) 360 goto err; 361 362 vma->bo = bo; 363 if (is_dpt) 364 ret = __xe_pin_fb_vma_dpt(obj, pin_params, vma); 365 else 366 ret = __xe_pin_fb_vma_ggtt(obj, pin_params, vma); 367 if (ret) 368 goto err_unpin; 369 370 return vma; 371 372 err_unpin: 373 ttm_bo_reserve(&bo->ttm, false, false, NULL); 374 xe_bo_unpin_account(bo); 375 ttm_bo_unreserve(&bo->ttm); 376 err: 377 kfree(vma); 378 return ERR_PTR(ret); 379 } 380 381 static void __xe_unpin_fb_vma(struct i915_vma *vma) 382 { 383 u8 tile_id = xe_device_get_root_tile(xe_bo_device(vma->bo))->id; 384 385 if (!refcount_dec_and_test(&vma->ref)) 386 return; 387 388 if (vma->dpt) 389 xe_bo_unpin_map_no_vm(vma->dpt); 390 else if (vma->bo->ggtt_node[tile_id] != vma->node) 391 xe_ggtt_node_remove(vma->node, false); 392 393 ttm_bo_reserve(&vma->bo->ttm, false, false, NULL); 394 xe_bo_unpin_account(vma->bo); 395 ttm_bo_unreserve(&vma->bo->ttm); 396 kfree(vma); 397 } 398 399 int xe_fb_pin_ggtt_pin(struct drm_gem_object *obj, 400 const struct intel_fb_pin_params *pin_params, 401 struct i915_vma **out_ggtt_vma, 402 u32 *out_offset, 403 int *out_fence_id) 404 { 405 struct i915_vma *ggtt_vma; 406 407 ggtt_vma = __xe_pin_fb_vma(obj, false, pin_params); 408 if (IS_ERR(ggtt_vma)) 409 return PTR_ERR(ggtt_vma); 410 411 *out_ggtt_vma = ggtt_vma; 412 *out_offset = xe_ggtt_node_addr(ggtt_vma->node); 413 if (out_fence_id) 414 *out_fence_id = -1; 415 416 return 0; 417 } 418 419 static void xe_fb_pin_ggtt_unpin(struct i915_vma *ggtt_vma, 420 int fence_id) 421 { 422 WARN_ON(fence_id >= 0); 423 424 __xe_unpin_fb_vma(ggtt_vma); 425 } 426 427 static int xe_fb_pin_dpt_pin(struct drm_gem_object *obj, struct intel_dpt *dpt, 428 const struct intel_fb_pin_params *pin_params, 429 struct i915_vma **out_dpt_vma, 430 struct i915_vma **out_ggtt_vma, 431 u32 *out_offset) 432 { 433 struct i915_vma *ggtt_vma; 434 435 WARN_ON(dpt); 436 437 ggtt_vma = __xe_pin_fb_vma(obj, true, pin_params); 438 if (IS_ERR(ggtt_vma)) 439 return PTR_ERR(ggtt_vma); 440 441 *out_dpt_vma = NULL; /* not used on xe */ 442 *out_ggtt_vma = ggtt_vma; 443 *out_offset = xe_ggtt_node_addr(ggtt_vma->node); 444 445 return 0; 446 } 447 448 static void xe_fb_pin_dpt_unpin(struct intel_dpt *dpt, 449 struct i915_vma *dpt_vma, 450 struct i915_vma *ggtt_vma) 451 { 452 WARN_ON(dpt || dpt_vma); 453 454 __xe_unpin_fb_vma(ggtt_vma); 455 } 456 457 static struct i915_vma * 458 xe_fb_pin_reuse_vma(struct i915_vma *old_ggtt_vma, 459 struct drm_gem_object *old_obj, 460 const struct intel_gtt_view *old_view, 461 struct drm_gem_object *new_obj, 462 const struct intel_gtt_view *new_view, 463 u32 *out_offset) 464 { 465 if (old_ggtt_vma && old_obj == new_obj && 466 !memcmp(old_view, new_view, sizeof(*new_view))) { 467 refcount_inc(&old_ggtt_vma->ref); 468 469 *out_offset = xe_ggtt_node_addr(old_ggtt_vma->node); 470 471 return old_ggtt_vma; 472 } 473 474 return NULL; 475 } 476 477 static void xe_fb_pin_get_map(struct i915_vma *vma, struct iosys_map *map) 478 { 479 *map = vma->bo->vmap; 480 } 481 482 const struct intel_display_fb_pin_interface xe_display_fb_pin_interface = { 483 .ggtt_pin = xe_fb_pin_ggtt_pin, 484 .ggtt_unpin = xe_fb_pin_ggtt_unpin, 485 .dpt_pin = xe_fb_pin_dpt_pin, 486 .dpt_unpin = xe_fb_pin_dpt_unpin, 487 .reuse_vma = xe_fb_pin_reuse_vma, 488 .get_map = xe_fb_pin_get_map, 489 }; 490