1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include <drm/ttm/ttm_bo.h> 7 8 #include "i915_vma.h" 9 #include "intel_display_types.h" 10 #include "intel_dpt.h" 11 #include "intel_fb.h" 12 #include "intel_fb_pin.h" 13 #include "intel_fbdev.h" 14 #include "xe_bo.h" 15 #include "xe_device.h" 16 #include "xe_ggtt.h" 17 #include "xe_pm.h" 18 19 static void 20 write_dpt_rotated(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, u32 bo_ofs, 21 u32 width, u32 height, u32 src_stride, u32 dst_stride) 22 { 23 struct xe_device *xe = xe_bo_device(bo); 24 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 25 u32 column, row; 26 27 /* TODO: Maybe rewrite so we can traverse the bo addresses sequentially, 28 * by writing dpt/ggtt in a different order? 29 */ 30 31 for (column = 0; column < width; column++) { 32 u32 src_idx = src_stride * (height - 1) + column + bo_ofs; 33 34 for (row = 0; row < height; row++) { 35 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, src_idx * XE_PAGE_SIZE, 36 xe->pat.idx[XE_CACHE_NONE]); 37 38 iosys_map_wr(map, *dpt_ofs, u64, pte); 39 *dpt_ofs += 8; 40 src_idx -= src_stride; 41 } 42 43 /* The DE ignores the PTEs for the padding tiles */ 44 *dpt_ofs += (dst_stride - height) * 8; 45 } 46 47 /* Align to next page */ 48 *dpt_ofs = ALIGN(*dpt_ofs, 4096); 49 } 50 51 static void 52 write_dpt_remapped(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, 53 u32 bo_ofs, u32 width, u32 height, u32 src_stride, 54 u32 dst_stride) 55 { 56 struct xe_device *xe = xe_bo_device(bo); 57 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 58 u64 (*pte_encode_bo)(struct xe_bo *bo, u64 bo_offset, u16 pat_index) 59 = ggtt->pt_ops->pte_encode_bo; 60 u32 column, row; 61 62 for (row = 0; row < height; row++) { 63 u32 src_idx = src_stride * row + bo_ofs; 64 65 for (column = 0; column < width; column++) { 66 iosys_map_wr(map, *dpt_ofs, u64, 67 pte_encode_bo(bo, src_idx * XE_PAGE_SIZE, 68 xe->pat.idx[XE_CACHE_NONE])); 69 70 *dpt_ofs += 8; 71 src_idx++; 72 } 73 74 /* The DE ignores the PTEs for the padding tiles */ 75 *dpt_ofs += (dst_stride - width) * 8; 76 } 77 78 /* Align to next page */ 79 *dpt_ofs = ALIGN(*dpt_ofs, 4096); 80 } 81 82 static int __xe_pin_fb_vma_dpt(const struct intel_framebuffer *fb, 83 const struct i915_gtt_view *view, 84 struct i915_vma *vma, 85 unsigned int alignment) 86 { 87 struct xe_device *xe = to_xe_device(fb->base.dev); 88 struct xe_tile *tile0 = xe_device_get_root_tile(xe); 89 struct xe_ggtt *ggtt = tile0->mem.ggtt; 90 struct drm_gem_object *obj = intel_fb_bo(&fb->base); 91 struct xe_bo *bo = gem_to_xe_bo(obj), *dpt; 92 u32 dpt_size, size = bo->ttm.base.size; 93 94 if (view->type == I915_GTT_VIEW_NORMAL) 95 dpt_size = ALIGN(size / XE_PAGE_SIZE * 8, XE_PAGE_SIZE); 96 else if (view->type == I915_GTT_VIEW_REMAPPED) 97 dpt_size = ALIGN(intel_remapped_info_size(&fb->remapped_view.gtt.remapped) * 8, 98 XE_PAGE_SIZE); 99 else 100 /* display uses 4K tiles instead of bytes here, convert to entries.. */ 101 dpt_size = ALIGN(intel_rotation_info_size(&view->rotated) * 8, 102 XE_PAGE_SIZE); 103 104 if (IS_DGFX(xe)) 105 dpt = xe_bo_create_pin_map_at_aligned(xe, tile0, NULL, 106 dpt_size, ~0ull, 107 ttm_bo_type_kernel, 108 XE_BO_FLAG_VRAM0 | 109 XE_BO_FLAG_GGTT | 110 XE_BO_FLAG_PAGETABLE, 111 alignment); 112 else 113 dpt = xe_bo_create_pin_map_at_aligned(xe, tile0, NULL, 114 dpt_size, ~0ull, 115 ttm_bo_type_kernel, 116 XE_BO_FLAG_STOLEN | 117 XE_BO_FLAG_GGTT | 118 XE_BO_FLAG_PAGETABLE, 119 alignment); 120 if (IS_ERR(dpt)) 121 dpt = xe_bo_create_pin_map_at_aligned(xe, tile0, NULL, 122 dpt_size, ~0ull, 123 ttm_bo_type_kernel, 124 XE_BO_FLAG_SYSTEM | 125 XE_BO_FLAG_GGTT | 126 XE_BO_FLAG_PAGETABLE, 127 alignment); 128 if (IS_ERR(dpt)) 129 return PTR_ERR(dpt); 130 131 if (view->type == I915_GTT_VIEW_NORMAL) { 132 u32 x; 133 134 for (x = 0; x < size / XE_PAGE_SIZE; x++) { 135 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, x * XE_PAGE_SIZE, 136 xe->pat.idx[XE_CACHE_NONE]); 137 138 iosys_map_wr(&dpt->vmap, x * 8, u64, pte); 139 } 140 } else if (view->type == I915_GTT_VIEW_REMAPPED) { 141 const struct intel_remapped_info *remap_info = &view->remapped; 142 u32 i, dpt_ofs = 0; 143 144 for (i = 0; i < ARRAY_SIZE(remap_info->plane); i++) 145 write_dpt_remapped(bo, &dpt->vmap, &dpt_ofs, 146 remap_info->plane[i].offset, 147 remap_info->plane[i].width, 148 remap_info->plane[i].height, 149 remap_info->plane[i].src_stride, 150 remap_info->plane[i].dst_stride); 151 152 } else { 153 const struct intel_rotation_info *rot_info = &view->rotated; 154 u32 i, dpt_ofs = 0; 155 156 for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++) 157 write_dpt_rotated(bo, &dpt->vmap, &dpt_ofs, 158 rot_info->plane[i].offset, 159 rot_info->plane[i].width, 160 rot_info->plane[i].height, 161 rot_info->plane[i].src_stride, 162 rot_info->plane[i].dst_stride); 163 } 164 165 vma->dpt = dpt; 166 vma->node = dpt->ggtt_node[tile0->id]; 167 168 /* Ensure DPT writes are flushed */ 169 xe_device_l2_flush(xe); 170 return 0; 171 } 172 173 static void 174 write_ggtt_rotated(struct xe_bo *bo, struct xe_ggtt *ggtt, u32 *ggtt_ofs, u32 bo_ofs, 175 u32 width, u32 height, u32 src_stride, u32 dst_stride) 176 { 177 struct xe_device *xe = xe_bo_device(bo); 178 u32 column, row; 179 180 for (column = 0; column < width; column++) { 181 u32 src_idx = src_stride * (height - 1) + column + bo_ofs; 182 183 for (row = 0; row < height; row++) { 184 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, src_idx * XE_PAGE_SIZE, 185 xe->pat.idx[XE_CACHE_NONE]); 186 187 ggtt->pt_ops->ggtt_set_pte(ggtt, *ggtt_ofs, pte); 188 *ggtt_ofs += XE_PAGE_SIZE; 189 src_idx -= src_stride; 190 } 191 192 /* The DE ignores the PTEs for the padding tiles */ 193 *ggtt_ofs += (dst_stride - height) * XE_PAGE_SIZE; 194 } 195 } 196 197 static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb, 198 const struct i915_gtt_view *view, 199 struct i915_vma *vma, 200 unsigned int alignment) 201 { 202 struct drm_gem_object *obj = intel_fb_bo(&fb->base); 203 struct xe_bo *bo = gem_to_xe_bo(obj); 204 struct xe_device *xe = to_xe_device(fb->base.dev); 205 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 206 u32 align; 207 int ret; 208 209 /* TODO: Consider sharing framebuffer mapping? 210 * embed i915_vma inside intel_framebuffer 211 */ 212 xe_pm_runtime_get_noresume(tile_to_xe(ggtt->tile)); 213 ret = mutex_lock_interruptible(&ggtt->lock); 214 if (ret) 215 goto out; 216 217 align = XE_PAGE_SIZE; 218 if (xe_bo_is_vram(bo) && ggtt->flags & XE_GGTT_FLAGS_64K) 219 align = max_t(u32, align, SZ_64K); 220 221 if (bo->ggtt_node[ggtt->tile->id] && view->type == I915_GTT_VIEW_NORMAL) { 222 vma->node = bo->ggtt_node[ggtt->tile->id]; 223 } else if (view->type == I915_GTT_VIEW_NORMAL) { 224 u32 x, size = bo->ttm.base.size; 225 226 vma->node = xe_ggtt_node_init(ggtt); 227 if (IS_ERR(vma->node)) { 228 ret = PTR_ERR(vma->node); 229 goto out_unlock; 230 } 231 232 ret = xe_ggtt_node_insert_locked(vma->node, size, align, 0); 233 if (ret) { 234 xe_ggtt_node_fini(vma->node); 235 goto out_unlock; 236 } 237 238 for (x = 0; x < size; x += XE_PAGE_SIZE) { 239 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, x, 240 xe->pat.idx[XE_CACHE_NONE]); 241 242 ggtt->pt_ops->ggtt_set_pte(ggtt, vma->node->base.start + x, pte); 243 } 244 } else { 245 u32 i, ggtt_ofs; 246 const struct intel_rotation_info *rot_info = &view->rotated; 247 248 /* display seems to use tiles instead of bytes here, so convert it back.. */ 249 u32 size = intel_rotation_info_size(rot_info) * XE_PAGE_SIZE; 250 251 vma->node = xe_ggtt_node_init(ggtt); 252 if (IS_ERR(vma->node)) { 253 ret = PTR_ERR(vma->node); 254 goto out_unlock; 255 } 256 257 ret = xe_ggtt_node_insert_locked(vma->node, size, align, 0); 258 if (ret) { 259 xe_ggtt_node_fini(vma->node); 260 goto out_unlock; 261 } 262 263 ggtt_ofs = vma->node->base.start; 264 265 for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++) 266 write_ggtt_rotated(bo, ggtt, &ggtt_ofs, 267 rot_info->plane[i].offset, 268 rot_info->plane[i].width, 269 rot_info->plane[i].height, 270 rot_info->plane[i].src_stride, 271 rot_info->plane[i].dst_stride); 272 } 273 274 out_unlock: 275 mutex_unlock(&ggtt->lock); 276 out: 277 xe_pm_runtime_put(tile_to_xe(ggtt->tile)); 278 return ret; 279 } 280 281 static struct i915_vma *__xe_pin_fb_vma(const struct intel_framebuffer *fb, 282 const struct i915_gtt_view *view, 283 unsigned int alignment) 284 { 285 struct drm_device *dev = fb->base.dev; 286 struct xe_device *xe = to_xe_device(dev); 287 struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL); 288 struct drm_gem_object *obj = intel_fb_bo(&fb->base); 289 struct xe_bo *bo = gem_to_xe_bo(obj); 290 int ret; 291 292 if (!vma) 293 return ERR_PTR(-ENODEV); 294 295 refcount_set(&vma->ref, 1); 296 if (IS_DGFX(to_xe_device(bo->ttm.base.dev)) && 297 intel_fb_rc_ccs_cc_plane(&fb->base) >= 0 && 298 !(bo->flags & XE_BO_FLAG_NEEDS_CPU_ACCESS)) { 299 struct xe_tile *tile = xe_device_get_root_tile(xe); 300 301 /* 302 * If we need to able to access the clear-color value stored in 303 * the buffer, then we require that such buffers are also CPU 304 * accessible. This is important on small-bar systems where 305 * only some subset of VRAM is CPU accessible. 306 */ 307 if (tile->mem.vram.io_size < tile->mem.vram.usable_size) { 308 ret = -EINVAL; 309 goto err; 310 } 311 } 312 313 /* 314 * Pin the framebuffer, we can't use xe_bo_(un)pin functions as the 315 * assumptions are incorrect for framebuffers 316 */ 317 ret = ttm_bo_reserve(&bo->ttm, false, false, NULL); 318 if (ret) 319 goto err; 320 321 if (IS_DGFX(xe)) 322 ret = xe_bo_migrate(bo, XE_PL_VRAM0); 323 else 324 ret = xe_bo_validate(bo, NULL, true); 325 if (!ret) 326 ttm_bo_pin(&bo->ttm); 327 ttm_bo_unreserve(&bo->ttm); 328 if (ret) 329 goto err; 330 331 vma->bo = bo; 332 if (intel_fb_uses_dpt(&fb->base)) 333 ret = __xe_pin_fb_vma_dpt(fb, view, vma, alignment); 334 else 335 ret = __xe_pin_fb_vma_ggtt(fb, view, vma, alignment); 336 if (ret) 337 goto err_unpin; 338 339 return vma; 340 341 err_unpin: 342 ttm_bo_reserve(&bo->ttm, false, false, NULL); 343 ttm_bo_unpin(&bo->ttm); 344 ttm_bo_unreserve(&bo->ttm); 345 err: 346 kfree(vma); 347 return ERR_PTR(ret); 348 } 349 350 static void __xe_unpin_fb_vma(struct i915_vma *vma) 351 { 352 u8 tile_id = vma->node->ggtt->tile->id; 353 354 if (!refcount_dec_and_test(&vma->ref)) 355 return; 356 357 if (vma->dpt) 358 xe_bo_unpin_map_no_vm(vma->dpt); 359 else if (!xe_ggtt_node_allocated(vma->bo->ggtt_node[tile_id]) || 360 vma->bo->ggtt_node[tile_id]->base.start != vma->node->base.start) 361 xe_ggtt_node_remove(vma->node, false); 362 363 ttm_bo_reserve(&vma->bo->ttm, false, false, NULL); 364 ttm_bo_unpin(&vma->bo->ttm); 365 ttm_bo_unreserve(&vma->bo->ttm); 366 kfree(vma); 367 } 368 369 struct i915_vma * 370 intel_fb_pin_to_ggtt(const struct drm_framebuffer *fb, 371 const struct i915_gtt_view *view, 372 unsigned int alignment, 373 unsigned int phys_alignment, 374 unsigned int vtd_guard, 375 bool uses_fence, 376 unsigned long *out_flags) 377 { 378 *out_flags = 0; 379 380 return __xe_pin_fb_vma(to_intel_framebuffer(fb), view, phys_alignment); 381 } 382 383 void intel_fb_unpin_vma(struct i915_vma *vma, unsigned long flags) 384 { 385 __xe_unpin_fb_vma(vma); 386 } 387 388 static bool reuse_vma(struct intel_plane_state *new_plane_state, 389 const struct intel_plane_state *old_plane_state) 390 { 391 struct intel_framebuffer *fb = to_intel_framebuffer(new_plane_state->hw.fb); 392 struct xe_device *xe = to_xe_device(fb->base.dev); 393 struct i915_vma *vma; 394 395 if (old_plane_state->hw.fb == new_plane_state->hw.fb && 396 !memcmp(&old_plane_state->view.gtt, 397 &new_plane_state->view.gtt, 398 sizeof(new_plane_state->view.gtt))) { 399 vma = old_plane_state->ggtt_vma; 400 goto found; 401 } 402 403 if (fb == intel_fbdev_framebuffer(xe->display.fbdev.fbdev)) { 404 vma = intel_fbdev_vma_pointer(xe->display.fbdev.fbdev); 405 if (vma) 406 goto found; 407 } 408 409 return false; 410 411 found: 412 refcount_inc(&vma->ref); 413 new_plane_state->ggtt_vma = vma; 414 return true; 415 } 416 417 int intel_plane_pin_fb(struct intel_plane_state *new_plane_state, 418 const struct intel_plane_state *old_plane_state) 419 { 420 struct drm_framebuffer *fb = new_plane_state->hw.fb; 421 struct drm_gem_object *obj = intel_fb_bo(fb); 422 struct xe_bo *bo = gem_to_xe_bo(obj); 423 struct i915_vma *vma; 424 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); 425 struct intel_plane *plane = to_intel_plane(new_plane_state->uapi.plane); 426 unsigned int alignment = plane->min_alignment(plane, fb, 0); 427 428 if (reuse_vma(new_plane_state, old_plane_state)) 429 return 0; 430 431 /* We reject creating !SCANOUT fb's, so this is weird.. */ 432 drm_WARN_ON(bo->ttm.base.dev, !(bo->flags & XE_BO_FLAG_SCANOUT)); 433 434 vma = __xe_pin_fb_vma(intel_fb, &new_plane_state->view.gtt, alignment); 435 436 if (IS_ERR(vma)) 437 return PTR_ERR(vma); 438 439 new_plane_state->ggtt_vma = vma; 440 return 0; 441 } 442 443 void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state) 444 { 445 __xe_unpin_fb_vma(old_plane_state->ggtt_vma); 446 old_plane_state->ggtt_vma = NULL; 447 } 448 449 /* 450 * For Xe introduce dummy intel_dpt_create which just return NULL, 451 * intel_dpt_destroy which does nothing, and fake intel_dpt_ofsset returning 0; 452 */ 453 struct i915_address_space *intel_dpt_create(struct intel_framebuffer *fb) 454 { 455 return NULL; 456 } 457 458 void intel_dpt_destroy(struct i915_address_space *vm) 459 { 460 return; 461 } 462 463 u64 intel_dpt_offset(struct i915_vma *dpt_vma) 464 { 465 return 0; 466 } 467