1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include <drm/ttm/ttm_bo.h> 7 8 #include "intel_display_types.h" 9 #include "intel_dpt.h" 10 #include "intel_fb.h" 11 #include "intel_fb_pin.h" 12 #include "xe_bo.h" 13 #include "xe_device.h" 14 #include "xe_ggtt.h" 15 #include "xe_pm.h" 16 17 static void 18 write_dpt_rotated(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, u32 bo_ofs, 19 u32 width, u32 height, u32 src_stride, u32 dst_stride) 20 { 21 struct xe_device *xe = xe_bo_device(bo); 22 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 23 u32 column, row; 24 25 /* TODO: Maybe rewrite so we can traverse the bo addresses sequentially, 26 * by writing dpt/ggtt in a different order? 27 */ 28 29 for (column = 0; column < width; column++) { 30 u32 src_idx = src_stride * (height - 1) + column + bo_ofs; 31 32 for (row = 0; row < height; row++) { 33 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, src_idx * XE_PAGE_SIZE, 34 xe->pat.idx[XE_CACHE_NONE]); 35 36 iosys_map_wr(map, *dpt_ofs, u64, pte); 37 *dpt_ofs += 8; 38 src_idx -= src_stride; 39 } 40 41 /* The DE ignores the PTEs for the padding tiles */ 42 *dpt_ofs += (dst_stride - height) * 8; 43 } 44 45 /* Align to next page */ 46 *dpt_ofs = ALIGN(*dpt_ofs, 4096); 47 } 48 49 static void 50 write_dpt_remapped(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, 51 u32 bo_ofs, u32 width, u32 height, u32 src_stride, 52 u32 dst_stride) 53 { 54 struct xe_device *xe = xe_bo_device(bo); 55 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 56 u64 (*pte_encode_bo)(struct xe_bo *bo, u64 bo_offset, u16 pat_index) 57 = ggtt->pt_ops->pte_encode_bo; 58 u32 column, row; 59 60 for (row = 0; row < height; row++) { 61 u32 src_idx = src_stride * row + bo_ofs; 62 63 for (column = 0; column < width; column++) { 64 iosys_map_wr(map, *dpt_ofs, u64, 65 pte_encode_bo(bo, src_idx * XE_PAGE_SIZE, 66 xe->pat.idx[XE_CACHE_NONE])); 67 68 *dpt_ofs += 8; 69 src_idx++; 70 } 71 72 /* The DE ignores the PTEs for the padding tiles */ 73 *dpt_ofs += (dst_stride - width) * 8; 74 } 75 76 /* Align to next page */ 77 *dpt_ofs = ALIGN(*dpt_ofs, 4096); 78 } 79 80 static int __xe_pin_fb_vma_dpt(const struct intel_framebuffer *fb, 81 const struct i915_gtt_view *view, 82 struct i915_vma *vma, 83 u64 physical_alignment) 84 { 85 struct xe_device *xe = to_xe_device(fb->base.dev); 86 struct xe_tile *tile0 = xe_device_get_root_tile(xe); 87 struct xe_ggtt *ggtt = tile0->mem.ggtt; 88 struct drm_gem_object *obj = intel_fb_bo(&fb->base); 89 struct xe_bo *bo = gem_to_xe_bo(obj), *dpt; 90 u32 dpt_size, size = bo->ttm.base.size; 91 92 if (view->type == I915_GTT_VIEW_NORMAL) 93 dpt_size = ALIGN(size / XE_PAGE_SIZE * 8, XE_PAGE_SIZE); 94 else if (view->type == I915_GTT_VIEW_REMAPPED) 95 dpt_size = ALIGN(intel_remapped_info_size(&fb->remapped_view.gtt.remapped) * 8, 96 XE_PAGE_SIZE); 97 else 98 /* display uses 4K tiles instead of bytes here, convert to entries.. */ 99 dpt_size = ALIGN(intel_rotation_info_size(&view->rotated) * 8, 100 XE_PAGE_SIZE); 101 102 if (IS_DGFX(xe)) 103 dpt = xe_bo_create_pin_map_at_aligned(xe, tile0, NULL, 104 dpt_size, ~0ull, 105 ttm_bo_type_kernel, 106 XE_BO_FLAG_VRAM0 | 107 XE_BO_FLAG_GGTT | 108 XE_BO_FLAG_PAGETABLE, 109 physical_alignment); 110 else 111 dpt = xe_bo_create_pin_map_at_aligned(xe, tile0, NULL, 112 dpt_size, ~0ull, 113 ttm_bo_type_kernel, 114 XE_BO_FLAG_STOLEN | 115 XE_BO_FLAG_GGTT | 116 XE_BO_FLAG_PAGETABLE, 117 physical_alignment); 118 if (IS_ERR(dpt)) 119 dpt = xe_bo_create_pin_map_at_aligned(xe, tile0, NULL, 120 dpt_size, ~0ull, 121 ttm_bo_type_kernel, 122 XE_BO_FLAG_SYSTEM | 123 XE_BO_FLAG_GGTT | 124 XE_BO_FLAG_PAGETABLE, 125 physical_alignment); 126 if (IS_ERR(dpt)) 127 return PTR_ERR(dpt); 128 129 if (view->type == I915_GTT_VIEW_NORMAL) { 130 u32 x; 131 132 for (x = 0; x < size / XE_PAGE_SIZE; x++) { 133 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, x * XE_PAGE_SIZE, 134 xe->pat.idx[XE_CACHE_NONE]); 135 136 iosys_map_wr(&dpt->vmap, x * 8, u64, pte); 137 } 138 } else if (view->type == I915_GTT_VIEW_REMAPPED) { 139 const struct intel_remapped_info *remap_info = &view->remapped; 140 u32 i, dpt_ofs = 0; 141 142 for (i = 0; i < ARRAY_SIZE(remap_info->plane); i++) 143 write_dpt_remapped(bo, &dpt->vmap, &dpt_ofs, 144 remap_info->plane[i].offset, 145 remap_info->plane[i].width, 146 remap_info->plane[i].height, 147 remap_info->plane[i].src_stride, 148 remap_info->plane[i].dst_stride); 149 150 } else { 151 const struct intel_rotation_info *rot_info = &view->rotated; 152 u32 i, dpt_ofs = 0; 153 154 for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++) 155 write_dpt_rotated(bo, &dpt->vmap, &dpt_ofs, 156 rot_info->plane[i].offset, 157 rot_info->plane[i].width, 158 rot_info->plane[i].height, 159 rot_info->plane[i].src_stride, 160 rot_info->plane[i].dst_stride); 161 } 162 163 vma->dpt = dpt; 164 vma->node = dpt->ggtt_node; 165 return 0; 166 } 167 168 static void 169 write_ggtt_rotated(struct xe_bo *bo, struct xe_ggtt *ggtt, u32 *ggtt_ofs, u32 bo_ofs, 170 u32 width, u32 height, u32 src_stride, u32 dst_stride) 171 { 172 struct xe_device *xe = xe_bo_device(bo); 173 u32 column, row; 174 175 for (column = 0; column < width; column++) { 176 u32 src_idx = src_stride * (height - 1) + column + bo_ofs; 177 178 for (row = 0; row < height; row++) { 179 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, src_idx * XE_PAGE_SIZE, 180 xe->pat.idx[XE_CACHE_NONE]); 181 182 ggtt->pt_ops->ggtt_set_pte(ggtt, *ggtt_ofs, pte); 183 *ggtt_ofs += XE_PAGE_SIZE; 184 src_idx -= src_stride; 185 } 186 187 /* The DE ignores the PTEs for the padding tiles */ 188 *ggtt_ofs += (dst_stride - height) * XE_PAGE_SIZE; 189 } 190 } 191 192 static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb, 193 const struct i915_gtt_view *view, 194 struct i915_vma *vma, 195 u64 physical_alignment) 196 { 197 struct drm_gem_object *obj = intel_fb_bo(&fb->base); 198 struct xe_bo *bo = gem_to_xe_bo(obj); 199 struct xe_device *xe = to_xe_device(fb->base.dev); 200 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 201 u32 align; 202 int ret; 203 204 /* TODO: Consider sharing framebuffer mapping? 205 * embed i915_vma inside intel_framebuffer 206 */ 207 xe_pm_runtime_get_noresume(tile_to_xe(ggtt->tile)); 208 ret = mutex_lock_interruptible(&ggtt->lock); 209 if (ret) 210 goto out; 211 212 align = XE_PAGE_SIZE; 213 if (xe_bo_is_vram(bo) && ggtt->flags & XE_GGTT_FLAGS_64K) 214 align = max_t(u32, align, SZ_64K); 215 216 if (bo->ggtt_node && view->type == I915_GTT_VIEW_NORMAL) { 217 vma->node = bo->ggtt_node; 218 } else if (view->type == I915_GTT_VIEW_NORMAL) { 219 u32 x, size = bo->ttm.base.size; 220 221 vma->node = xe_ggtt_node_init(ggtt); 222 if (IS_ERR(vma->node)) { 223 ret = PTR_ERR(vma->node); 224 goto out_unlock; 225 } 226 227 ret = xe_ggtt_node_insert_locked(vma->node, size, align, 0); 228 if (ret) { 229 xe_ggtt_node_fini(vma->node); 230 goto out_unlock; 231 } 232 233 for (x = 0; x < size; x += XE_PAGE_SIZE) { 234 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, x, 235 xe->pat.idx[XE_CACHE_NONE]); 236 237 ggtt->pt_ops->ggtt_set_pte(ggtt, vma->node->base.start + x, pte); 238 } 239 } else { 240 u32 i, ggtt_ofs; 241 const struct intel_rotation_info *rot_info = &view->rotated; 242 243 /* display seems to use tiles instead of bytes here, so convert it back.. */ 244 u32 size = intel_rotation_info_size(rot_info) * XE_PAGE_SIZE; 245 246 vma->node = xe_ggtt_node_init(ggtt); 247 if (IS_ERR(vma->node)) { 248 ret = PTR_ERR(vma->node); 249 goto out_unlock; 250 } 251 252 ret = xe_ggtt_node_insert_locked(vma->node, size, align, 0); 253 if (ret) { 254 xe_ggtt_node_fini(vma->node); 255 goto out_unlock; 256 } 257 258 ggtt_ofs = vma->node->base.start; 259 260 for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++) 261 write_ggtt_rotated(bo, ggtt, &ggtt_ofs, 262 rot_info->plane[i].offset, 263 rot_info->plane[i].width, 264 rot_info->plane[i].height, 265 rot_info->plane[i].src_stride, 266 rot_info->plane[i].dst_stride); 267 } 268 269 out_unlock: 270 mutex_unlock(&ggtt->lock); 271 out: 272 xe_pm_runtime_put(tile_to_xe(ggtt->tile)); 273 return ret; 274 } 275 276 static struct i915_vma *__xe_pin_fb_vma(const struct intel_framebuffer *fb, 277 const struct i915_gtt_view *view, 278 u64 physical_alignment) 279 { 280 struct drm_device *dev = fb->base.dev; 281 struct xe_device *xe = to_xe_device(dev); 282 struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL); 283 struct drm_gem_object *obj = intel_fb_bo(&fb->base); 284 struct xe_bo *bo = gem_to_xe_bo(obj); 285 int ret; 286 287 if (!vma) 288 return ERR_PTR(-ENODEV); 289 290 if (IS_DGFX(to_xe_device(bo->ttm.base.dev)) && 291 intel_fb_rc_ccs_cc_plane(&fb->base) >= 0 && 292 !(bo->flags & XE_BO_FLAG_NEEDS_CPU_ACCESS)) { 293 struct xe_tile *tile = xe_device_get_root_tile(xe); 294 295 /* 296 * If we need to able to access the clear-color value stored in 297 * the buffer, then we require that such buffers are also CPU 298 * accessible. This is important on small-bar systems where 299 * only some subset of VRAM is CPU accessible. 300 */ 301 if (tile->mem.vram.io_size < tile->mem.vram.usable_size) { 302 ret = -EINVAL; 303 goto err; 304 } 305 } 306 307 /* 308 * Pin the framebuffer, we can't use xe_bo_(un)pin functions as the 309 * assumptions are incorrect for framebuffers 310 */ 311 ret = ttm_bo_reserve(&bo->ttm, false, false, NULL); 312 if (ret) 313 goto err; 314 315 if (IS_DGFX(xe)) 316 ret = xe_bo_migrate(bo, XE_PL_VRAM0); 317 else 318 ret = xe_bo_validate(bo, NULL, true); 319 if (!ret) 320 ttm_bo_pin(&bo->ttm); 321 ttm_bo_unreserve(&bo->ttm); 322 if (ret) 323 goto err; 324 325 vma->bo = bo; 326 if (intel_fb_uses_dpt(&fb->base)) 327 ret = __xe_pin_fb_vma_dpt(fb, view, vma, physical_alignment); 328 else 329 ret = __xe_pin_fb_vma_ggtt(fb, view, vma, physical_alignment); 330 if (ret) 331 goto err_unpin; 332 333 /* Ensure DPT writes are flushed */ 334 xe_device_l2_flush(xe); 335 return vma; 336 337 err_unpin: 338 ttm_bo_reserve(&bo->ttm, false, false, NULL); 339 ttm_bo_unpin(&bo->ttm); 340 ttm_bo_unreserve(&bo->ttm); 341 err: 342 kfree(vma); 343 return ERR_PTR(ret); 344 } 345 346 static void __xe_unpin_fb_vma(struct i915_vma *vma) 347 { 348 if (vma->dpt) 349 xe_bo_unpin_map_no_vm(vma->dpt); 350 else if (!xe_ggtt_node_allocated(vma->bo->ggtt_node) || 351 vma->bo->ggtt_node->base.start != vma->node->base.start) 352 xe_ggtt_node_remove(vma->node, false); 353 354 ttm_bo_reserve(&vma->bo->ttm, false, false, NULL); 355 ttm_bo_unpin(&vma->bo->ttm); 356 ttm_bo_unreserve(&vma->bo->ttm); 357 kfree(vma); 358 } 359 360 struct i915_vma * 361 intel_fb_pin_to_ggtt(const struct drm_framebuffer *fb, 362 const struct i915_gtt_view *view, 363 unsigned int alignment, 364 unsigned int phys_alignment, 365 bool uses_fence, 366 unsigned long *out_flags) 367 { 368 *out_flags = 0; 369 370 return __xe_pin_fb_vma(to_intel_framebuffer(fb), view, phys_alignment); 371 } 372 373 void intel_fb_unpin_vma(struct i915_vma *vma, unsigned long flags) 374 { 375 __xe_unpin_fb_vma(vma); 376 } 377 378 int intel_plane_pin_fb(struct intel_plane_state *plane_state) 379 { 380 struct drm_framebuffer *fb = plane_state->hw.fb; 381 struct drm_gem_object *obj = intel_fb_bo(fb); 382 struct xe_bo *bo = gem_to_xe_bo(obj); 383 struct i915_vma *vma; 384 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); 385 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); 386 u64 phys_alignment = plane->min_alignment(plane, fb, 0); 387 388 /* We reject creating !SCANOUT fb's, so this is weird.. */ 389 drm_WARN_ON(bo->ttm.base.dev, !(bo->flags & XE_BO_FLAG_SCANOUT)); 390 391 vma = __xe_pin_fb_vma(intel_fb, &plane_state->view.gtt, phys_alignment); 392 393 if (IS_ERR(vma)) 394 return PTR_ERR(vma); 395 396 plane_state->ggtt_vma = vma; 397 return 0; 398 } 399 400 void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state) 401 { 402 __xe_unpin_fb_vma(old_plane_state->ggtt_vma); 403 old_plane_state->ggtt_vma = NULL; 404 } 405 406 /* 407 * For Xe introduce dummy intel_dpt_create which just return NULL, 408 * intel_dpt_destroy which does nothing, and fake intel_dpt_ofsset returning 0; 409 */ 410 struct i915_address_space *intel_dpt_create(struct intel_framebuffer *fb) 411 { 412 return NULL; 413 } 414 415 void intel_dpt_destroy(struct i915_address_space *vm) 416 { 417 return; 418 } 419 420 u64 intel_dpt_offset(struct i915_vma *dpt_vma) 421 { 422 return 0; 423 } 424