1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include <drm/ttm/ttm_bo.h> 7 8 #include "intel_display_types.h" 9 #include "intel_dpt.h" 10 #include "intel_fb.h" 11 #include "intel_fb_pin.h" 12 #include "xe_bo.h" 13 #include "xe_device.h" 14 #include "xe_ggtt.h" 15 #include "xe_gt.h" 16 #include "xe_pm.h" 17 18 static void 19 write_dpt_rotated(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, u32 bo_ofs, 20 u32 width, u32 height, u32 src_stride, u32 dst_stride) 21 { 22 struct xe_device *xe = xe_bo_device(bo); 23 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 24 u32 column, row; 25 26 /* TODO: Maybe rewrite so we can traverse the bo addresses sequentially, 27 * by writing dpt/ggtt in a different order? 28 */ 29 30 for (column = 0; column < width; column++) { 31 u32 src_idx = src_stride * (height - 1) + column + bo_ofs; 32 33 for (row = 0; row < height; row++) { 34 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, src_idx * XE_PAGE_SIZE, 35 xe->pat.idx[XE_CACHE_NONE]); 36 37 iosys_map_wr(map, *dpt_ofs, u64, pte); 38 *dpt_ofs += 8; 39 src_idx -= src_stride; 40 } 41 42 /* The DE ignores the PTEs for the padding tiles */ 43 *dpt_ofs += (dst_stride - height) * 8; 44 } 45 46 /* Align to next page */ 47 *dpt_ofs = ALIGN(*dpt_ofs, 4096); 48 } 49 50 static void 51 write_dpt_remapped(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, 52 u32 bo_ofs, u32 width, u32 height, u32 src_stride, 53 u32 dst_stride) 54 { 55 struct xe_device *xe = xe_bo_device(bo); 56 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 57 u64 (*pte_encode_bo)(struct xe_bo *bo, u64 bo_offset, u16 pat_index) 58 = ggtt->pt_ops->pte_encode_bo; 59 u32 column, row; 60 61 for (row = 0; row < height; row++) { 62 u32 src_idx = src_stride * row + bo_ofs; 63 64 for (column = 0; column < width; column++) { 65 iosys_map_wr(map, *dpt_ofs, u64, 66 pte_encode_bo(bo, src_idx * XE_PAGE_SIZE, 67 xe->pat.idx[XE_CACHE_NONE])); 68 69 *dpt_ofs += 8; 70 src_idx++; 71 } 72 73 /* The DE ignores the PTEs for the padding tiles */ 74 *dpt_ofs += (dst_stride - width) * 8; 75 } 76 77 /* Align to next page */ 78 *dpt_ofs = ALIGN(*dpt_ofs, 4096); 79 } 80 81 static int __xe_pin_fb_vma_dpt(const struct intel_framebuffer *fb, 82 const struct i915_gtt_view *view, 83 struct i915_vma *vma) 84 { 85 struct xe_device *xe = to_xe_device(fb->base.dev); 86 struct xe_tile *tile0 = xe_device_get_root_tile(xe); 87 struct xe_ggtt *ggtt = tile0->mem.ggtt; 88 struct xe_bo *bo = intel_fb_obj(&fb->base), *dpt; 89 u32 dpt_size, size = bo->ttm.base.size; 90 91 if (view->type == I915_GTT_VIEW_NORMAL) 92 dpt_size = ALIGN(size / XE_PAGE_SIZE * 8, XE_PAGE_SIZE); 93 else if (view->type == I915_GTT_VIEW_REMAPPED) 94 dpt_size = ALIGN(intel_remapped_info_size(&fb->remapped_view.gtt.remapped) * 8, 95 XE_PAGE_SIZE); 96 else 97 /* display uses 4K tiles instead of bytes here, convert to entries.. */ 98 dpt_size = ALIGN(intel_rotation_info_size(&view->rotated) * 8, 99 XE_PAGE_SIZE); 100 101 if (IS_DGFX(xe)) 102 dpt = xe_bo_create_pin_map(xe, tile0, NULL, dpt_size, 103 ttm_bo_type_kernel, 104 XE_BO_FLAG_VRAM0 | 105 XE_BO_FLAG_GGTT | 106 XE_BO_FLAG_PAGETABLE); 107 else 108 dpt = xe_bo_create_pin_map(xe, tile0, NULL, dpt_size, 109 ttm_bo_type_kernel, 110 XE_BO_FLAG_STOLEN | 111 XE_BO_FLAG_GGTT | 112 XE_BO_FLAG_PAGETABLE); 113 if (IS_ERR(dpt)) 114 dpt = xe_bo_create_pin_map(xe, tile0, NULL, dpt_size, 115 ttm_bo_type_kernel, 116 XE_BO_FLAG_SYSTEM | 117 XE_BO_FLAG_GGTT | 118 XE_BO_FLAG_PAGETABLE); 119 if (IS_ERR(dpt)) 120 return PTR_ERR(dpt); 121 122 if (view->type == I915_GTT_VIEW_NORMAL) { 123 u32 x; 124 125 for (x = 0; x < size / XE_PAGE_SIZE; x++) { 126 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, x * XE_PAGE_SIZE, 127 xe->pat.idx[XE_CACHE_NONE]); 128 129 iosys_map_wr(&dpt->vmap, x * 8, u64, pte); 130 } 131 } else if (view->type == I915_GTT_VIEW_REMAPPED) { 132 const struct intel_remapped_info *remap_info = &view->remapped; 133 u32 i, dpt_ofs = 0; 134 135 for (i = 0; i < ARRAY_SIZE(remap_info->plane); i++) 136 write_dpt_remapped(bo, &dpt->vmap, &dpt_ofs, 137 remap_info->plane[i].offset, 138 remap_info->plane[i].width, 139 remap_info->plane[i].height, 140 remap_info->plane[i].src_stride, 141 remap_info->plane[i].dst_stride); 142 143 } else { 144 const struct intel_rotation_info *rot_info = &view->rotated; 145 u32 i, dpt_ofs = 0; 146 147 for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++) 148 write_dpt_rotated(bo, &dpt->vmap, &dpt_ofs, 149 rot_info->plane[i].offset, 150 rot_info->plane[i].width, 151 rot_info->plane[i].height, 152 rot_info->plane[i].src_stride, 153 rot_info->plane[i].dst_stride); 154 } 155 156 vma->dpt = dpt; 157 vma->node = dpt->ggtt_node; 158 return 0; 159 } 160 161 static void 162 write_ggtt_rotated(struct xe_bo *bo, struct xe_ggtt *ggtt, u32 *ggtt_ofs, u32 bo_ofs, 163 u32 width, u32 height, u32 src_stride, u32 dst_stride) 164 { 165 struct xe_device *xe = xe_bo_device(bo); 166 u32 column, row; 167 168 for (column = 0; column < width; column++) { 169 u32 src_idx = src_stride * (height - 1) + column + bo_ofs; 170 171 for (row = 0; row < height; row++) { 172 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, src_idx * XE_PAGE_SIZE, 173 xe->pat.idx[XE_CACHE_NONE]); 174 175 ggtt->pt_ops->ggtt_set_pte(ggtt, *ggtt_ofs, pte); 176 *ggtt_ofs += XE_PAGE_SIZE; 177 src_idx -= src_stride; 178 } 179 180 /* The DE ignores the PTEs for the padding tiles */ 181 *ggtt_ofs += (dst_stride - height) * XE_PAGE_SIZE; 182 } 183 } 184 185 static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb, 186 const struct i915_gtt_view *view, 187 struct i915_vma *vma) 188 { 189 struct xe_bo *bo = intel_fb_obj(&fb->base); 190 struct xe_device *xe = to_xe_device(fb->base.dev); 191 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 192 u32 align; 193 int ret; 194 195 /* TODO: Consider sharing framebuffer mapping? 196 * embed i915_vma inside intel_framebuffer 197 */ 198 xe_pm_runtime_get_noresume(tile_to_xe(ggtt->tile)); 199 ret = mutex_lock_interruptible(&ggtt->lock); 200 if (ret) 201 goto out; 202 203 align = XE_PAGE_SIZE; 204 if (xe_bo_is_vram(bo) && ggtt->flags & XE_GGTT_FLAGS_64K) 205 align = max_t(u32, align, SZ_64K); 206 207 if (bo->ggtt_node.size && view->type == I915_GTT_VIEW_NORMAL) { 208 vma->node = bo->ggtt_node; 209 } else if (view->type == I915_GTT_VIEW_NORMAL) { 210 u32 x, size = bo->ttm.base.size; 211 212 ret = xe_ggtt_insert_special_node_locked(ggtt, &vma->node, size, 213 align, 0); 214 if (ret) 215 goto out_unlock; 216 217 for (x = 0; x < size; x += XE_PAGE_SIZE) { 218 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, x, 219 xe->pat.idx[XE_CACHE_NONE]); 220 221 ggtt->pt_ops->ggtt_set_pte(ggtt, vma->node.start + x, pte); 222 } 223 } else { 224 u32 i, ggtt_ofs; 225 const struct intel_rotation_info *rot_info = &view->rotated; 226 227 /* display seems to use tiles instead of bytes here, so convert it back.. */ 228 u32 size = intel_rotation_info_size(rot_info) * XE_PAGE_SIZE; 229 230 ret = xe_ggtt_insert_special_node_locked(ggtt, &vma->node, size, 231 align, 0); 232 if (ret) 233 goto out_unlock; 234 235 ggtt_ofs = vma->node.start; 236 237 for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++) 238 write_ggtt_rotated(bo, ggtt, &ggtt_ofs, 239 rot_info->plane[i].offset, 240 rot_info->plane[i].width, 241 rot_info->plane[i].height, 242 rot_info->plane[i].src_stride, 243 rot_info->plane[i].dst_stride); 244 } 245 246 out_unlock: 247 mutex_unlock(&ggtt->lock); 248 out: 249 xe_pm_runtime_put(tile_to_xe(ggtt->tile)); 250 return ret; 251 } 252 253 static struct i915_vma *__xe_pin_fb_vma(const struct intel_framebuffer *fb, 254 const struct i915_gtt_view *view) 255 { 256 struct drm_device *dev = fb->base.dev; 257 struct xe_device *xe = to_xe_device(dev); 258 struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL); 259 struct xe_bo *bo = intel_fb_obj(&fb->base); 260 int ret; 261 262 if (!vma) 263 return ERR_PTR(-ENODEV); 264 265 if (IS_DGFX(to_xe_device(bo->ttm.base.dev)) && 266 intel_fb_rc_ccs_cc_plane(&fb->base) >= 0 && 267 !(bo->flags & XE_BO_FLAG_NEEDS_CPU_ACCESS)) { 268 struct xe_tile *tile = xe_device_get_root_tile(xe); 269 270 /* 271 * If we need to able to access the clear-color value stored in 272 * the buffer, then we require that such buffers are also CPU 273 * accessible. This is important on small-bar systems where 274 * only some subset of VRAM is CPU accessible. 275 */ 276 if (tile->mem.vram.io_size < tile->mem.vram.usable_size) { 277 ret = -EINVAL; 278 goto err; 279 } 280 } 281 282 /* 283 * Pin the framebuffer, we can't use xe_bo_(un)pin functions as the 284 * assumptions are incorrect for framebuffers 285 */ 286 ret = ttm_bo_reserve(&bo->ttm, false, false, NULL); 287 if (ret) 288 goto err; 289 290 if (IS_DGFX(xe)) 291 ret = xe_bo_migrate(bo, XE_PL_VRAM0); 292 else 293 ret = xe_bo_validate(bo, NULL, true); 294 if (!ret) 295 ttm_bo_pin(&bo->ttm); 296 ttm_bo_unreserve(&bo->ttm); 297 if (ret) 298 goto err; 299 300 vma->bo = bo; 301 if (intel_fb_uses_dpt(&fb->base)) 302 ret = __xe_pin_fb_vma_dpt(fb, view, vma); 303 else 304 ret = __xe_pin_fb_vma_ggtt(fb, view, vma); 305 if (ret) 306 goto err_unpin; 307 308 /* Ensure DPT writes are flushed */ 309 xe_device_l2_flush(xe); 310 return vma; 311 312 err_unpin: 313 ttm_bo_reserve(&bo->ttm, false, false, NULL); 314 ttm_bo_unpin(&bo->ttm); 315 ttm_bo_unreserve(&bo->ttm); 316 err: 317 kfree(vma); 318 return ERR_PTR(ret); 319 } 320 321 static void __xe_unpin_fb_vma(struct i915_vma *vma) 322 { 323 struct xe_device *xe = to_xe_device(vma->bo->ttm.base.dev); 324 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 325 326 if (vma->dpt) 327 xe_bo_unpin_map_no_vm(vma->dpt); 328 else if (!drm_mm_node_allocated(&vma->bo->ggtt_node) || 329 vma->bo->ggtt_node.start != vma->node.start) 330 xe_ggtt_remove_node(ggtt, &vma->node, false); 331 332 ttm_bo_reserve(&vma->bo->ttm, false, false, NULL); 333 ttm_bo_unpin(&vma->bo->ttm); 334 ttm_bo_unreserve(&vma->bo->ttm); 335 kfree(vma); 336 } 337 338 struct i915_vma * 339 intel_fb_pin_to_ggtt(const struct drm_framebuffer *fb, 340 const struct i915_gtt_view *view, 341 unsigned int alignment, 342 unsigned int phys_alignment, 343 bool uses_fence, 344 unsigned long *out_flags) 345 { 346 *out_flags = 0; 347 348 return __xe_pin_fb_vma(to_intel_framebuffer(fb), view); 349 } 350 351 void intel_fb_unpin_vma(struct i915_vma *vma, unsigned long flags) 352 { 353 __xe_unpin_fb_vma(vma); 354 } 355 356 int intel_plane_pin_fb(struct intel_plane_state *plane_state) 357 { 358 struct drm_framebuffer *fb = plane_state->hw.fb; 359 struct xe_bo *bo = intel_fb_obj(fb); 360 struct i915_vma *vma; 361 362 /* We reject creating !SCANOUT fb's, so this is weird.. */ 363 drm_WARN_ON(bo->ttm.base.dev, !(bo->flags & XE_BO_FLAG_SCANOUT)); 364 365 vma = __xe_pin_fb_vma(to_intel_framebuffer(fb), &plane_state->view.gtt); 366 if (IS_ERR(vma)) 367 return PTR_ERR(vma); 368 369 plane_state->ggtt_vma = vma; 370 return 0; 371 } 372 373 void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state) 374 { 375 __xe_unpin_fb_vma(old_plane_state->ggtt_vma); 376 old_plane_state->ggtt_vma = NULL; 377 } 378 379 /* 380 * For Xe introduce dummy intel_dpt_create which just return NULL and 381 * intel_dpt_destroy which does nothing. 382 */ 383 struct i915_address_space *intel_dpt_create(struct intel_framebuffer *fb) 384 { 385 return NULL; 386 } 387 388 void intel_dpt_destroy(struct i915_address_space *vm) 389 { 390 return; 391 } 392