1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "i915_drv.h" 7 #include "intel_display_types.h" 8 #include "intel_dpt.h" 9 #include "intel_fb.h" 10 #include "intel_fb_pin.h" 11 #include "xe_ggtt.h" 12 #include "xe_gt.h" 13 #include "xe_pm.h" 14 15 #include <drm/ttm/ttm_bo.h> 16 17 static void 18 write_dpt_rotated(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, u32 bo_ofs, 19 u32 width, u32 height, u32 src_stride, u32 dst_stride) 20 { 21 struct xe_device *xe = xe_bo_device(bo); 22 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 23 u32 column, row; 24 25 /* TODO: Maybe rewrite so we can traverse the bo addresses sequentially, 26 * by writing dpt/ggtt in a different order? 27 */ 28 29 for (column = 0; column < width; column++) { 30 u32 src_idx = src_stride * (height - 1) + column + bo_ofs; 31 32 for (row = 0; row < height; row++) { 33 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, src_idx * XE_PAGE_SIZE, 34 xe->pat.idx[XE_CACHE_NONE]); 35 36 iosys_map_wr(map, *dpt_ofs, u64, pte); 37 *dpt_ofs += 8; 38 src_idx -= src_stride; 39 } 40 41 /* The DE ignores the PTEs for the padding tiles */ 42 *dpt_ofs += (dst_stride - height) * 8; 43 } 44 45 /* Align to next page */ 46 *dpt_ofs = ALIGN(*dpt_ofs, 4096); 47 } 48 49 static void 50 write_dpt_remapped(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, 51 u32 bo_ofs, u32 width, u32 height, u32 src_stride, 52 u32 dst_stride) 53 { 54 struct xe_device *xe = xe_bo_device(bo); 55 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 56 u64 (*pte_encode_bo)(struct xe_bo *bo, u64 bo_offset, u16 pat_index) 57 = ggtt->pt_ops->pte_encode_bo; 58 u32 column, row; 59 60 for (row = 0; row < height; row++) { 61 u32 src_idx = src_stride * row + bo_ofs; 62 63 for (column = 0; column < width; column++) { 64 iosys_map_wr(map, *dpt_ofs, u64, 65 pte_encode_bo(bo, src_idx * XE_PAGE_SIZE, 66 xe->pat.idx[XE_CACHE_NONE])); 67 68 *dpt_ofs += 8; 69 src_idx++; 70 } 71 72 /* The DE ignores the PTEs for the padding tiles */ 73 *dpt_ofs += (dst_stride - width) * 8; 74 } 75 76 /* Align to next page */ 77 *dpt_ofs = ALIGN(*dpt_ofs, 4096); 78 } 79 80 static int __xe_pin_fb_vma_dpt(struct intel_framebuffer *fb, 81 const struct i915_gtt_view *view, 82 struct i915_vma *vma) 83 { 84 struct xe_device *xe = to_xe_device(fb->base.dev); 85 struct xe_tile *tile0 = xe_device_get_root_tile(xe); 86 struct xe_ggtt *ggtt = tile0->mem.ggtt; 87 struct xe_bo *bo = intel_fb_obj(&fb->base), *dpt; 88 u32 dpt_size, size = bo->ttm.base.size; 89 90 if (view->type == I915_GTT_VIEW_NORMAL) 91 dpt_size = ALIGN(size / XE_PAGE_SIZE * 8, XE_PAGE_SIZE); 92 else if (view->type == I915_GTT_VIEW_REMAPPED) 93 dpt_size = ALIGN(intel_remapped_info_size(&fb->remapped_view.gtt.remapped) * 8, 94 XE_PAGE_SIZE); 95 else 96 /* display uses 4K tiles instead of bytes here, convert to entries.. */ 97 dpt_size = ALIGN(intel_rotation_info_size(&view->rotated) * 8, 98 XE_PAGE_SIZE); 99 100 if (IS_DGFX(xe)) 101 dpt = xe_bo_create_pin_map(xe, tile0, NULL, dpt_size, 102 ttm_bo_type_kernel, 103 XE_BO_FLAG_VRAM0 | 104 XE_BO_FLAG_GGTT | 105 XE_BO_FLAG_PAGETABLE); 106 else 107 dpt = xe_bo_create_pin_map(xe, tile0, NULL, dpt_size, 108 ttm_bo_type_kernel, 109 XE_BO_FLAG_STOLEN | 110 XE_BO_FLAG_GGTT | 111 XE_BO_FLAG_PAGETABLE); 112 if (IS_ERR(dpt)) 113 dpt = xe_bo_create_pin_map(xe, tile0, NULL, dpt_size, 114 ttm_bo_type_kernel, 115 XE_BO_FLAG_SYSTEM | 116 XE_BO_FLAG_GGTT | 117 XE_BO_FLAG_PAGETABLE); 118 if (IS_ERR(dpt)) 119 return PTR_ERR(dpt); 120 121 if (view->type == I915_GTT_VIEW_NORMAL) { 122 u32 x; 123 124 for (x = 0; x < size / XE_PAGE_SIZE; x++) { 125 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, x * XE_PAGE_SIZE, 126 xe->pat.idx[XE_CACHE_NONE]); 127 128 iosys_map_wr(&dpt->vmap, x * 8, u64, pte); 129 } 130 } else if (view->type == I915_GTT_VIEW_REMAPPED) { 131 const struct intel_remapped_info *remap_info = &view->remapped; 132 u32 i, dpt_ofs = 0; 133 134 for (i = 0; i < ARRAY_SIZE(remap_info->plane); i++) 135 write_dpt_remapped(bo, &dpt->vmap, &dpt_ofs, 136 remap_info->plane[i].offset, 137 remap_info->plane[i].width, 138 remap_info->plane[i].height, 139 remap_info->plane[i].src_stride, 140 remap_info->plane[i].dst_stride); 141 142 } else { 143 const struct intel_rotation_info *rot_info = &view->rotated; 144 u32 i, dpt_ofs = 0; 145 146 for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++) 147 write_dpt_rotated(bo, &dpt->vmap, &dpt_ofs, 148 rot_info->plane[i].offset, 149 rot_info->plane[i].width, 150 rot_info->plane[i].height, 151 rot_info->plane[i].src_stride, 152 rot_info->plane[i].dst_stride); 153 } 154 155 vma->dpt = dpt; 156 vma->node = dpt->ggtt_node; 157 return 0; 158 } 159 160 static void 161 write_ggtt_rotated(struct xe_bo *bo, struct xe_ggtt *ggtt, u32 *ggtt_ofs, u32 bo_ofs, 162 u32 width, u32 height, u32 src_stride, u32 dst_stride) 163 { 164 struct xe_device *xe = xe_bo_device(bo); 165 u32 column, row; 166 167 for (column = 0; column < width; column++) { 168 u32 src_idx = src_stride * (height - 1) + column + bo_ofs; 169 170 for (row = 0; row < height; row++) { 171 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, src_idx * XE_PAGE_SIZE, 172 xe->pat.idx[XE_CACHE_NONE]); 173 174 xe_ggtt_set_pte(ggtt, *ggtt_ofs, pte); 175 *ggtt_ofs += XE_PAGE_SIZE; 176 src_idx -= src_stride; 177 } 178 179 /* The DE ignores the PTEs for the padding tiles */ 180 *ggtt_ofs += (dst_stride - height) * XE_PAGE_SIZE; 181 } 182 } 183 184 static int __xe_pin_fb_vma_ggtt(struct intel_framebuffer *fb, 185 const struct i915_gtt_view *view, 186 struct i915_vma *vma) 187 { 188 struct xe_bo *bo = intel_fb_obj(&fb->base); 189 struct xe_device *xe = to_xe_device(fb->base.dev); 190 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 191 u32 align; 192 int ret; 193 194 /* TODO: Consider sharing framebuffer mapping? 195 * embed i915_vma inside intel_framebuffer 196 */ 197 xe_pm_runtime_get_noresume(tile_to_xe(ggtt->tile)); 198 ret = mutex_lock_interruptible(&ggtt->lock); 199 if (ret) 200 goto out; 201 202 align = XE_PAGE_SIZE; 203 if (xe_bo_is_vram(bo) && ggtt->flags & XE_GGTT_FLAGS_64K) 204 align = max_t(u32, align, SZ_64K); 205 206 if (bo->ggtt_node.size && view->type == I915_GTT_VIEW_NORMAL) { 207 vma->node = bo->ggtt_node; 208 } else if (view->type == I915_GTT_VIEW_NORMAL) { 209 u32 x, size = bo->ttm.base.size; 210 211 ret = xe_ggtt_insert_special_node_locked(ggtt, &vma->node, size, 212 align, 0); 213 if (ret) 214 goto out_unlock; 215 216 for (x = 0; x < size; x += XE_PAGE_SIZE) { 217 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, x, 218 xe->pat.idx[XE_CACHE_NONE]); 219 220 xe_ggtt_set_pte(ggtt, vma->node.start + x, pte); 221 } 222 } else { 223 u32 i, ggtt_ofs; 224 const struct intel_rotation_info *rot_info = &view->rotated; 225 226 /* display seems to use tiles instead of bytes here, so convert it back.. */ 227 u32 size = intel_rotation_info_size(rot_info) * XE_PAGE_SIZE; 228 229 ret = xe_ggtt_insert_special_node_locked(ggtt, &vma->node, size, 230 align, 0); 231 if (ret) 232 goto out_unlock; 233 234 ggtt_ofs = vma->node.start; 235 236 for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++) 237 write_ggtt_rotated(bo, ggtt, &ggtt_ofs, 238 rot_info->plane[i].offset, 239 rot_info->plane[i].width, 240 rot_info->plane[i].height, 241 rot_info->plane[i].src_stride, 242 rot_info->plane[i].dst_stride); 243 } 244 245 out_unlock: 246 mutex_unlock(&ggtt->lock); 247 out: 248 xe_pm_runtime_put(tile_to_xe(ggtt->tile)); 249 return ret; 250 } 251 252 static struct i915_vma *__xe_pin_fb_vma(struct intel_framebuffer *fb, 253 const struct i915_gtt_view *view) 254 { 255 struct drm_device *dev = fb->base.dev; 256 struct xe_device *xe = to_xe_device(dev); 257 struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL); 258 struct xe_bo *bo = intel_fb_obj(&fb->base); 259 int ret; 260 261 if (!vma) 262 return ERR_PTR(-ENODEV); 263 264 if (IS_DGFX(to_xe_device(bo->ttm.base.dev)) && 265 intel_fb_rc_ccs_cc_plane(&fb->base) >= 0 && 266 !(bo->flags & XE_BO_FLAG_NEEDS_CPU_ACCESS)) { 267 struct xe_tile *tile = xe_device_get_root_tile(xe); 268 269 /* 270 * If we need to able to access the clear-color value stored in 271 * the buffer, then we require that such buffers are also CPU 272 * accessible. This is important on small-bar systems where 273 * only some subset of VRAM is CPU accessible. 274 */ 275 if (tile->mem.vram.io_size < tile->mem.vram.usable_size) { 276 ret = -EINVAL; 277 goto err; 278 } 279 } 280 281 /* 282 * Pin the framebuffer, we can't use xe_bo_(un)pin functions as the 283 * assumptions are incorrect for framebuffers 284 */ 285 ret = ttm_bo_reserve(&bo->ttm, false, false, NULL); 286 if (ret) 287 goto err; 288 289 if (IS_DGFX(xe)) 290 ret = xe_bo_migrate(bo, XE_PL_VRAM0); 291 else 292 ret = xe_bo_validate(bo, NULL, true); 293 if (!ret) 294 ttm_bo_pin(&bo->ttm); 295 ttm_bo_unreserve(&bo->ttm); 296 if (ret) 297 goto err; 298 299 vma->bo = bo; 300 if (intel_fb_uses_dpt(&fb->base)) 301 ret = __xe_pin_fb_vma_dpt(fb, view, vma); 302 else 303 ret = __xe_pin_fb_vma_ggtt(fb, view, vma); 304 if (ret) 305 goto err_unpin; 306 307 return vma; 308 309 err_unpin: 310 ttm_bo_reserve(&bo->ttm, false, false, NULL); 311 ttm_bo_unpin(&bo->ttm); 312 ttm_bo_unreserve(&bo->ttm); 313 err: 314 kfree(vma); 315 return ERR_PTR(ret); 316 } 317 318 static void __xe_unpin_fb_vma(struct i915_vma *vma) 319 { 320 struct xe_device *xe = to_xe_device(vma->bo->ttm.base.dev); 321 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 322 323 if (vma->dpt) 324 xe_bo_unpin_map_no_vm(vma->dpt); 325 else if (!drm_mm_node_allocated(&vma->bo->ggtt_node) || 326 vma->bo->ggtt_node.start != vma->node.start) 327 xe_ggtt_remove_node(ggtt, &vma->node, false); 328 329 ttm_bo_reserve(&vma->bo->ttm, false, false, NULL); 330 ttm_bo_unpin(&vma->bo->ttm); 331 ttm_bo_unreserve(&vma->bo->ttm); 332 kfree(vma); 333 } 334 335 struct i915_vma * 336 intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, 337 bool phys_cursor, 338 const struct i915_gtt_view *view, 339 bool uses_fence, 340 unsigned long *out_flags) 341 { 342 *out_flags = 0; 343 344 return __xe_pin_fb_vma(to_intel_framebuffer(fb), view); 345 } 346 347 void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags) 348 { 349 __xe_unpin_fb_vma(vma); 350 } 351 352 int intel_plane_pin_fb(struct intel_plane_state *plane_state) 353 { 354 struct drm_framebuffer *fb = plane_state->hw.fb; 355 struct xe_bo *bo = intel_fb_obj(fb); 356 struct i915_vma *vma; 357 358 /* We reject creating !SCANOUT fb's, so this is weird.. */ 359 drm_WARN_ON(bo->ttm.base.dev, !(bo->flags & XE_BO_FLAG_SCANOUT)); 360 361 vma = __xe_pin_fb_vma(to_intel_framebuffer(fb), &plane_state->view.gtt); 362 if (IS_ERR(vma)) 363 return PTR_ERR(vma); 364 365 plane_state->ggtt_vma = vma; 366 return 0; 367 } 368 369 void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state) 370 { 371 __xe_unpin_fb_vma(old_plane_state->ggtt_vma); 372 old_plane_state->ggtt_vma = NULL; 373 } 374 375 /* 376 * For Xe introduce dummy intel_dpt_create which just return NULL and 377 * intel_dpt_destroy which does nothing. 378 */ 379 struct i915_address_space *intel_dpt_create(struct intel_framebuffer *fb) 380 { 381 return NULL; 382 } 383 384 void intel_dpt_destroy(struct i915_address_space *vm) 385 { 386 return; 387 } 388