1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "i915_drv.h" 7 #include "intel_display_types.h" 8 #include "intel_dpt.h" 9 #include "intel_fb.h" 10 #include "intel_fb_pin.h" 11 #include "xe_ggtt.h" 12 #include "xe_gt.h" 13 14 #include <drm/ttm/ttm_bo.h> 15 16 static void 17 write_dpt_rotated(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, u32 bo_ofs, 18 u32 width, u32 height, u32 src_stride, u32 dst_stride) 19 { 20 struct xe_device *xe = xe_bo_device(bo); 21 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 22 u32 column, row; 23 24 /* TODO: Maybe rewrite so we can traverse the bo addresses sequentially, 25 * by writing dpt/ggtt in a different order? 26 */ 27 28 for (column = 0; column < width; column++) { 29 u32 src_idx = src_stride * (height - 1) + column + bo_ofs; 30 31 for (row = 0; row < height; row++) { 32 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, src_idx * XE_PAGE_SIZE, 33 xe->pat.idx[XE_CACHE_WB]); 34 35 iosys_map_wr(map, *dpt_ofs, u64, pte); 36 *dpt_ofs += 8; 37 src_idx -= src_stride; 38 } 39 40 /* The DE ignores the PTEs for the padding tiles */ 41 *dpt_ofs += (dst_stride - height) * 8; 42 } 43 44 /* Align to next page */ 45 *dpt_ofs = ALIGN(*dpt_ofs, 4096); 46 } 47 48 static void 49 write_dpt_remapped(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, 50 u32 bo_ofs, u32 width, u32 height, u32 src_stride, 51 u32 dst_stride) 52 { 53 struct xe_device *xe = xe_bo_device(bo); 54 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 55 u64 (*pte_encode_bo)(struct xe_bo *bo, u64 bo_offset, u16 pat_index) 56 = ggtt->pt_ops->pte_encode_bo; 57 u32 column, row; 58 59 for (row = 0; row < height; row++) { 60 u32 src_idx = src_stride * row + bo_ofs; 61 62 for (column = 0; column < width; column++) { 63 iosys_map_wr(map, *dpt_ofs, u64, 64 pte_encode_bo(bo, src_idx * XE_PAGE_SIZE, 65 xe->pat.idx[XE_CACHE_WB])); 66 67 *dpt_ofs += 8; 68 src_idx++; 69 } 70 71 /* The DE ignores the PTEs for the padding tiles */ 72 *dpt_ofs += (dst_stride - width) * 8; 73 } 74 75 /* Align to next page */ 76 *dpt_ofs = ALIGN(*dpt_ofs, 4096); 77 } 78 79 static int __xe_pin_fb_vma_dpt(struct intel_framebuffer *fb, 80 const struct i915_gtt_view *view, 81 struct i915_vma *vma) 82 { 83 struct xe_device *xe = to_xe_device(fb->base.dev); 84 struct xe_tile *tile0 = xe_device_get_root_tile(xe); 85 struct xe_ggtt *ggtt = tile0->mem.ggtt; 86 struct xe_bo *bo = intel_fb_obj(&fb->base), *dpt; 87 u32 dpt_size, size = bo->ttm.base.size; 88 89 if (view->type == I915_GTT_VIEW_NORMAL) 90 dpt_size = ALIGN(size / XE_PAGE_SIZE * 8, XE_PAGE_SIZE); 91 else if (view->type == I915_GTT_VIEW_REMAPPED) 92 dpt_size = ALIGN(intel_remapped_info_size(&fb->remapped_view.gtt.remapped) * 8, 93 XE_PAGE_SIZE); 94 else 95 /* display uses 4K tiles instead of bytes here, convert to entries.. */ 96 dpt_size = ALIGN(intel_rotation_info_size(&view->rotated) * 8, 97 XE_PAGE_SIZE); 98 99 if (IS_DGFX(xe)) 100 dpt = xe_bo_create_pin_map(xe, tile0, NULL, dpt_size, 101 ttm_bo_type_kernel, 102 XE_BO_CREATE_VRAM0_BIT | 103 XE_BO_CREATE_GGTT_BIT); 104 else 105 dpt = xe_bo_create_pin_map(xe, tile0, NULL, dpt_size, 106 ttm_bo_type_kernel, 107 XE_BO_CREATE_STOLEN_BIT | 108 XE_BO_CREATE_GGTT_BIT); 109 if (IS_ERR(dpt)) 110 dpt = xe_bo_create_pin_map(xe, tile0, NULL, dpt_size, 111 ttm_bo_type_kernel, 112 XE_BO_CREATE_SYSTEM_BIT | 113 XE_BO_CREATE_GGTT_BIT); 114 if (IS_ERR(dpt)) 115 return PTR_ERR(dpt); 116 117 if (view->type == I915_GTT_VIEW_NORMAL) { 118 u32 x; 119 120 for (x = 0; x < size / XE_PAGE_SIZE; x++) { 121 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, x * XE_PAGE_SIZE, 122 xe->pat.idx[XE_CACHE_WB]); 123 124 iosys_map_wr(&dpt->vmap, x * 8, u64, pte); 125 } 126 } else if (view->type == I915_GTT_VIEW_REMAPPED) { 127 const struct intel_remapped_info *remap_info = &view->remapped; 128 u32 i, dpt_ofs = 0; 129 130 for (i = 0; i < ARRAY_SIZE(remap_info->plane); i++) 131 write_dpt_remapped(bo, &dpt->vmap, &dpt_ofs, 132 remap_info->plane[i].offset, 133 remap_info->plane[i].width, 134 remap_info->plane[i].height, 135 remap_info->plane[i].src_stride, 136 remap_info->plane[i].dst_stride); 137 138 } else { 139 const struct intel_rotation_info *rot_info = &view->rotated; 140 u32 i, dpt_ofs = 0; 141 142 for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++) 143 write_dpt_rotated(bo, &dpt->vmap, &dpt_ofs, 144 rot_info->plane[i].offset, 145 rot_info->plane[i].width, 146 rot_info->plane[i].height, 147 rot_info->plane[i].src_stride, 148 rot_info->plane[i].dst_stride); 149 } 150 151 vma->dpt = dpt; 152 vma->node = dpt->ggtt_node; 153 return 0; 154 } 155 156 static void 157 write_ggtt_rotated(struct xe_bo *bo, struct xe_ggtt *ggtt, u32 *ggtt_ofs, u32 bo_ofs, 158 u32 width, u32 height, u32 src_stride, u32 dst_stride) 159 { 160 struct xe_device *xe = xe_bo_device(bo); 161 u32 column, row; 162 163 for (column = 0; column < width; column++) { 164 u32 src_idx = src_stride * (height - 1) + column + bo_ofs; 165 166 for (row = 0; row < height; row++) { 167 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, src_idx * XE_PAGE_SIZE, 168 xe->pat.idx[XE_CACHE_WB]); 169 170 xe_ggtt_set_pte(ggtt, *ggtt_ofs, pte); 171 *ggtt_ofs += XE_PAGE_SIZE; 172 src_idx -= src_stride; 173 } 174 175 /* The DE ignores the PTEs for the padding tiles */ 176 *ggtt_ofs += (dst_stride - height) * XE_PAGE_SIZE; 177 } 178 } 179 180 static int __xe_pin_fb_vma_ggtt(struct intel_framebuffer *fb, 181 const struct i915_gtt_view *view, 182 struct i915_vma *vma) 183 { 184 struct xe_bo *bo = intel_fb_obj(&fb->base); 185 struct xe_device *xe = to_xe_device(fb->base.dev); 186 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 187 u32 align; 188 int ret; 189 190 /* TODO: Consider sharing framebuffer mapping? 191 * embed i915_vma inside intel_framebuffer 192 */ 193 xe_device_mem_access_get(tile_to_xe(ggtt->tile)); 194 ret = mutex_lock_interruptible(&ggtt->lock); 195 if (ret) 196 goto out; 197 198 align = XE_PAGE_SIZE; 199 if (xe_bo_is_vram(bo) && ggtt->flags & XE_GGTT_FLAGS_64K) 200 align = max_t(u32, align, SZ_64K); 201 202 if (bo->ggtt_node.size && view->type == I915_GTT_VIEW_NORMAL) { 203 vma->node = bo->ggtt_node; 204 } else if (view->type == I915_GTT_VIEW_NORMAL) { 205 u32 x, size = bo->ttm.base.size; 206 207 ret = xe_ggtt_insert_special_node_locked(ggtt, &vma->node, size, 208 align, 0); 209 if (ret) 210 goto out_unlock; 211 212 for (x = 0; x < size; x += XE_PAGE_SIZE) { 213 u64 pte = ggtt->pt_ops->pte_encode_bo(bo, x, 214 xe->pat.idx[XE_CACHE_WB]); 215 216 xe_ggtt_set_pte(ggtt, vma->node.start + x, pte); 217 } 218 } else { 219 u32 i, ggtt_ofs; 220 const struct intel_rotation_info *rot_info = &view->rotated; 221 222 /* display seems to use tiles instead of bytes here, so convert it back.. */ 223 u32 size = intel_rotation_info_size(rot_info) * XE_PAGE_SIZE; 224 225 ret = xe_ggtt_insert_special_node_locked(ggtt, &vma->node, size, 226 align, 0); 227 if (ret) 228 goto out_unlock; 229 230 ggtt_ofs = vma->node.start; 231 232 for (i = 0; i < ARRAY_SIZE(rot_info->plane); i++) 233 write_ggtt_rotated(bo, ggtt, &ggtt_ofs, 234 rot_info->plane[i].offset, 235 rot_info->plane[i].width, 236 rot_info->plane[i].height, 237 rot_info->plane[i].src_stride, 238 rot_info->plane[i].dst_stride); 239 } 240 241 xe_ggtt_invalidate(ggtt); 242 out_unlock: 243 mutex_unlock(&ggtt->lock); 244 out: 245 xe_device_mem_access_put(tile_to_xe(ggtt->tile)); 246 return ret; 247 } 248 249 static struct i915_vma *__xe_pin_fb_vma(struct intel_framebuffer *fb, 250 const struct i915_gtt_view *view) 251 { 252 struct drm_device *dev = fb->base.dev; 253 struct xe_device *xe = to_xe_device(dev); 254 struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL); 255 struct xe_bo *bo = intel_fb_obj(&fb->base); 256 int ret; 257 258 if (!vma) 259 return ERR_PTR(-ENODEV); 260 261 if (IS_DGFX(to_xe_device(bo->ttm.base.dev)) && 262 intel_fb_rc_ccs_cc_plane(&fb->base) >= 0 && 263 !(bo->flags & XE_BO_NEEDS_CPU_ACCESS)) { 264 struct xe_tile *tile = xe_device_get_root_tile(xe); 265 266 /* 267 * If we need to able to access the clear-color value stored in 268 * the buffer, then we require that such buffers are also CPU 269 * accessible. This is important on small-bar systems where 270 * only some subset of VRAM is CPU accessible. 271 */ 272 if (tile->mem.vram.io_size < tile->mem.vram.usable_size) { 273 ret = -EINVAL; 274 goto err; 275 } 276 } 277 278 /* 279 * Pin the framebuffer, we can't use xe_bo_(un)pin functions as the 280 * assumptions are incorrect for framebuffers 281 */ 282 ret = ttm_bo_reserve(&bo->ttm, false, false, NULL); 283 if (ret) 284 goto err; 285 286 if (IS_DGFX(xe)) 287 ret = xe_bo_migrate(bo, XE_PL_VRAM0); 288 else 289 ret = xe_bo_validate(bo, NULL, true); 290 if (!ret) 291 ttm_bo_pin(&bo->ttm); 292 ttm_bo_unreserve(&bo->ttm); 293 if (ret) 294 goto err; 295 296 vma->bo = bo; 297 if (intel_fb_uses_dpt(&fb->base)) 298 ret = __xe_pin_fb_vma_dpt(fb, view, vma); 299 else 300 ret = __xe_pin_fb_vma_ggtt(fb, view, vma); 301 if (ret) 302 goto err_unpin; 303 304 return vma; 305 306 err_unpin: 307 ttm_bo_reserve(&bo->ttm, false, false, NULL); 308 ttm_bo_unpin(&bo->ttm); 309 ttm_bo_unreserve(&bo->ttm); 310 err: 311 kfree(vma); 312 return ERR_PTR(ret); 313 } 314 315 static void __xe_unpin_fb_vma(struct i915_vma *vma) 316 { 317 struct xe_device *xe = to_xe_device(vma->bo->ttm.base.dev); 318 struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt; 319 320 if (vma->dpt) 321 xe_bo_unpin_map_no_vm(vma->dpt); 322 else if (!drm_mm_node_allocated(&vma->bo->ggtt_node) || 323 vma->bo->ggtt_node.start != vma->node.start) 324 xe_ggtt_remove_node(ggtt, &vma->node); 325 326 ttm_bo_reserve(&vma->bo->ttm, false, false, NULL); 327 ttm_bo_unpin(&vma->bo->ttm); 328 ttm_bo_unreserve(&vma->bo->ttm); 329 kfree(vma); 330 } 331 332 struct i915_vma * 333 intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, 334 bool phys_cursor, 335 const struct i915_gtt_view *view, 336 bool uses_fence, 337 unsigned long *out_flags) 338 { 339 *out_flags = 0; 340 341 return __xe_pin_fb_vma(to_intel_framebuffer(fb), view); 342 } 343 344 void intel_unpin_fb_vma(struct i915_vma *vma, unsigned long flags) 345 { 346 __xe_unpin_fb_vma(vma); 347 } 348 349 int intel_plane_pin_fb(struct intel_plane_state *plane_state) 350 { 351 struct drm_framebuffer *fb = plane_state->hw.fb; 352 struct xe_bo *bo = intel_fb_obj(fb); 353 struct i915_vma *vma; 354 355 /* We reject creating !SCANOUT fb's, so this is weird.. */ 356 drm_WARN_ON(bo->ttm.base.dev, !(bo->flags & XE_BO_SCANOUT_BIT)); 357 358 vma = __xe_pin_fb_vma(to_intel_framebuffer(fb), &plane_state->view.gtt); 359 if (IS_ERR(vma)) 360 return PTR_ERR(vma); 361 362 plane_state->ggtt_vma = vma; 363 return 0; 364 } 365 366 void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state) 367 { 368 __xe_unpin_fb_vma(old_plane_state->ggtt_vma); 369 old_plane_state->ggtt_vma = NULL; 370 } 371 372 /* 373 * For Xe introduce dummy intel_dpt_create which just return NULL and 374 * intel_dpt_destroy which does nothing. 375 */ 376 struct i915_address_space *intel_dpt_create(struct intel_framebuffer *fb) 377 { 378 return NULL; 379 } 380 381 void intel_dpt_destroy(struct i915_address_space *vm) 382 { 383 return; 384 }