1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_ggtt.h" 7 8 #include <linux/sizes.h> 9 10 #include <drm/drm_managed.h> 11 #include <drm/i915_drm.h> 12 13 #include "regs/xe_gt_regs.h" 14 #include "regs/xe_regs.h" 15 #include "xe_bo.h" 16 #include "xe_device.h" 17 #include "xe_gt.h" 18 #include "xe_gt_tlb_invalidation.h" 19 #include "xe_map.h" 20 #include "xe_mmio.h" 21 #include "xe_wopcm.h" 22 23 #define XELPG_GGTT_PTE_PAT0 BIT_ULL(52) 24 #define XELPG_GGTT_PTE_PAT1 BIT_ULL(53) 25 26 /* GuC addresses above GUC_GGTT_TOP also don't map through the GTT */ 27 #define GUC_GGTT_TOP 0xFEE00000 28 29 static u64 xelp_ggtt_pte_encode_bo(struct xe_bo *bo, u64 bo_offset, 30 u16 pat_index) 31 { 32 u64 pte; 33 34 pte = xe_bo_addr(bo, bo_offset, XE_PAGE_SIZE); 35 pte |= XE_PAGE_PRESENT; 36 37 if (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo)) 38 pte |= XE_GGTT_PTE_DM; 39 40 return pte; 41 } 42 43 static u64 xelpg_ggtt_pte_encode_bo(struct xe_bo *bo, u64 bo_offset, 44 u16 pat_index) 45 { 46 struct xe_device *xe = xe_bo_device(bo); 47 u64 pte; 48 49 pte = xelp_ggtt_pte_encode_bo(bo, bo_offset, pat_index); 50 51 xe_assert(xe, pat_index <= 3); 52 53 if (pat_index & BIT(0)) 54 pte |= XELPG_GGTT_PTE_PAT0; 55 56 if (pat_index & BIT(1)) 57 pte |= XELPG_GGTT_PTE_PAT1; 58 59 return pte; 60 } 61 62 static unsigned int probe_gsm_size(struct pci_dev *pdev) 63 { 64 u16 gmch_ctl, ggms; 65 66 pci_read_config_word(pdev, SNB_GMCH_CTRL, &gmch_ctl); 67 ggms = (gmch_ctl >> BDW_GMCH_GGMS_SHIFT) & BDW_GMCH_GGMS_MASK; 68 return ggms ? SZ_1M << ggms : 0; 69 } 70 71 void xe_ggtt_set_pte(struct xe_ggtt *ggtt, u64 addr, u64 pte) 72 { 73 xe_tile_assert(ggtt->tile, !(addr & XE_PTE_MASK)); 74 xe_tile_assert(ggtt->tile, addr < ggtt->size); 75 76 writeq(pte, &ggtt->gsm[addr >> XE_PTE_SHIFT]); 77 } 78 79 static void xe_ggtt_clear(struct xe_ggtt *ggtt, u64 start, u64 size) 80 { 81 u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[XE_CACHE_WB]; 82 u64 end = start + size - 1; 83 u64 scratch_pte; 84 85 xe_tile_assert(ggtt->tile, start < end); 86 87 if (ggtt->scratch) 88 scratch_pte = ggtt->pt_ops->pte_encode_bo(ggtt->scratch, 0, 89 pat_index); 90 else 91 scratch_pte = 0; 92 93 while (start < end) { 94 xe_ggtt_set_pte(ggtt, start, scratch_pte); 95 start += XE_PAGE_SIZE; 96 } 97 } 98 99 static void ggtt_fini_noalloc(struct drm_device *drm, void *arg) 100 { 101 struct xe_ggtt *ggtt = arg; 102 103 mutex_destroy(&ggtt->lock); 104 drm_mm_takedown(&ggtt->mm); 105 106 xe_bo_unpin_map_no_vm(ggtt->scratch); 107 } 108 109 static void primelockdep(struct xe_ggtt *ggtt) 110 { 111 if (!IS_ENABLED(CONFIG_LOCKDEP)) 112 return; 113 114 fs_reclaim_acquire(GFP_KERNEL); 115 might_lock(&ggtt->lock); 116 fs_reclaim_release(GFP_KERNEL); 117 } 118 119 static const struct xe_ggtt_pt_ops xelp_pt_ops = { 120 .pte_encode_bo = xelp_ggtt_pte_encode_bo, 121 }; 122 123 static const struct xe_ggtt_pt_ops xelpg_pt_ops = { 124 .pte_encode_bo = xelpg_ggtt_pte_encode_bo, 125 }; 126 127 int xe_ggtt_init_noalloc(struct xe_ggtt *ggtt) 128 { 129 struct xe_device *xe = tile_to_xe(ggtt->tile); 130 struct pci_dev *pdev = to_pci_dev(xe->drm.dev); 131 unsigned int gsm_size; 132 133 gsm_size = probe_gsm_size(pdev); 134 if (gsm_size == 0) { 135 drm_err(&xe->drm, "Hardware reported no preallocated GSM\n"); 136 return -ENOMEM; 137 } 138 139 ggtt->gsm = ggtt->tile->mmio.regs + SZ_8M; 140 ggtt->size = (gsm_size / 8) * (u64) XE_PAGE_SIZE; 141 142 if (IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) 143 ggtt->flags |= XE_GGTT_FLAGS_64K; 144 145 /* 146 * 8B per entry, each points to a 4KB page. 147 * 148 * The GuC address space is limited on both ends of the GGTT, because 149 * the GuC shim HW redirects accesses to those addresses to other HW 150 * areas instead of going through the GGTT. On the bottom end, the GuC 151 * can't access offsets below the WOPCM size, while on the top side the 152 * limit is fixed at GUC_GGTT_TOP. To keep things simple, instead of 153 * checking each object to see if they are accessed by GuC or not, we 154 * just exclude those areas from the allocator. Additionally, to 155 * simplify the driver load, we use the maximum WOPCM size in this logic 156 * instead of the programmed one, so we don't need to wait until the 157 * actual size to be programmed is determined (which requires FW fetch) 158 * before initializing the GGTT. These simplifications might waste space 159 * in the GGTT (about 20-25 MBs depending on the platform) but we can 160 * live with this. 161 * 162 * Another benifit of this is the GuC bootrom can't access anything 163 * below the WOPCM max size so anything the bootom needs to access (e.g. 164 * a RSA key) needs to be placed in the GGTT above the WOPCM max size. 165 * Starting the GGTT allocations above the WOPCM max give us the correct 166 * placement for free. 167 */ 168 if (ggtt->size > GUC_GGTT_TOP) 169 ggtt->size = GUC_GGTT_TOP; 170 171 if (GRAPHICS_VERx100(xe) >= 1270) 172 ggtt->pt_ops = &xelpg_pt_ops; 173 else 174 ggtt->pt_ops = &xelp_pt_ops; 175 176 drm_mm_init(&ggtt->mm, xe_wopcm_size(xe), 177 ggtt->size - xe_wopcm_size(xe)); 178 mutex_init(&ggtt->lock); 179 primelockdep(ggtt); 180 181 return drmm_add_action_or_reset(&xe->drm, ggtt_fini_noalloc, ggtt); 182 } 183 184 static void xe_ggtt_initial_clear(struct xe_ggtt *ggtt) 185 { 186 struct drm_mm_node *hole; 187 u64 start, end; 188 189 /* Display may have allocated inside ggtt, so be careful with clearing here */ 190 xe_device_mem_access_get(tile_to_xe(ggtt->tile)); 191 mutex_lock(&ggtt->lock); 192 drm_mm_for_each_hole(hole, &ggtt->mm, start, end) 193 xe_ggtt_clear(ggtt, start, end - start); 194 195 xe_ggtt_invalidate(ggtt); 196 mutex_unlock(&ggtt->lock); 197 xe_device_mem_access_put(tile_to_xe(ggtt->tile)); 198 } 199 200 int xe_ggtt_init(struct xe_ggtt *ggtt) 201 { 202 struct xe_device *xe = tile_to_xe(ggtt->tile); 203 unsigned int flags; 204 int err; 205 206 /* 207 * So we don't need to worry about 64K GGTT layout when dealing with 208 * scratch entires, rather keep the scratch page in system memory on 209 * platforms where 64K pages are needed for VRAM. 210 */ 211 flags = XE_BO_CREATE_PINNED_BIT; 212 if (ggtt->flags & XE_GGTT_FLAGS_64K) 213 flags |= XE_BO_CREATE_SYSTEM_BIT; 214 else 215 flags |= XE_BO_CREATE_VRAM_IF_DGFX(ggtt->tile); 216 217 ggtt->scratch = xe_bo_create_pin_map(xe, ggtt->tile, NULL, XE_PAGE_SIZE, 218 ttm_bo_type_kernel, 219 flags); 220 221 if (IS_ERR(ggtt->scratch)) { 222 err = PTR_ERR(ggtt->scratch); 223 goto err; 224 } 225 226 xe_map_memset(xe, &ggtt->scratch->vmap, 0, 0, ggtt->scratch->size); 227 228 xe_ggtt_initial_clear(ggtt); 229 return 0; 230 err: 231 ggtt->scratch = NULL; 232 return err; 233 } 234 235 #define GUC_TLB_INV_CR XE_REG(0xcee8) 236 #define GUC_TLB_INV_CR_INVALIDATE REG_BIT(0) 237 #define PVC_GUC_TLB_INV_DESC0 XE_REG(0xcf7c) 238 #define PVC_GUC_TLB_INV_DESC0_VALID REG_BIT(0) 239 #define PVC_GUC_TLB_INV_DESC1 XE_REG(0xcf80) 240 #define PVC_GUC_TLB_INV_DESC1_INVALIDATE REG_BIT(6) 241 242 static void ggtt_invalidate_gt_tlb(struct xe_gt *gt) 243 { 244 if (!gt) 245 return; 246 247 /* 248 * Invalidation can happen when there's no in-flight work keeping the 249 * GT awake. We need to explicitly grab forcewake to ensure the GT 250 * and GuC are accessible. 251 */ 252 xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 253 254 /* TODO: vfunc for GuC vs. non-GuC */ 255 256 if (gt->uc.guc.submission_state.enabled) { 257 int seqno; 258 259 seqno = xe_gt_tlb_invalidation_guc(gt); 260 xe_gt_assert(gt, seqno > 0); 261 if (seqno > 0) 262 xe_gt_tlb_invalidation_wait(gt, seqno); 263 } else if (xe_device_uc_enabled(gt_to_xe(gt))) { 264 struct xe_device *xe = gt_to_xe(gt); 265 266 if (xe->info.platform == XE_PVC || GRAPHICS_VER(xe) >= 20) { 267 xe_mmio_write32(gt, PVC_GUC_TLB_INV_DESC1, 268 PVC_GUC_TLB_INV_DESC1_INVALIDATE); 269 xe_mmio_write32(gt, PVC_GUC_TLB_INV_DESC0, 270 PVC_GUC_TLB_INV_DESC0_VALID); 271 } else 272 xe_mmio_write32(gt, GUC_TLB_INV_CR, 273 GUC_TLB_INV_CR_INVALIDATE); 274 } 275 276 xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); 277 } 278 279 void xe_ggtt_invalidate(struct xe_ggtt *ggtt) 280 { 281 /* Each GT in a tile has its own TLB to cache GGTT lookups */ 282 ggtt_invalidate_gt_tlb(ggtt->tile->primary_gt); 283 ggtt_invalidate_gt_tlb(ggtt->tile->media_gt); 284 } 285 286 void xe_ggtt_printk(struct xe_ggtt *ggtt, const char *prefix) 287 { 288 u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[XE_CACHE_WB]; 289 u64 addr, scratch_pte; 290 291 scratch_pte = ggtt->pt_ops->pte_encode_bo(ggtt->scratch, 0, pat_index); 292 293 printk("%sGlobal GTT:", prefix); 294 for (addr = 0; addr < ggtt->size; addr += XE_PAGE_SIZE) { 295 unsigned int i = addr / XE_PAGE_SIZE; 296 297 xe_tile_assert(ggtt->tile, addr <= U32_MAX); 298 if (ggtt->gsm[i] == scratch_pte) 299 continue; 300 301 printk("%s ggtt[0x%08x] = 0x%016llx", 302 prefix, (u32)addr, ggtt->gsm[i]); 303 } 304 } 305 306 int xe_ggtt_insert_special_node_locked(struct xe_ggtt *ggtt, struct drm_mm_node *node, 307 u32 size, u32 align, u32 mm_flags) 308 { 309 return drm_mm_insert_node_generic(&ggtt->mm, node, size, align, 0, 310 mm_flags); 311 } 312 313 int xe_ggtt_insert_special_node(struct xe_ggtt *ggtt, struct drm_mm_node *node, 314 u32 size, u32 align) 315 { 316 int ret; 317 318 mutex_lock(&ggtt->lock); 319 ret = xe_ggtt_insert_special_node_locked(ggtt, node, size, 320 align, DRM_MM_INSERT_HIGH); 321 mutex_unlock(&ggtt->lock); 322 323 return ret; 324 } 325 326 void xe_ggtt_map_bo(struct xe_ggtt *ggtt, struct xe_bo *bo) 327 { 328 u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[XE_CACHE_WB]; 329 u64 start = bo->ggtt_node.start; 330 u64 offset, pte; 331 332 for (offset = 0; offset < bo->size; offset += XE_PAGE_SIZE) { 333 pte = ggtt->pt_ops->pte_encode_bo(bo, offset, pat_index); 334 xe_ggtt_set_pte(ggtt, start + offset, pte); 335 } 336 337 xe_ggtt_invalidate(ggtt); 338 } 339 340 static int __xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo, 341 u64 start, u64 end) 342 { 343 int err; 344 u64 alignment = XE_PAGE_SIZE; 345 346 if (xe_bo_is_vram(bo) && ggtt->flags & XE_GGTT_FLAGS_64K) 347 alignment = SZ_64K; 348 349 if (XE_WARN_ON(bo->ggtt_node.size)) { 350 /* Someone's already inserted this BO in the GGTT */ 351 xe_tile_assert(ggtt->tile, bo->ggtt_node.size == bo->size); 352 return 0; 353 } 354 355 err = xe_bo_validate(bo, NULL, false); 356 if (err) 357 return err; 358 359 xe_device_mem_access_get(tile_to_xe(ggtt->tile)); 360 mutex_lock(&ggtt->lock); 361 err = drm_mm_insert_node_in_range(&ggtt->mm, &bo->ggtt_node, bo->size, 362 alignment, 0, start, end, 0); 363 if (!err) 364 xe_ggtt_map_bo(ggtt, bo); 365 mutex_unlock(&ggtt->lock); 366 xe_device_mem_access_put(tile_to_xe(ggtt->tile)); 367 368 return err; 369 } 370 371 int xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo, 372 u64 start, u64 end) 373 { 374 return __xe_ggtt_insert_bo_at(ggtt, bo, start, end); 375 } 376 377 int xe_ggtt_insert_bo(struct xe_ggtt *ggtt, struct xe_bo *bo) 378 { 379 return __xe_ggtt_insert_bo_at(ggtt, bo, 0, U64_MAX); 380 } 381 382 void xe_ggtt_remove_node(struct xe_ggtt *ggtt, struct drm_mm_node *node) 383 { 384 xe_device_mem_access_get(tile_to_xe(ggtt->tile)); 385 mutex_lock(&ggtt->lock); 386 387 xe_ggtt_clear(ggtt, node->start, node->size); 388 drm_mm_remove_node(node); 389 node->size = 0; 390 391 xe_ggtt_invalidate(ggtt); 392 393 mutex_unlock(&ggtt->lock); 394 xe_device_mem_access_put(tile_to_xe(ggtt->tile)); 395 } 396 397 void xe_ggtt_remove_bo(struct xe_ggtt *ggtt, struct xe_bo *bo) 398 { 399 if (XE_WARN_ON(!bo->ggtt_node.size)) 400 return; 401 402 /* This BO is not currently in the GGTT */ 403 xe_tile_assert(ggtt->tile, bo->ggtt_node.size == bo->size); 404 405 xe_ggtt_remove_node(ggtt, &bo->ggtt_node); 406 } 407 408 int xe_ggtt_dump(struct xe_ggtt *ggtt, struct drm_printer *p) 409 { 410 int err; 411 412 err = mutex_lock_interruptible(&ggtt->lock); 413 if (err) 414 return err; 415 416 drm_mm_print(&ggtt->mm, p); 417 mutex_unlock(&ggtt->lock); 418 return err; 419 } 420