1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021-2024 Intel Corporation 4 */ 5 6 #include <kunit/visibility.h> 7 #include <linux/pci.h> 8 9 #include <drm/drm_managed.h> 10 #include <drm/drm_print.h> 11 12 #include "regs/xe_bars.h" 13 #include "regs/xe_gt_regs.h" 14 #include "regs/xe_regs.h" 15 #include "xe_assert.h" 16 #include "xe_bo.h" 17 #include "xe_device.h" 18 #include "xe_force_wake.h" 19 #include "xe_gt_mcr.h" 20 #include "xe_mmio.h" 21 #include "xe_sriov.h" 22 #include "xe_tile_sriov_vf.h" 23 #include "xe_ttm_vram_mgr.h" 24 #include "xe_vram.h" 25 #include "xe_vram_types.h" 26 27 static bool resource_is_valid(struct pci_dev *pdev, int bar) 28 { 29 if (!pci_resource_flags(pdev, bar)) 30 return false; 31 32 if (pci_resource_flags(pdev, bar) & IORESOURCE_UNSET) 33 return false; 34 35 if (!pci_resource_len(pdev, bar)) 36 return false; 37 38 return true; 39 } 40 41 static int determine_lmem_bar_size(struct xe_device *xe, struct xe_vram_region *lmem_bar) 42 { 43 struct pci_dev *pdev = to_pci_dev(xe->drm.dev); 44 45 if (!resource_is_valid(pdev, LMEM_BAR)) { 46 drm_err(&xe->drm, "pci resource is not valid\n"); 47 return -ENXIO; 48 } 49 50 lmem_bar->io_start = pci_resource_start(pdev, LMEM_BAR); 51 lmem_bar->io_size = pci_resource_len(pdev, LMEM_BAR); 52 if (!lmem_bar->io_size) 53 return -EIO; 54 55 /* XXX: Need to change when xe link code is ready */ 56 lmem_bar->dpa_base = 0; 57 58 /* set up a map to the total memory area. */ 59 lmem_bar->mapping = devm_ioremap_wc(&pdev->dev, lmem_bar->io_start, lmem_bar->io_size); 60 61 return 0; 62 } 63 64 static int get_flat_ccs_offset(struct xe_gt *gt, u64 tile_size, u64 *poffset) 65 { 66 struct xe_device *xe = gt_to_xe(gt); 67 u64 offset; 68 u32 reg; 69 70 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 71 if (!fw_ref.domains) 72 return -ETIMEDOUT; 73 74 if (GRAPHICS_VER(xe) >= 20) { 75 u64 ccs_size = tile_size / 512; 76 u64 offset_hi, offset_lo; 77 u32 nodes, num_enabled; 78 79 reg = xe_mmio_read32(>->mmio, MIRROR_FUSE3); 80 nodes = REG_FIELD_GET(XE2_NODE_ENABLE_MASK, reg); 81 num_enabled = hweight32(nodes); /* Number of enabled l3 nodes */ 82 83 reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_LOWER); 84 offset_lo = REG_FIELD_GET(XE2_FLAT_CCS_BASE_LOWER_ADDR_MASK, reg); 85 86 reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_UPPER); 87 offset_hi = REG_FIELD_GET(XE2_FLAT_CCS_BASE_UPPER_ADDR_MASK, reg); 88 89 offset = offset_hi << 32; /* HW view bits 39:32 */ 90 offset |= offset_lo << 6; /* HW view bits 31:6 */ 91 offset *= num_enabled; /* convert to SW view */ 92 93 /* 94 * Everything below this offset is handed to the VRAM 95 * allocator, so it has to be the *first* address the 96 * compression hardware owns, rounded down. Rounding it up 97 * publishes CCS storage as free memory. 98 */ 99 offset = round_down(offset, SZ_4K); 100 101 /* 102 * CCS storage must not run into GSM. The old check compared 103 * the offset against GSMBASE - ccs_size for equality, which 104 * could not fail: that value is 128K aligned, so it agreed 105 * with the rounded-up offset even when the base was not 128K 106 * aligned - exactly the case this fixes. 107 */ 108 xe_assert_msg(xe, offset + ccs_size <= 109 xe_mmio_read64_2x32(>_to_tile(gt)->mmio, GSMBASE), 110 "CCS overlaps GSM.\n"); 111 } else { 112 reg = xe_gt_mcr_unicast_read_any(gt, XEHP_FLAT_CCS_BASE_ADDR); 113 offset = (u64)REG_FIELD_GET(XEHP_FLAT_CCS_PTR, reg) * SZ_64K; 114 } 115 116 *poffset = offset; 117 118 return 0; 119 } 120 121 /* 122 * tile_vram_size() - Collect vram size and offset information 123 * @tile: tile to get info for 124 * @vram_size: available vram (size - device reserved portions) 125 * @tile_size: actual vram size 126 * @tile_offset: physical start point in the vram address space 127 * 128 * There are 4 places for size information: 129 * - io size (from pci_resource_len of LMEM bar) (only used for small bar and DG1) 130 * - TILEx size (actual vram size) 131 * - GSMBASE offset (TILEx - "stolen") 132 * - CSSBASE offset (TILEx - CSS space necessary) 133 * 134 * CSSBASE is always a lower/smaller offset then GSMBASE. 135 * 136 * The actual available size of memory is to the CCS or GSM base. 137 * NOTE: multi-tile bases will include the tile offset. 138 * 139 */ 140 static int tile_vram_size(struct xe_tile *tile, u64 *vram_size, 141 u64 *tile_size, u64 *tile_offset) 142 { 143 struct xe_device *xe = tile_to_xe(tile); 144 struct xe_gt *gt = tile->primary_gt; 145 u64 offset; 146 u32 reg; 147 148 if (IS_SRIOV_VF(xe)) { 149 struct xe_tile *t; 150 int id; 151 152 offset = 0; 153 for_each_tile(t, xe, id) 154 for_each_if(t->id < tile->id) 155 offset += xe_tile_sriov_vf_lmem(t); 156 157 *tile_size = xe_tile_sriov_vf_lmem(tile); 158 *vram_size = *tile_size; 159 *tile_offset = offset; 160 161 return 0; 162 } 163 164 /* actual size */ 165 if (unlikely(xe->info.platform == XE_DG1)) { 166 *tile_size = pci_resource_len(to_pci_dev(xe->drm.dev), LMEM_BAR); 167 *tile_offset = 0; 168 } else { 169 reg = xe_mmio_read32(&tile->mmio, SG_TILE_ADDR_RANGE(tile->id)); 170 *tile_size = (u64)REG_FIELD_GET(GENMASK(17, 8), reg) * SZ_1G; 171 *tile_offset = (u64)REG_FIELD_GET(GENMASK(7, 1), reg) * SZ_1G; 172 } 173 174 /* minus device usage */ 175 if (xe->info.has_flat_ccs) { 176 int ret = get_flat_ccs_offset(gt, *tile_size, &offset); 177 178 if (ret) 179 return ret; 180 } else { 181 offset = xe_mmio_read64_2x32(&tile->mmio, GSMBASE); 182 } 183 184 /* remove the tile offset so we have just the available size */ 185 *vram_size = offset - *tile_offset; 186 187 return 0; 188 } 189 190 static void vram_fini(void *arg) 191 { 192 struct xe_device *xe = arg; 193 struct xe_tile *tile; 194 int id; 195 196 xe->mem.vram->mapping = NULL; 197 198 for_each_tile(tile, xe, id) { 199 tile->mem.vram->mapping = NULL; 200 if (tile->mem.kernel_vram) 201 tile->mem.kernel_vram->mapping = NULL; 202 } 203 } 204 205 struct xe_vram_region *xe_vram_region_alloc(struct xe_device *xe, u8 id, u32 placement) 206 { 207 struct xe_vram_region *vram; 208 struct drm_device *drm = &xe->drm; 209 210 xe_assert(xe, id < xe->info.tile_count); 211 212 vram = drmm_kzalloc(drm, sizeof(*vram), GFP_KERNEL); 213 if (!vram) 214 return NULL; 215 216 vram->xe = xe; 217 vram->id = id; 218 vram->placement = placement; 219 #if defined(CONFIG_DRM_XE_PAGEMAP) 220 vram->migrate = xe->tiles[id].migrate; 221 #endif 222 return vram; 223 } 224 225 static void print_vram_region_info(struct xe_device *xe, struct xe_vram_region *vram) 226 { 227 struct drm_device *drm = &xe->drm; 228 229 if (vram->io_size < vram->usable_size) 230 drm_info(drm, "Small BAR device\n"); 231 232 drm_info(drm, 233 "VRAM[%u]: Actual physical size %pa, usable size exclude stolen %pa, CPU accessible size %pa\n", 234 vram->id, &vram->actual_physical_size, &vram->usable_size, &vram->io_size); 235 drm_info(drm, "VRAM[%u]: DPA range: [%pa-%llx], io range: [%pa-%llx]\n", 236 vram->id, &vram->dpa_base, vram->dpa_base + (u64)vram->actual_physical_size, 237 &vram->io_start, vram->io_start + (u64)vram->io_size); 238 } 239 240 static int vram_region_init(struct xe_device *xe, struct xe_vram_region *vram, 241 struct xe_vram_region *lmem_bar, u64 offset, u64 usable_size, 242 u64 region_size, resource_size_t remain_io_size) 243 { 244 /* Check if VRAM region is already initialized */ 245 if (vram->mapping) 246 return 0; 247 248 vram->actual_physical_size = region_size; 249 vram->io_start = lmem_bar->io_start + offset; 250 vram->io_size = min_t(u64, usable_size, remain_io_size); 251 252 if (!vram->io_size) { 253 drm_err(&xe->drm, "Tile without any CPU visible VRAM. Aborting.\n"); 254 return -ENODEV; 255 } 256 257 vram->dpa_base = lmem_bar->dpa_base + offset; 258 vram->mapping = lmem_bar->mapping + offset; 259 vram->usable_size = usable_size; 260 261 print_vram_region_info(xe, vram); 262 263 return 0; 264 } 265 266 /** 267 * xe_map_resource_to_region - Map ttm resource to vram memory region 268 * @res: The ttm resource 269 * 270 * Get vram memory region using vram memory manager managing this resource 271 * 272 * Returns: pointer to xe_vram_region 273 */ 274 struct xe_vram_region *xe_map_resource_to_region(struct ttm_resource *res) 275 { 276 struct xe_device *xe = ttm_to_xe_device(res->bo->bdev); 277 struct ttm_resource_manager *mgr; 278 struct xe_ttm_vram_mgr *vram_mgr; 279 280 xe_assert(xe, mem_type_is_vram(res->mem_type)); 281 mgr = ttm_manager_type(&xe->ttm, res->mem_type); 282 vram_mgr = to_xe_ttm_vram_mgr(mgr); 283 284 return container_of(vram_mgr, struct xe_vram_region, ttm); 285 } 286 287 /** 288 * xe_vram_probe() - Probe VRAM configuration 289 * @xe: the &xe_device 290 * 291 * Collect VRAM size and offset information for all tiles. 292 * 293 * Return: 0 on success, error code on failure 294 */ 295 int xe_vram_probe(struct xe_device *xe) 296 { 297 struct xe_tile *tile; 298 struct xe_vram_region lmem_bar; 299 resource_size_t remain_io_size; 300 u64 available_size = 0; 301 u64 total_size = 0; 302 int err; 303 u8 id; 304 305 if (!IS_DGFX(xe)) 306 return 0; 307 308 err = determine_lmem_bar_size(xe, &lmem_bar); 309 if (err) 310 return err; 311 drm_info(&xe->drm, "VISIBLE VRAM: %pa, %pa\n", &lmem_bar.io_start, &lmem_bar.io_size); 312 313 remain_io_size = lmem_bar.io_size; 314 315 for_each_tile(tile, xe, id) { 316 u64 region_size; 317 u64 usable_size; 318 u64 tile_offset; 319 320 err = tile_vram_size(tile, &usable_size, ®ion_size, &tile_offset); 321 if (err) 322 return err; 323 324 total_size += region_size; 325 available_size += usable_size; 326 327 err = vram_region_init(xe, tile->mem.vram, &lmem_bar, tile_offset, usable_size, 328 region_size, remain_io_size); 329 if (err) 330 return err; 331 332 if (total_size > lmem_bar.io_size) { 333 drm_info(&xe->drm, "VRAM: %pa is larger than resource %pa\n", 334 &total_size, &lmem_bar.io_size); 335 } 336 337 remain_io_size -= min_t(u64, tile->mem.vram->actual_physical_size, remain_io_size); 338 } 339 340 err = vram_region_init(xe, xe->mem.vram, &lmem_bar, 0, available_size, total_size, 341 lmem_bar.io_size); 342 if (err) 343 return err; 344 345 return devm_add_action_or_reset(xe->drm.dev, vram_fini, xe); 346 } 347 348 /** 349 * xe_vram_region_io_start - Get the IO start of a VRAM region 350 * @vram: the VRAM region 351 * 352 * Return: the IO start of the VRAM region, or 0 if not valid 353 */ 354 resource_size_t xe_vram_region_io_start(const struct xe_vram_region *vram) 355 { 356 return vram ? vram->io_start : 0; 357 } 358 359 /** 360 * xe_vram_region_io_size - Get the IO size of a VRAM region 361 * @vram: the VRAM region 362 * 363 * Return: the IO size of the VRAM region, or 0 if not valid 364 */ 365 resource_size_t xe_vram_region_io_size(const struct xe_vram_region *vram) 366 { 367 return vram ? vram->io_size : 0; 368 } 369 370 /** 371 * xe_vram_region_dpa_base - Get the DPA base of a VRAM region 372 * @vram: the VRAM region 373 * 374 * Return: the DPA base of the VRAM region, or 0 if not valid 375 */ 376 resource_size_t xe_vram_region_dpa_base(const struct xe_vram_region *vram) 377 { 378 return vram ? vram->dpa_base : 0; 379 } 380 381 /** 382 * xe_vram_region_usable_size - Get the usable size of a VRAM region 383 * @vram: the VRAM region 384 * 385 * Return: the usable size of the VRAM region, or 0 if not valid 386 */ 387 resource_size_t xe_vram_region_usable_size(const struct xe_vram_region *vram) 388 { 389 return vram ? vram->usable_size : 0; 390 } 391 392 /** 393 * xe_vram_region_actual_physical_size - Get the actual physical size of a VRAM region 394 * @vram: the VRAM region 395 * 396 * Return: the actual physical size of the VRAM region, or 0 if not valid 397 */ 398 resource_size_t xe_vram_region_actual_physical_size(const struct xe_vram_region *vram) 399 { 400 return vram ? vram->actual_physical_size : 0; 401 } 402 EXPORT_SYMBOL_IF_KUNIT(xe_vram_region_actual_physical_size); 403