1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021-2024 Intel Corporation 4 */ 5 6 #include <kunit/visibility.h> 7 #include <linux/pci.h> 8 9 #include <drm/drm_managed.h> 10 #include <drm/drm_print.h> 11 12 #include "regs/xe_bars.h" 13 #include "regs/xe_gt_regs.h" 14 #include "regs/xe_regs.h" 15 #include "xe_assert.h" 16 #include "xe_bo.h" 17 #include "xe_device.h" 18 #include "xe_force_wake.h" 19 #include "xe_gt_mcr.h" 20 #include "xe_mmio.h" 21 #include "xe_module.h" 22 #include "xe_sriov.h" 23 #include "xe_tile_sriov_vf.h" 24 #include "xe_ttm_vram_mgr.h" 25 #include "xe_vram.h" 26 #include "xe_vram_types.h" 27 28 static void resize_bar(struct xe_device *xe, int resno, resource_size_t size) 29 { 30 struct pci_dev *pdev = to_pci_dev(xe->drm.dev); 31 int bar_size = pci_rebar_bytes_to_size(size); 32 int ret; 33 34 ret = pci_resize_resource(pdev, resno, bar_size, 0); 35 if (ret) { 36 drm_info(&xe->drm, "Failed to resize BAR%d to %dM (%pe). Consider enabling 'Resizable BAR' support in your BIOS\n", 37 resno, 1 << bar_size, ERR_PTR(ret)); 38 return; 39 } 40 41 drm_info(&xe->drm, "BAR%d resized to %dM\n", resno, 1 << bar_size); 42 } 43 44 /* 45 * if force_vram_bar_size is set, attempt to set to the requested size 46 * else set to maximum possible size 47 */ 48 void xe_vram_resize_bar(struct xe_device *xe) 49 { 50 int force_vram_bar_size = xe_modparam.force_vram_bar_size; 51 struct pci_dev *pdev = to_pci_dev(xe->drm.dev); 52 struct pci_bus *root = pdev->bus; 53 resource_size_t current_size; 54 resource_size_t rebar_size; 55 struct resource *root_res; 56 int max_size, i; 57 u32 pci_cmd; 58 59 /* gather some relevant info */ 60 current_size = pci_resource_len(pdev, LMEM_BAR); 61 62 if (force_vram_bar_size < 0) 63 return; 64 65 /* set to a specific size? */ 66 if (force_vram_bar_size) { 67 rebar_size = pci_rebar_bytes_to_size(force_vram_bar_size * 68 (resource_size_t)SZ_1M); 69 70 if (!pci_rebar_size_supported(pdev, LMEM_BAR, rebar_size)) { 71 drm_info(&xe->drm, 72 "Requested size: %lluMiB is not supported by rebar sizes: 0x%llx. Leaving default: %lluMiB\n", 73 (u64)pci_rebar_size_to_bytes(rebar_size) >> 20, 74 pci_rebar_get_possible_sizes(pdev, LMEM_BAR), 75 (u64)current_size >> 20); 76 return; 77 } 78 79 rebar_size = pci_rebar_size_to_bytes(rebar_size); 80 if (rebar_size == current_size) 81 return; 82 } else { 83 max_size = pci_rebar_get_max_size(pdev, LMEM_BAR); 84 if (max_size < 0) 85 return; 86 rebar_size = pci_rebar_size_to_bytes(max_size); 87 88 /* only resize if larger than current */ 89 if (rebar_size <= current_size) 90 return; 91 } 92 93 drm_info(&xe->drm, "Attempting to resize bar from %lluMiB -> %lluMiB\n", 94 (u64)current_size >> 20, (u64)rebar_size >> 20); 95 96 while (root->parent) 97 root = root->parent; 98 99 pci_bus_for_each_resource(root, root_res, i) { 100 if (root_res && root_res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) && 101 (u64)root_res->start > 0x100000000ul) 102 break; 103 } 104 105 if (!root_res) { 106 drm_info(&xe->drm, "Can't resize VRAM BAR - platform support is missing. Consider enabling 'Resizable BAR' support in your BIOS\n"); 107 return; 108 } 109 110 pci_read_config_dword(pdev, PCI_COMMAND, &pci_cmd); 111 pci_write_config_dword(pdev, PCI_COMMAND, pci_cmd & ~PCI_COMMAND_MEMORY); 112 113 resize_bar(xe, LMEM_BAR, rebar_size); 114 115 pci_assign_unassigned_bus_resources(pdev->bus); 116 pci_write_config_dword(pdev, PCI_COMMAND, pci_cmd); 117 } 118 119 static bool resource_is_valid(struct pci_dev *pdev, int bar) 120 { 121 if (!pci_resource_flags(pdev, bar)) 122 return false; 123 124 if (pci_resource_flags(pdev, bar) & IORESOURCE_UNSET) 125 return false; 126 127 if (!pci_resource_len(pdev, bar)) 128 return false; 129 130 return true; 131 } 132 133 static int determine_lmem_bar_size(struct xe_device *xe, struct xe_vram_region *lmem_bar) 134 { 135 struct pci_dev *pdev = to_pci_dev(xe->drm.dev); 136 137 if (!resource_is_valid(pdev, LMEM_BAR)) { 138 drm_err(&xe->drm, "pci resource is not valid\n"); 139 return -ENXIO; 140 } 141 142 lmem_bar->io_start = pci_resource_start(pdev, LMEM_BAR); 143 lmem_bar->io_size = pci_resource_len(pdev, LMEM_BAR); 144 if (!lmem_bar->io_size) 145 return -EIO; 146 147 /* XXX: Need to change when xe link code is ready */ 148 lmem_bar->dpa_base = 0; 149 150 /* set up a map to the total memory area. */ 151 lmem_bar->mapping = devm_ioremap_wc(&pdev->dev, lmem_bar->io_start, lmem_bar->io_size); 152 153 return 0; 154 } 155 156 static int get_flat_ccs_offset(struct xe_gt *gt, u64 tile_size, u64 *poffset) 157 { 158 struct xe_device *xe = gt_to_xe(gt); 159 unsigned int fw_ref; 160 u64 offset; 161 u32 reg; 162 163 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 164 if (!fw_ref) 165 return -ETIMEDOUT; 166 167 if (GRAPHICS_VER(xe) >= 20) { 168 u64 ccs_size = tile_size / 512; 169 u64 offset_hi, offset_lo; 170 u32 nodes, num_enabled; 171 172 reg = xe_mmio_read32(>->mmio, MIRROR_FUSE3); 173 nodes = REG_FIELD_GET(XE2_NODE_ENABLE_MASK, reg); 174 num_enabled = hweight32(nodes); /* Number of enabled l3 nodes */ 175 176 reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_LOWER); 177 offset_lo = REG_FIELD_GET(XE2_FLAT_CCS_BASE_LOWER_ADDR_MASK, reg); 178 179 reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_UPPER); 180 offset_hi = REG_FIELD_GET(XE2_FLAT_CCS_BASE_UPPER_ADDR_MASK, reg); 181 182 offset = offset_hi << 32; /* HW view bits 39:32 */ 183 offset |= offset_lo << 6; /* HW view bits 31:6 */ 184 offset *= num_enabled; /* convert to SW view */ 185 offset = round_up(offset, SZ_128K); /* SW must round up to nearest 128K */ 186 187 /* We don't expect any holes */ 188 xe_assert_msg(xe, offset == (xe_mmio_read64_2x32(>_to_tile(gt)->mmio, GSMBASE) - 189 ccs_size), 190 "Hole between CCS and GSM.\n"); 191 } else { 192 reg = xe_gt_mcr_unicast_read_any(gt, XEHP_FLAT_CCS_BASE_ADDR); 193 offset = (u64)REG_FIELD_GET(XEHP_FLAT_CCS_PTR, reg) * SZ_64K; 194 } 195 196 xe_force_wake_put(gt_to_fw(gt), fw_ref); 197 *poffset = offset; 198 199 return 0; 200 } 201 202 /* 203 * tile_vram_size() - Collect vram size and offset information 204 * @tile: tile to get info for 205 * @vram_size: available vram (size - device reserved portions) 206 * @tile_size: actual vram size 207 * @tile_offset: physical start point in the vram address space 208 * 209 * There are 4 places for size information: 210 * - io size (from pci_resource_len of LMEM bar) (only used for small bar and DG1) 211 * - TILEx size (actual vram size) 212 * - GSMBASE offset (TILEx - "stolen") 213 * - CSSBASE offset (TILEx - CSS space necessary) 214 * 215 * CSSBASE is always a lower/smaller offset then GSMBASE. 216 * 217 * The actual available size of memory is to the CCS or GSM base. 218 * NOTE: multi-tile bases will include the tile offset. 219 * 220 */ 221 static int tile_vram_size(struct xe_tile *tile, u64 *vram_size, 222 u64 *tile_size, u64 *tile_offset) 223 { 224 struct xe_device *xe = tile_to_xe(tile); 225 struct xe_gt *gt = tile->primary_gt; 226 u64 offset; 227 u32 reg; 228 229 if (IS_SRIOV_VF(xe)) { 230 struct xe_tile *t; 231 int id; 232 233 offset = 0; 234 for_each_tile(t, xe, id) 235 for_each_if(t->id < tile->id) 236 offset += xe_tile_sriov_vf_lmem(t); 237 238 *tile_size = xe_tile_sriov_vf_lmem(tile); 239 *vram_size = *tile_size; 240 *tile_offset = offset; 241 242 return 0; 243 } 244 245 /* actual size */ 246 if (unlikely(xe->info.platform == XE_DG1)) { 247 *tile_size = pci_resource_len(to_pci_dev(xe->drm.dev), LMEM_BAR); 248 *tile_offset = 0; 249 } else { 250 reg = xe_mmio_read32(&tile->mmio, SG_TILE_ADDR_RANGE(tile->id)); 251 *tile_size = (u64)REG_FIELD_GET(GENMASK(14, 8), reg) * SZ_1G; 252 *tile_offset = (u64)REG_FIELD_GET(GENMASK(7, 1), reg) * SZ_1G; 253 } 254 255 /* minus device usage */ 256 if (xe->info.has_flat_ccs) { 257 int ret = get_flat_ccs_offset(gt, *tile_size, &offset); 258 259 if (ret) 260 return ret; 261 } else { 262 offset = xe_mmio_read64_2x32(&tile->mmio, GSMBASE); 263 } 264 265 /* remove the tile offset so we have just the available size */ 266 *vram_size = offset - *tile_offset; 267 268 return 0; 269 } 270 271 static void vram_fini(void *arg) 272 { 273 struct xe_device *xe = arg; 274 struct xe_tile *tile; 275 int id; 276 277 xe->mem.vram->mapping = NULL; 278 279 for_each_tile(tile, xe, id) { 280 tile->mem.vram->mapping = NULL; 281 if (tile->mem.kernel_vram) 282 tile->mem.kernel_vram->mapping = NULL; 283 } 284 } 285 286 struct xe_vram_region *xe_vram_region_alloc(struct xe_device *xe, u8 id, u32 placement) 287 { 288 struct xe_vram_region *vram; 289 struct drm_device *drm = &xe->drm; 290 291 xe_assert(xe, id < xe->info.tile_count); 292 293 vram = drmm_kzalloc(drm, sizeof(*vram), GFP_KERNEL); 294 if (!vram) 295 return NULL; 296 297 vram->xe = xe; 298 vram->id = id; 299 vram->placement = placement; 300 #if defined(CONFIG_DRM_XE_PAGEMAP) 301 vram->migrate = xe->tiles[id].migrate; 302 #endif 303 return vram; 304 } 305 306 static void print_vram_region_info(struct xe_device *xe, struct xe_vram_region *vram) 307 { 308 struct drm_device *drm = &xe->drm; 309 310 if (vram->io_size < vram->usable_size) 311 drm_info(drm, "Small BAR device\n"); 312 313 drm_info(drm, 314 "VRAM[%u]: Actual physical size %pa, usable size exclude stolen %pa, CPU accessible size %pa\n", 315 vram->id, &vram->actual_physical_size, &vram->usable_size, &vram->io_size); 316 drm_info(drm, "VRAM[%u]: DPA range: [%pa-%llx], io range: [%pa-%llx]\n", 317 vram->id, &vram->dpa_base, vram->dpa_base + (u64)vram->actual_physical_size, 318 &vram->io_start, vram->io_start + (u64)vram->io_size); 319 } 320 321 static int vram_region_init(struct xe_device *xe, struct xe_vram_region *vram, 322 struct xe_vram_region *lmem_bar, u64 offset, u64 usable_size, 323 u64 region_size, resource_size_t remain_io_size) 324 { 325 /* Check if VRAM region is already initialized */ 326 if (vram->mapping) 327 return 0; 328 329 vram->actual_physical_size = region_size; 330 vram->io_start = lmem_bar->io_start + offset; 331 vram->io_size = min_t(u64, usable_size, remain_io_size); 332 333 if (!vram->io_size) { 334 drm_err(&xe->drm, "Tile without any CPU visible VRAM. Aborting.\n"); 335 return -ENODEV; 336 } 337 338 vram->dpa_base = lmem_bar->dpa_base + offset; 339 vram->mapping = lmem_bar->mapping + offset; 340 vram->usable_size = usable_size; 341 342 print_vram_region_info(xe, vram); 343 344 return 0; 345 } 346 347 /** 348 * xe_vram_probe() - Probe VRAM configuration 349 * @xe: the &xe_device 350 * 351 * Collect VRAM size and offset information for all tiles. 352 * 353 * Return: 0 on success, error code on failure 354 */ 355 int xe_vram_probe(struct xe_device *xe) 356 { 357 struct xe_tile *tile; 358 struct xe_vram_region lmem_bar; 359 resource_size_t remain_io_size; 360 u64 available_size = 0; 361 u64 total_size = 0; 362 int err; 363 u8 id; 364 365 if (!IS_DGFX(xe)) 366 return 0; 367 368 err = determine_lmem_bar_size(xe, &lmem_bar); 369 if (err) 370 return err; 371 drm_info(&xe->drm, "VISIBLE VRAM: %pa, %pa\n", &lmem_bar.io_start, &lmem_bar.io_size); 372 373 remain_io_size = lmem_bar.io_size; 374 375 for_each_tile(tile, xe, id) { 376 u64 region_size; 377 u64 usable_size; 378 u64 tile_offset; 379 380 err = tile_vram_size(tile, &usable_size, ®ion_size, &tile_offset); 381 if (err) 382 return err; 383 384 total_size += region_size; 385 available_size += usable_size; 386 387 err = vram_region_init(xe, tile->mem.vram, &lmem_bar, tile_offset, usable_size, 388 region_size, remain_io_size); 389 if (err) 390 return err; 391 392 if (total_size > lmem_bar.io_size) { 393 drm_info(&xe->drm, "VRAM: %pa is larger than resource %pa\n", 394 &total_size, &lmem_bar.io_size); 395 } 396 397 remain_io_size -= min_t(u64, tile->mem.vram->actual_physical_size, remain_io_size); 398 } 399 400 err = vram_region_init(xe, xe->mem.vram, &lmem_bar, 0, available_size, total_size, 401 lmem_bar.io_size); 402 if (err) 403 return err; 404 405 return devm_add_action_or_reset(xe->drm.dev, vram_fini, xe); 406 } 407 408 /** 409 * xe_vram_region_io_start - Get the IO start of a VRAM region 410 * @vram: the VRAM region 411 * 412 * Return: the IO start of the VRAM region, or 0 if not valid 413 */ 414 resource_size_t xe_vram_region_io_start(const struct xe_vram_region *vram) 415 { 416 return vram ? vram->io_start : 0; 417 } 418 419 /** 420 * xe_vram_region_io_size - Get the IO size of a VRAM region 421 * @vram: the VRAM region 422 * 423 * Return: the IO size of the VRAM region, or 0 if not valid 424 */ 425 resource_size_t xe_vram_region_io_size(const struct xe_vram_region *vram) 426 { 427 return vram ? vram->io_size : 0; 428 } 429 430 /** 431 * xe_vram_region_dpa_base - Get the DPA base of a VRAM region 432 * @vram: the VRAM region 433 * 434 * Return: the DPA base of the VRAM region, or 0 if not valid 435 */ 436 resource_size_t xe_vram_region_dpa_base(const struct xe_vram_region *vram) 437 { 438 return vram ? vram->dpa_base : 0; 439 } 440 441 /** 442 * xe_vram_region_usable_size - Get the usable size of a VRAM region 443 * @vram: the VRAM region 444 * 445 * Return: the usable size of the VRAM region, or 0 if not valid 446 */ 447 resource_size_t xe_vram_region_usable_size(const struct xe_vram_region *vram) 448 { 449 return vram ? vram->usable_size : 0; 450 } 451 452 /** 453 * xe_vram_region_actual_physical_size - Get the actual physical size of a VRAM region 454 * @vram: the VRAM region 455 * 456 * Return: the actual physical size of the VRAM region, or 0 if not valid 457 */ 458 resource_size_t xe_vram_region_actual_physical_size(const struct xe_vram_region *vram) 459 { 460 return vram ? vram->actual_physical_size : 0; 461 } 462 EXPORT_SYMBOL_IF_KUNIT(xe_vram_region_actual_physical_size); 463