1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021-2023 Intel Corporation 4 */ 5 6 #include "xe_mmio.h" 7 8 #include <linux/delay.h> 9 #include <linux/io-64-nonatomic-lo-hi.h> 10 #include <linux/minmax.h> 11 #include <linux/pci.h> 12 13 #include <drm/drm_managed.h> 14 #include <drm/drm_print.h> 15 16 #include "regs/xe_bars.h" 17 #include "regs/xe_regs.h" 18 #include "xe_device.h" 19 #include "xe_gt.h" 20 #include "xe_gt_printk.h" 21 #include "xe_gt_sriov_vf.h" 22 #include "xe_macros.h" 23 #include "xe_sriov.h" 24 #include "xe_trace.h" 25 26 static void tiles_fini(void *arg) 27 { 28 struct xe_device *xe = arg; 29 struct xe_tile *tile; 30 int id; 31 32 for_each_remote_tile(tile, xe, id) 33 tile->mmio.regs = NULL; 34 } 35 36 /* 37 * On multi-tile devices, partition the BAR space for MMIO on each tile, 38 * possibly accounting for register override on the number of tiles available. 39 * tile_mmio_size contains both the tile's 4MB register space, as well as 40 * additional space for the GTT and other (possibly unused) regions). 41 * Resulting memory layout is like below: 42 * 43 * .----------------------. <- tile_count * tile_mmio_size 44 * | .... | 45 * |----------------------| <- 2 * tile_mmio_size 46 * | tile1 GTT + other | 47 * |----------------------| <- 1 * tile_mmio_size + 4MB 48 * | tile1->mmio.regs | 49 * |----------------------| <- 1 * tile_mmio_size 50 * | tile0 GTT + other | 51 * |----------------------| <- 4MB 52 * | tile0->mmio.regs | 53 * '----------------------' <- 0MB 54 */ 55 static void mmio_multi_tile_setup(struct xe_device *xe, size_t tile_mmio_size) 56 { 57 struct xe_tile *tile; 58 void __iomem *regs; 59 u8 id; 60 61 /* 62 * Nothing to be done as tile 0 has already been setup earlier with the 63 * entire BAR mapped - see xe_mmio_init() 64 */ 65 if (xe->info.tile_count == 1) 66 return; 67 68 /* Possibly override number of tile based on configuration register */ 69 if (!xe->info.skip_mtcfg) { 70 struct xe_mmio *mmio = xe_root_tile_mmio(xe); 71 u8 tile_count; 72 u32 mtcfg; 73 74 /* 75 * Although the per-tile mmio regs are not yet initialized, this 76 * is fine as it's going to the root tile's mmio, that's 77 * guaranteed to be initialized earlier in xe_mmio_init() 78 */ 79 mtcfg = xe_mmio_read64_2x32(mmio, XEHP_MTCFG_ADDR); 80 tile_count = REG_FIELD_GET(TILE_COUNT, mtcfg) + 1; 81 82 if (tile_count < xe->info.tile_count) { 83 drm_info(&xe->drm, "tile_count: %d, reduced_tile_count %d\n", 84 xe->info.tile_count, tile_count); 85 xe->info.tile_count = tile_count; 86 87 /* 88 * FIXME: Needs some work for standalone media, but 89 * should be impossible with multi-tile for now: 90 * multi-tile platform with standalone media doesn't 91 * exist 92 */ 93 xe->info.gt_count = xe->info.tile_count; 94 } 95 } 96 97 regs = xe->mmio.regs; 98 for_each_tile(tile, xe, id) { 99 tile->mmio.regs_size = SZ_4M; 100 tile->mmio.regs = regs; 101 tile->mmio.tile = tile; 102 regs += tile_mmio_size; 103 } 104 } 105 106 int xe_mmio_probe_tiles(struct xe_device *xe) 107 { 108 size_t tile_mmio_size = SZ_16M; 109 110 mmio_multi_tile_setup(xe, tile_mmio_size); 111 112 return devm_add_action_or_reset(xe->drm.dev, tiles_fini, xe); 113 } 114 115 static void mmio_fini(void *arg) 116 { 117 struct xe_device *xe = arg; 118 struct xe_tile *root_tile = xe_device_get_root_tile(xe); 119 120 pci_iounmap(to_pci_dev(xe->drm.dev), xe->mmio.regs); 121 xe->mmio.regs = NULL; 122 root_tile->mmio.regs = NULL; 123 } 124 125 int xe_mmio_init(struct xe_device *xe) 126 { 127 struct xe_tile *root_tile = xe_device_get_root_tile(xe); 128 struct pci_dev *pdev = to_pci_dev(xe->drm.dev); 129 130 /* 131 * Map the entire BAR. 132 * The first 16MB of the BAR, belong to the root tile, and include: 133 * registers (0-4MB), reserved space (4MB-8MB) and GGTT (8MB-16MB). 134 */ 135 xe->mmio.size = pci_resource_len(pdev, GTTMMADR_BAR); 136 xe->mmio.regs = pci_iomap(pdev, GTTMMADR_BAR, 0); 137 if (xe->mmio.regs == NULL) { 138 drm_err(&xe->drm, "failed to map registers\n"); 139 return -EIO; 140 } 141 142 /* Setup first tile; other tiles (if present) will be setup later. */ 143 root_tile->mmio.regs_size = SZ_4M; 144 root_tile->mmio.regs = xe->mmio.regs; 145 root_tile->mmio.tile = root_tile; 146 147 return devm_add_action_or_reset(xe->drm.dev, mmio_fini, xe); 148 } 149 150 static void mmio_flush_pending_writes(struct xe_mmio *mmio) 151 { 152 #define DUMMY_REG_OFFSET 0x130030 153 int i; 154 155 if (mmio->tile->xe->info.platform != XE_LUNARLAKE) 156 return; 157 158 /* 4 dummy writes */ 159 for (i = 0; i < 4; i++) 160 writel(0, mmio->regs + DUMMY_REG_OFFSET); 161 } 162 163 u8 xe_mmio_read8(struct xe_mmio *mmio, struct xe_reg reg) 164 { 165 u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr); 166 u8 val; 167 168 /* Wa_15015404425 */ 169 mmio_flush_pending_writes(mmio); 170 171 val = readb(mmio->regs + addr); 172 trace_xe_reg_rw(mmio, false, addr, val, sizeof(val)); 173 174 return val; 175 } 176 177 u16 xe_mmio_read16(struct xe_mmio *mmio, struct xe_reg reg) 178 { 179 u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr); 180 u16 val; 181 182 /* Wa_15015404425 */ 183 mmio_flush_pending_writes(mmio); 184 185 val = readw(mmio->regs + addr); 186 trace_xe_reg_rw(mmio, false, addr, val, sizeof(val)); 187 188 return val; 189 } 190 191 void xe_mmio_write32(struct xe_mmio *mmio, struct xe_reg reg, u32 val) 192 { 193 u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr); 194 195 trace_xe_reg_rw(mmio, true, addr, val, sizeof(val)); 196 197 if (!reg.vf && mmio->sriov_vf_gt) 198 xe_gt_sriov_vf_write32(mmio->sriov_vf_gt, reg, val); 199 else 200 writel(val, mmio->regs + addr); 201 } 202 203 u32 xe_mmio_read32(struct xe_mmio *mmio, struct xe_reg reg) 204 { 205 u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr); 206 u32 val; 207 208 /* Wa_15015404425 */ 209 mmio_flush_pending_writes(mmio); 210 211 if (!reg.vf && mmio->sriov_vf_gt) 212 val = xe_gt_sriov_vf_read32(mmio->sriov_vf_gt, reg); 213 else 214 val = readl(mmio->regs + addr); 215 216 trace_xe_reg_rw(mmio, false, addr, val, sizeof(val)); 217 218 return val; 219 } 220 221 u32 xe_mmio_rmw32(struct xe_mmio *mmio, struct xe_reg reg, u32 clr, u32 set) 222 { 223 u32 old, reg_val; 224 225 old = xe_mmio_read32(mmio, reg); 226 reg_val = (old & ~clr) | set; 227 xe_mmio_write32(mmio, reg, reg_val); 228 229 return old; 230 } 231 232 int xe_mmio_write32_and_verify(struct xe_mmio *mmio, 233 struct xe_reg reg, u32 val, u32 mask, u32 eval) 234 { 235 u32 reg_val; 236 237 xe_mmio_write32(mmio, reg, val); 238 reg_val = xe_mmio_read32(mmio, reg); 239 240 return (reg_val & mask) != eval ? -EINVAL : 0; 241 } 242 243 bool xe_mmio_in_range(const struct xe_mmio *mmio, 244 const struct xe_mmio_range *range, 245 struct xe_reg reg) 246 { 247 u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr); 248 249 return range && addr >= range->start && addr <= range->end; 250 } 251 252 /** 253 * xe_mmio_read64_2x32() - Read a 64-bit register as two 32-bit reads 254 * @mmio: MMIO target 255 * @reg: register to read value from 256 * 257 * Although Intel GPUs have some 64-bit registers, the hardware officially 258 * only supports GTTMMADR register reads of 32 bits or smaller. Even if 259 * a readq operation may return a reasonable value, that violation of the 260 * spec shouldn't be relied upon and all 64-bit register reads should be 261 * performed as two 32-bit reads of the upper and lower dwords. 262 * 263 * When reading registers that may be changing (such as 264 * counters), a rollover of the lower dword between the two 32-bit reads 265 * can be problematic. This function attempts to ensure the upper dword has 266 * stabilized before returning the 64-bit value. 267 * 268 * Note that because this function may re-read the register multiple times 269 * while waiting for the value to stabilize it should not be used to read 270 * any registers where read operations have side effects. 271 * 272 * Returns the value of the 64-bit register. 273 */ 274 u64 xe_mmio_read64_2x32(struct xe_mmio *mmio, struct xe_reg reg) 275 { 276 struct xe_reg reg_udw = { .addr = reg.addr + 0x4 }; 277 u32 ldw, udw, oldudw, retries; 278 279 reg.addr = xe_mmio_adjusted_addr(mmio, reg.addr); 280 reg_udw.addr = xe_mmio_adjusted_addr(mmio, reg_udw.addr); 281 282 /* we shouldn't adjust just one register address */ 283 xe_tile_assert(mmio->tile, reg_udw.addr == reg.addr + 0x4); 284 285 oldudw = xe_mmio_read32(mmio, reg_udw); 286 for (retries = 5; retries; --retries) { 287 ldw = xe_mmio_read32(mmio, reg); 288 udw = xe_mmio_read32(mmio, reg_udw); 289 290 if (udw == oldudw) 291 break; 292 293 oldudw = udw; 294 } 295 296 drm_WARN(&mmio->tile->xe->drm, retries == 0, 297 "64-bit read of %#x did not stabilize\n", reg.addr); 298 299 return (u64)udw << 32 | ldw; 300 } 301 302 static int __xe_mmio_wait32(struct xe_mmio *mmio, struct xe_reg reg, u32 mask, u32 val, u32 timeout_us, 303 u32 *out_val, bool atomic, bool expect_match) 304 { 305 ktime_t cur = ktime_get_raw(); 306 const ktime_t end = ktime_add_us(cur, timeout_us); 307 int ret = -ETIMEDOUT; 308 s64 wait = 10; 309 u32 read; 310 bool check; 311 312 for (;;) { 313 read = xe_mmio_read32(mmio, reg); 314 315 check = (read & mask) == val; 316 if (!expect_match) 317 check = !check; 318 319 if (check) { 320 ret = 0; 321 break; 322 } 323 324 cur = ktime_get_raw(); 325 if (!ktime_before(cur, end)) 326 break; 327 328 if (ktime_after(ktime_add_us(cur, wait), end)) 329 wait = ktime_us_delta(end, cur); 330 331 if (atomic) 332 udelay(wait); 333 else 334 usleep_range(wait, wait << 1); 335 wait <<= 1; 336 } 337 338 if (ret != 0) { 339 read = xe_mmio_read32(mmio, reg); 340 341 check = (read & mask) == val; 342 if (!expect_match) 343 check = !check; 344 345 if (check) 346 ret = 0; 347 } 348 349 if (out_val) 350 *out_val = read; 351 352 return ret; 353 } 354 355 /** 356 * xe_mmio_wait32() - Wait for a register to match the desired masked value 357 * @mmio: MMIO target 358 * @reg: register to read value from 359 * @mask: mask to be applied to the value read from the register 360 * @val: desired value after applying the mask 361 * @timeout_us: time out after this period of time. Wait logic tries to be 362 * smart, applying an exponential backoff until @timeout_us is reached. 363 * @out_val: if not NULL, points where to store the last unmasked value 364 * @atomic: needs to be true if calling from an atomic context 365 * 366 * This function polls for the desired masked value and returns zero on success 367 * or -ETIMEDOUT if timed out. 368 * 369 * Note that @timeout_us represents the minimum amount of time to wait before 370 * giving up. The actual time taken by this function can be a little more than 371 * @timeout_us for different reasons, specially in non-atomic contexts. Thus, 372 * it is possible that this function succeeds even after @timeout_us has passed. 373 */ 374 int xe_mmio_wait32(struct xe_mmio *mmio, struct xe_reg reg, u32 mask, u32 val, u32 timeout_us, 375 u32 *out_val, bool atomic) 376 { 377 return __xe_mmio_wait32(mmio, reg, mask, val, timeout_us, out_val, atomic, true); 378 } 379 380 /** 381 * xe_mmio_wait32_not() - Wait for a register to return anything other than the given masked value 382 * @mmio: MMIO target 383 * @reg: register to read value from 384 * @mask: mask to be applied to the value read from the register 385 * @val: value not to be matched after applying the mask 386 * @timeout_us: time out after this period of time 387 * @out_val: if not NULL, points where to store the last unmasked value 388 * @atomic: needs to be true if calling from an atomic context 389 * 390 * This function works exactly like xe_mmio_wait32() with the exception that 391 * @val is expected not to be matched. 392 */ 393 int xe_mmio_wait32_not(struct xe_mmio *mmio, struct xe_reg reg, u32 mask, u32 val, u32 timeout_us, 394 u32 *out_val, bool atomic) 395 { 396 return __xe_mmio_wait32(mmio, reg, mask, val, timeout_us, out_val, atomic, false); 397 } 398