1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021-2023 Intel Corporation 4 */ 5 6 #include "xe_mmio.h" 7 8 #include <linux/delay.h> 9 #include <linux/error-injection.h> 10 #include <linux/io-64-nonatomic-lo-hi.h> 11 #include <linux/minmax.h> 12 #include <linux/pci.h> 13 14 #include <drm/drm_managed.h> 15 16 #include "regs/xe_bars.h" 17 #include "xe_device.h" 18 #include "xe_gt_sriov_vf.h" 19 #include "xe_printk.h" 20 #include "xe_sriov.h" 21 #include "xe_tile_printk.h" 22 #include "xe_trace.h" 23 #include "xe_wa.h" 24 25 #include "generated/xe_device_wa_oob.h" 26 27 /* 28 * On multi-tile devices, partition the BAR space for MMIO on each tile, 29 * possibly accounting for register override on the number of tiles available. 30 * tile_mmio_size contains both the tile's 4MB register space, as well as 31 * additional space for the GTT and other (possibly unused) regions). 32 * Resulting memory layout is like below: 33 * 34 * .----------------------. <- tile_count * tile_mmio_size 35 * | .... | 36 * |----------------------| <- 2 * tile_mmio_size 37 * | tile1 GTT + other | 38 * |----------------------| <- 1 * tile_mmio_size + 4MB 39 * | tile1->mmio.regs | 40 * |----------------------| <- 1 * tile_mmio_size 41 * | tile0 GTT + other | 42 * |----------------------| <- 4MB 43 * | tile0->mmio.regs | 44 * '----------------------' <- 0MB 45 */ 46 static void mmio_multi_tile_setup(struct xe_device *xe, size_t tile_mmio_size) 47 { 48 struct xe_tile *tile; 49 u8 id; 50 51 for_each_remote_tile(tile, xe, id) 52 xe_mmio_init(&tile->mmio, tile, xe->mmio.regs + id * tile_mmio_size, SZ_4M); 53 } 54 55 /** 56 * xe_mmio_probe_tiles() - Initialize all tiles' MMIO 57 * @xe: the &xe_device 58 * 59 * Initialize the remaining tiles' MMIO instances. 60 * 61 * Return: 0 on success or a negative error code on failure. 62 */ 63 int xe_mmio_probe_tiles(struct xe_device *xe) 64 { 65 size_t tile_mmio_size = SZ_16M; 66 67 /* 68 * Nothing to be done as tile 0 has already been setup earlier with the 69 * entire BAR mapped - see xe_mmio_probe_early() 70 */ 71 if (xe->info.tile_count == 1) 72 return 0; 73 74 if (xe->mmio.size < xe->info.tile_count * tile_mmio_size) { 75 xe_err(xe, "GTTMMADR_BAR is too small for %d tiles: %zu\n", 76 xe->info.tile_count, xe->mmio.size); 77 return -EIO; 78 } 79 80 mmio_multi_tile_setup(xe, tile_mmio_size); 81 return 0; 82 } 83 84 static void mmio_fini(void *arg) 85 { 86 struct xe_device *xe = arg; 87 88 xe->mmio.regs = NULL; 89 } 90 91 /** 92 * xe_mmio_probe_early() - Probe and initialize device's MMIO 93 * @xe: the &xe_device 94 * 95 * Map the entire GTTMMADR_BAR and initialize the first tile's MMIO instance. 96 * 97 * The first 16MB of the GTTMMADR_BAR always belongs to the root tile, and 98 * includes: registers (0-4MB), reserved space (4MB-8MB) and GGTT (8MB-16MB). 99 * 100 * Return: 0 on success or a negative error code on failure. 101 */ 102 int xe_mmio_probe_early(struct xe_device *xe) 103 { 104 struct xe_tile *root_tile = xe_device_get_root_tile(xe); 105 struct pci_dev *pdev = to_pci_dev(xe->drm.dev); 106 107 xe->mmio.regs = pcim_iomap(pdev, GTTMMADR_BAR, 0); 108 if (!xe->mmio.regs) { 109 xe_err(xe, "Failed to map GTTMMADR_BAR\n"); 110 return -EIO; 111 } 112 113 xe->mmio.size = pci_resource_len(pdev, GTTMMADR_BAR); 114 if (xe->mmio.size < SZ_16M) { 115 xe_err(xe, "GTTMMADR_BAR is too small: %zu\n", xe->mmio.size); 116 return -EIO; 117 } 118 119 /* Setup first tile; other tiles (if present) will be setup later. */ 120 xe_mmio_init(&root_tile->mmio, root_tile, xe->mmio.regs, SZ_4M); 121 122 return devm_add_action_or_reset(xe->drm.dev, mmio_fini, xe); 123 } 124 ALLOW_ERROR_INJECTION(xe_mmio_probe_early, ERRNO); /* See xe_pci_probe() */ 125 126 /** 127 * xe_mmio_init() - Initialize an MMIO instance 128 * @mmio: Pointer to the MMIO instance to initialize 129 * @tile: The tile to which the MMIO region belongs 130 * @ptr: Pointer to the start of the MMIO region 131 * @size: The size of the MMIO region in bytes 132 * 133 * This is a convenience function for minimal initialization of struct xe_mmio. 134 */ 135 void xe_mmio_init(struct xe_mmio *mmio, struct xe_tile *tile, void __iomem *ptr, u32 size) 136 { 137 xe_tile_assert(tile, size <= XE_REG_ADDR_MAX); 138 139 mmio->regs = ptr; 140 mmio->regs_size = size; 141 mmio->tile = tile; 142 } 143 144 static bool mmio_available(struct xe_mmio *mmio) 145 { 146 return !xe_tile_WARN_ON_ONCE(mmio->tile, !mmio->tile->xe->mmio.regs); 147 } 148 149 static void mmio_flush_pending_writes(struct xe_mmio *mmio) 150 { 151 #define DUMMY_REG_OFFSET 0x130030 152 int i; 153 154 if (!XE_DEVICE_WA(mmio->tile->xe, 15015404425)) 155 return; 156 157 /* 4 dummy writes */ 158 for (i = 0; i < 4; i++) 159 writel(0, mmio->regs + DUMMY_REG_OFFSET); 160 } 161 162 u8 xe_mmio_read8(struct xe_mmio *mmio, struct xe_reg reg) 163 { 164 u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr); 165 u8 val; 166 167 if (!mmio_available(mmio)) 168 return 0; 169 170 mmio_flush_pending_writes(mmio); 171 172 val = readb(mmio->regs + addr); 173 trace_xe_reg_rw(mmio, false, addr, val, sizeof(val)); 174 175 return val; 176 } 177 178 void xe_mmio_write8(struct xe_mmio *mmio, struct xe_reg reg, u8 val) 179 { 180 u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr); 181 182 if (!mmio_available(mmio)) 183 return; 184 185 trace_xe_reg_rw(mmio, true, addr, val, sizeof(val)); 186 187 writeb(val, mmio->regs + addr); 188 } 189 190 u16 xe_mmio_read16(struct xe_mmio *mmio, struct xe_reg reg) 191 { 192 u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr); 193 u16 val; 194 195 if (!mmio_available(mmio)) 196 return 0; 197 198 mmio_flush_pending_writes(mmio); 199 200 val = readw(mmio->regs + addr); 201 trace_xe_reg_rw(mmio, false, addr, val, sizeof(val)); 202 203 return val; 204 } 205 206 void xe_mmio_write32(struct xe_mmio *mmio, struct xe_reg reg, u32 val) 207 { 208 u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr); 209 210 if (!mmio_available(mmio)) 211 return; 212 213 trace_xe_reg_rw(mmio, true, addr, val, sizeof(val)); 214 215 if (!reg.vf && IS_SRIOV_VF(mmio->tile->xe)) 216 xe_gt_sriov_vf_write32(mmio->sriov_vf_gt ?: 217 mmio->tile->primary_gt, reg, val); 218 else 219 writel(val, mmio->regs + addr); 220 } 221 222 u32 xe_mmio_read32(struct xe_mmio *mmio, struct xe_reg reg) 223 { 224 u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr); 225 u32 val; 226 227 if (!mmio_available(mmio)) 228 return 0; 229 230 mmio_flush_pending_writes(mmio); 231 232 if (!reg.vf && IS_SRIOV_VF(mmio->tile->xe)) 233 val = xe_gt_sriov_vf_read32(mmio->sriov_vf_gt ?: 234 mmio->tile->primary_gt, reg); 235 else 236 val = readl(mmio->regs + addr); 237 238 trace_xe_reg_rw(mmio, false, addr, val, sizeof(val)); 239 240 return val; 241 } 242 243 u32 xe_mmio_rmw32(struct xe_mmio *mmio, struct xe_reg reg, u32 clr, u32 set) 244 { 245 u32 old, reg_val; 246 247 old = xe_mmio_read32(mmio, reg); 248 reg_val = (old & ~clr) | set; 249 xe_mmio_write32(mmio, reg, reg_val); 250 251 return old; 252 } 253 254 int xe_mmio_write32_and_verify(struct xe_mmio *mmio, 255 struct xe_reg reg, u32 val, u32 mask, u32 eval) 256 { 257 u32 reg_val; 258 259 xe_mmio_write32(mmio, reg, val); 260 reg_val = xe_mmio_read32(mmio, reg); 261 262 return (reg_val & mask) != eval ? -EINVAL : 0; 263 } 264 265 bool xe_mmio_in_range(const struct xe_mmio *mmio, 266 const struct xe_mmio_range *range, 267 struct xe_reg reg) 268 { 269 u32 addr = xe_mmio_adjusted_addr(mmio, reg.addr); 270 271 return range && addr >= range->start && addr <= range->end; 272 } 273 274 /** 275 * xe_mmio_read64_2x32() - Read a 64-bit register as two 32-bit reads 276 * @mmio: MMIO target 277 * @reg: register to read value from 278 * 279 * Although Intel GPUs have some 64-bit registers, the hardware officially 280 * only supports GTTMMADR register reads of 32 bits or smaller. Even if 281 * a readq operation may return a reasonable value, that violation of the 282 * spec shouldn't be relied upon and all 64-bit register reads should be 283 * performed as two 32-bit reads of the upper and lower dwords. 284 * 285 * When reading registers that may be changing (such as 286 * counters), a rollover of the lower dword between the two 32-bit reads 287 * can be problematic. This function attempts to ensure the upper dword has 288 * stabilized before returning the 64-bit value. 289 * 290 * Note that because this function may re-read the register multiple times 291 * while waiting for the value to stabilize it should not be used to read 292 * any registers where read operations have side effects. 293 * 294 * Returns the value of the 64-bit register. 295 */ 296 u64 xe_mmio_read64_2x32(struct xe_mmio *mmio, struct xe_reg reg) 297 { 298 struct xe_reg reg_udw = { .addr = reg.addr + 0x4 }; 299 u32 ldw, udw, oldudw, retries; 300 301 /* 302 * The two dwords of a 64-bit register can never straddle the offset 303 * adjustment cutoff. 304 */ 305 xe_tile_assert(mmio->tile, !in_range(mmio->adj_limit, reg.addr + 1, 7)); 306 307 oldudw = xe_mmio_read32(mmio, reg_udw); 308 for (retries = 5; retries; --retries) { 309 ldw = xe_mmio_read32(mmio, reg); 310 udw = xe_mmio_read32(mmio, reg_udw); 311 312 if (udw == oldudw) 313 break; 314 315 oldudw = udw; 316 } 317 318 xe_tile_WARN(mmio->tile, retries == 0, 319 "MMIO: 64-bit read of %#x did not stabilize\n", reg.addr); 320 321 return (u64)udw << 32 | ldw; 322 } 323 324 #define __XE_MMIO_WAIT_MAX_BACKOFF_100MS (100 * USEC_PER_MSEC) 325 static int __xe_mmio_wait32(struct xe_mmio *mmio, struct xe_reg reg, u32 mask, u32 val, 326 u32 timeout_us, u32 *out_val, bool atomic, bool expect_match) 327 { 328 ktime_t cur = ktime_get_raw(); 329 const ktime_t end = ktime_add_us(cur, timeout_us); 330 int ret = -ETIMEDOUT; 331 s64 wait = 10; 332 u32 read; 333 bool check; 334 335 for (;;) { 336 read = xe_mmio_read32(mmio, reg); 337 338 check = (read & mask) == val; 339 if (!expect_match) 340 check = !check; 341 342 if (check) { 343 ret = 0; 344 break; 345 } 346 347 cur = ktime_get_raw(); 348 if (!ktime_before(cur, end)) 349 break; 350 351 if (ktime_after(ktime_add_us(cur, wait), end)) 352 wait = ktime_us_delta(end, cur); 353 354 if (atomic) 355 udelay(min_t(s64, wait, (MAX_UDELAY_MS * 1000))); 356 else 357 usleep_range(wait, wait << 1); 358 wait = min_t(s64, wait << 1, __XE_MMIO_WAIT_MAX_BACKOFF_100MS); 359 } 360 361 if (ret != 0) { 362 read = xe_mmio_read32(mmio, reg); 363 364 check = (read & mask) == val; 365 if (!expect_match) 366 check = !check; 367 368 if (check) 369 ret = 0; 370 } 371 372 if (out_val) 373 *out_val = read; 374 375 return ret; 376 } 377 378 /** 379 * xe_mmio_wait32() - Wait for a register to match the desired masked value 380 * @mmio: MMIO target 381 * @reg: register to read value from 382 * @mask: mask to be applied to the value read from the register 383 * @val: desired value after applying the mask 384 * @timeout_us: time out after this period of time. Wait logic tries to be 385 * smart, applying an exponential backoff until @timeout_us is reached. 386 * @out_val: if not NULL, points where to store the last unmasked value 387 * @atomic: needs to be true if calling from an atomic context 388 * 389 * This function polls for the desired masked value and returns zero on success 390 * or -ETIMEDOUT if timed out. 391 * 392 * Note that @timeout_us represents the minimum amount of time to wait before 393 * giving up. The actual time taken by this function can be a little more than 394 * @timeout_us for different reasons, specially in non-atomic contexts. Thus, 395 * it is possible that this function succeeds even after @timeout_us has passed. 396 */ 397 int xe_mmio_wait32(struct xe_mmio *mmio, struct xe_reg reg, u32 mask, u32 val, u32 timeout_us, 398 u32 *out_val, bool atomic) 399 { 400 return __xe_mmio_wait32(mmio, reg, mask, val, timeout_us, out_val, atomic, true); 401 } 402 403 /** 404 * xe_mmio_wait32_not() - Wait for a register to return anything other than the given masked value 405 * @mmio: MMIO target 406 * @reg: register to read value from 407 * @mask: mask to be applied to the value read from the register 408 * @val: value not to be matched after applying the mask 409 * @timeout_us: time out after this period of time 410 * @out_val: if not NULL, points where to store the last unmasked value 411 * @atomic: needs to be true if calling from an atomic context 412 * 413 * This function works exactly like xe_mmio_wait32() with the exception that 414 * @val is expected not to be matched. 415 */ 416 int xe_mmio_wait32_not(struct xe_mmio *mmio, struct xe_reg reg, u32 mask, u32 val, u32 timeout_us, 417 u32 *out_val, bool atomic) 418 { 419 return __xe_mmio_wait32(mmio, reg, mask, val, timeout_us, out_val, atomic, false); 420 } 421 422 #ifdef CONFIG_PCI_IOV 423 static size_t vf_regs_stride(struct xe_device *xe) 424 { 425 return GRAPHICS_VERx100(xe) > 1200 ? 0x400 : 0x1000; 426 } 427 428 /** 429 * xe_mmio_init_vf_view() - Initialize an MMIO instance for accesses like the VF 430 * @mmio: the target &xe_mmio to initialize as VF's view 431 * @base: the source &xe_mmio to initialize from 432 * @vfid: the VF identifier 433 */ 434 void xe_mmio_init_vf_view(struct xe_mmio *mmio, const struct xe_mmio *base, unsigned int vfid) 435 { 436 struct xe_tile *tile = base->tile; 437 struct xe_device *xe = tile->xe; 438 size_t offset = vf_regs_stride(xe) * vfid; 439 440 xe_assert(xe, IS_SRIOV_PF(xe)); 441 xe_assert(xe, vfid); 442 xe_assert(xe, !base->sriov_vf_gt); 443 xe_assert(xe, base->regs_size > offset); 444 445 *mmio = *base; 446 mmio->regs += offset; 447 mmio->regs_size -= offset; 448 } 449 #endif 450