1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright 2025 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include <linux/delay.h> 25 26 #include "amdgpu.h" 27 #include "amdgpu_reset.h" 28 #include "amdgpu_trace.h" 29 #include "amdgpu_virt.h" 30 #include "amdgpu_reg_access.h" 31 32 #define AMDGPU_PCIE_INDEX_FALLBACK (0x38 >> 2) 33 #define AMDGPU_PCIE_INDEX_HI_FALLBACK (0x44 >> 2) 34 #define AMDGPU_PCIE_DATA_FALLBACK (0x3C >> 2) 35 36 void amdgpu_reg_access_init(struct amdgpu_device *adev) 37 { 38 spin_lock_init(&adev->reg.smc.lock); 39 adev->reg.smc.rreg = NULL; 40 adev->reg.smc.wreg = NULL; 41 42 spin_lock_init(&adev->reg.uvd_ctx.lock); 43 adev->reg.uvd_ctx.rreg = NULL; 44 adev->reg.uvd_ctx.wreg = NULL; 45 46 spin_lock_init(&adev->reg.didt.lock); 47 adev->reg.didt.rreg = NULL; 48 adev->reg.didt.wreg = NULL; 49 50 spin_lock_init(&adev->reg.gc_cac.lock); 51 adev->reg.gc_cac.rreg = NULL; 52 adev->reg.gc_cac.wreg = NULL; 53 54 spin_lock_init(&adev->reg.se_cac.lock); 55 adev->reg.se_cac.rreg = NULL; 56 adev->reg.se_cac.wreg = NULL; 57 58 spin_lock_init(&adev->reg.audio_endpt.lock); 59 adev->reg.audio_endpt.rreg = NULL; 60 adev->reg.audio_endpt.wreg = NULL; 61 62 spin_lock_init(&adev->reg.pcie.lock); 63 adev->reg.pcie.rreg = NULL; 64 adev->reg.pcie.wreg = NULL; 65 adev->reg.pcie.rreg_ext = NULL; 66 adev->reg.pcie.wreg_ext = NULL; 67 adev->reg.pcie.rreg64 = NULL; 68 adev->reg.pcie.wreg64 = NULL; 69 adev->reg.pcie.rreg64_ext = NULL; 70 adev->reg.pcie.wreg64_ext = NULL; 71 adev->reg.pcie.port_rreg = NULL; 72 adev->reg.pcie.port_wreg = NULL; 73 } 74 75 uint32_t amdgpu_reg_smc_rd32(struct amdgpu_device *adev, uint32_t reg) 76 { 77 if (!adev->reg.smc.rreg) { 78 dev_err_once(adev->dev, "SMC register read not supported\n"); 79 return 0; 80 } 81 return adev->reg.smc.rreg(adev, reg); 82 } 83 84 void amdgpu_reg_smc_wr32(struct amdgpu_device *adev, uint32_t reg, uint32_t v) 85 { 86 if (!adev->reg.smc.wreg) { 87 dev_err_once(adev->dev, "SMC register write not supported\n"); 88 return; 89 } 90 adev->reg.smc.wreg(adev, reg, v); 91 } 92 93 uint32_t amdgpu_reg_uvd_ctx_rd32(struct amdgpu_device *adev, uint32_t reg) 94 { 95 if (!adev->reg.uvd_ctx.rreg) { 96 dev_err_once(adev->dev, 97 "UVD_CTX register read not supported\n"); 98 return 0; 99 } 100 return adev->reg.uvd_ctx.rreg(adev, reg); 101 } 102 103 void amdgpu_reg_uvd_ctx_wr32(struct amdgpu_device *adev, uint32_t reg, 104 uint32_t v) 105 { 106 if (!adev->reg.uvd_ctx.wreg) { 107 dev_err_once(adev->dev, 108 "UVD_CTX register write not supported\n"); 109 return; 110 } 111 adev->reg.uvd_ctx.wreg(adev, reg, v); 112 } 113 114 uint32_t amdgpu_reg_didt_rd32(struct amdgpu_device *adev, uint32_t reg) 115 { 116 if (!adev->reg.didt.rreg) { 117 dev_err_once(adev->dev, "DIDT register read not supported\n"); 118 return 0; 119 } 120 return adev->reg.didt.rreg(adev, reg); 121 } 122 123 void amdgpu_reg_didt_wr32(struct amdgpu_device *adev, uint32_t reg, uint32_t v) 124 { 125 if (!adev->reg.didt.wreg) { 126 dev_err_once(adev->dev, "DIDT register write not supported\n"); 127 return; 128 } 129 adev->reg.didt.wreg(adev, reg, v); 130 } 131 132 uint32_t amdgpu_reg_gc_cac_rd32(struct amdgpu_device *adev, uint32_t reg) 133 { 134 if (!adev->reg.gc_cac.rreg) { 135 dev_err_once(adev->dev, "GC_CAC register read not supported\n"); 136 return 0; 137 } 138 return adev->reg.gc_cac.rreg(adev, reg); 139 } 140 141 void amdgpu_reg_gc_cac_wr32(struct amdgpu_device *adev, uint32_t reg, 142 uint32_t v) 143 { 144 if (!adev->reg.gc_cac.wreg) { 145 dev_err_once(adev->dev, 146 "GC_CAC register write not supported\n"); 147 return; 148 } 149 adev->reg.gc_cac.wreg(adev, reg, v); 150 } 151 152 uint32_t amdgpu_reg_se_cac_rd32(struct amdgpu_device *adev, uint32_t reg) 153 { 154 if (!adev->reg.se_cac.rreg) { 155 dev_err_once(adev->dev, "SE_CAC register read not supported\n"); 156 return 0; 157 } 158 return adev->reg.se_cac.rreg(adev, reg); 159 } 160 161 void amdgpu_reg_se_cac_wr32(struct amdgpu_device *adev, uint32_t reg, 162 uint32_t v) 163 { 164 if (!adev->reg.se_cac.wreg) { 165 dev_err_once(adev->dev, 166 "SE_CAC register write not supported\n"); 167 return; 168 } 169 adev->reg.se_cac.wreg(adev, reg, v); 170 } 171 172 uint32_t amdgpu_reg_audio_endpt_rd32(struct amdgpu_device *adev, uint32_t block, 173 uint32_t reg) 174 { 175 if (!adev->reg.audio_endpt.rreg) { 176 dev_err_once(adev->dev, 177 "AUDIO_ENDPT register read not supported\n"); 178 return 0; 179 } 180 return adev->reg.audio_endpt.rreg(adev, block, reg); 181 } 182 183 void amdgpu_reg_audio_endpt_wr32(struct amdgpu_device *adev, uint32_t block, 184 uint32_t reg, uint32_t v) 185 { 186 if (!adev->reg.audio_endpt.wreg) { 187 dev_err_once(adev->dev, 188 "AUDIO_ENDPT register write not supported\n"); 189 return; 190 } 191 adev->reg.audio_endpt.wreg(adev, block, reg, v); 192 } 193 194 uint32_t amdgpu_reg_pcie_rd32(struct amdgpu_device *adev, uint32_t reg) 195 { 196 if (!adev->reg.pcie.rreg) { 197 dev_err_once(adev->dev, "PCIE register read not supported\n"); 198 return 0; 199 } 200 return adev->reg.pcie.rreg(adev, reg); 201 } 202 203 void amdgpu_reg_pcie_wr32(struct amdgpu_device *adev, uint32_t reg, uint32_t v) 204 { 205 if (!adev->reg.pcie.wreg) { 206 dev_err_once(adev->dev, "PCIE register write not supported\n"); 207 return; 208 } 209 adev->reg.pcie.wreg(adev, reg, v); 210 } 211 212 uint32_t amdgpu_reg_pcie_ext_rd32(struct amdgpu_device *adev, uint64_t reg) 213 { 214 if (!adev->reg.pcie.rreg_ext) { 215 dev_err_once(adev->dev, "PCIE EXT register read not supported\n"); 216 return 0; 217 } 218 return adev->reg.pcie.rreg_ext(adev, reg); 219 } 220 221 void amdgpu_reg_pcie_ext_wr32(struct amdgpu_device *adev, uint64_t reg, 222 uint32_t v) 223 { 224 if (!adev->reg.pcie.wreg_ext) { 225 dev_err_once(adev->dev, "PCIE EXT register write not supported\n"); 226 return; 227 } 228 adev->reg.pcie.wreg_ext(adev, reg, v); 229 } 230 231 uint64_t amdgpu_reg_pcie_rd64(struct amdgpu_device *adev, uint32_t reg) 232 { 233 if (!adev->reg.pcie.rreg64) { 234 dev_err_once(adev->dev, "PCIE 64-bit register read not supported\n"); 235 return 0; 236 } 237 return adev->reg.pcie.rreg64(adev, reg); 238 } 239 240 void amdgpu_reg_pcie_wr64(struct amdgpu_device *adev, uint32_t reg, uint64_t v) 241 { 242 if (!adev->reg.pcie.wreg64) { 243 dev_err_once(adev->dev, "PCIE 64-bit register write not supported\n"); 244 return; 245 } 246 adev->reg.pcie.wreg64(adev, reg, v); 247 } 248 249 uint64_t amdgpu_reg_pcie_ext_rd64(struct amdgpu_device *adev, uint64_t reg) 250 { 251 if (!adev->reg.pcie.rreg64_ext) { 252 dev_err_once(adev->dev, "PCIE EXT 64-bit register read not supported\n"); 253 return 0; 254 } 255 return adev->reg.pcie.rreg64_ext(adev, reg); 256 } 257 258 void amdgpu_reg_pcie_ext_wr64(struct amdgpu_device *adev, uint64_t reg, 259 uint64_t v) 260 { 261 if (!adev->reg.pcie.wreg64_ext) { 262 dev_err_once(adev->dev, "PCIE EXT 64-bit register write not supported\n"); 263 return; 264 } 265 adev->reg.pcie.wreg64_ext(adev, reg, v); 266 } 267 268 uint32_t amdgpu_reg_pciep_rd32(struct amdgpu_device *adev, uint32_t reg) 269 { 270 if (!adev->reg.pcie.port_rreg) { 271 dev_err_once(adev->dev, "PCIEP register read not supported\n"); 272 return 0; 273 } 274 return adev->reg.pcie.port_rreg(adev, reg); 275 } 276 277 void amdgpu_reg_pciep_wr32(struct amdgpu_device *adev, uint32_t reg, uint32_t v) 278 { 279 if (!adev->reg.pcie.port_wreg) { 280 dev_err_once(adev->dev, "PCIEP register write not supported\n"); 281 return; 282 } 283 adev->reg.pcie.port_wreg(adev, reg, v); 284 } 285 286 static int amdgpu_reg_get_smn_base_version(struct amdgpu_device *adev) 287 { 288 struct pci_dev *pdev = adev->pdev; 289 int id; 290 291 if (amdgpu_sriov_vf(adev)) 292 return -EOPNOTSUPP; 293 294 id = (pdev->device >> 4) & 0xFFFF; 295 if (id == 0x74A || id == 0x74B || id == 0x75A || id == 0x75B) 296 return 1; 297 298 return -EOPNOTSUPP; 299 } 300 301 uint64_t amdgpu_reg_get_smn_base64(struct amdgpu_device *adev, 302 enum amd_hw_ip_block_type block, 303 int die_inst) 304 { 305 if (!adev->reg.smn.get_smn_base) { 306 int version = amdgpu_reg_get_smn_base_version(adev); 307 switch (version) { 308 case 1: 309 return amdgpu_reg_smn_v1_0_get_base(adev, block, 310 die_inst); 311 default: 312 dev_err_once( 313 adev->dev, 314 "SMN base address query not supported for this device\n"); 315 return 0; 316 } 317 } 318 return adev->reg.smn.get_smn_base(adev, block, die_inst); 319 } 320 321 uint64_t amdgpu_reg_smn_v1_0_get_base(struct amdgpu_device *adev, 322 enum amd_hw_ip_block_type block, 323 int die_inst) 324 { 325 uint64_t smn_base; 326 327 if (die_inst == 0) 328 return 0; 329 330 switch (block) { 331 case XGMI_HWIP: 332 case NBIO_HWIP: 333 case MP0_HWIP: 334 case UMC_HWIP: 335 case DF_HWIP: 336 smn_base = ((uint64_t)(die_inst & 0x3) << 32) | (1ULL << 34); 337 break; 338 default: 339 dev_warn_once( 340 adev->dev, 341 "SMN base address query not supported for this block %d\n", 342 block); 343 smn_base = 0; 344 break; 345 } 346 347 return smn_base; 348 } 349 350 /* 351 * register access helper functions. 352 */ 353 354 /** 355 * amdgpu_device_rreg - read a memory mapped IO or indirect register 356 * 357 * @adev: amdgpu_device pointer 358 * @reg: dword aligned register offset 359 * @acc_flags: access flags which require special behavior 360 * 361 * Returns the 32 bit value from the offset specified. 362 */ 363 uint32_t amdgpu_device_rreg(struct amdgpu_device *adev, uint32_t reg, 364 uint32_t acc_flags) 365 { 366 uint32_t ret; 367 368 if (amdgpu_device_skip_hw_access(adev)) 369 return 0; 370 371 if ((reg * 4) < adev->rmmio_size) { 372 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && 373 amdgpu_sriov_runtime(adev) && 374 down_read_trylock(&adev->reset_domain->sem)) { 375 ret = amdgpu_kiq_rreg(adev, reg, 0); 376 up_read(&adev->reset_domain->sem); 377 } else { 378 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); 379 } 380 } else { 381 ret = amdgpu_reg_pcie_rd32(adev, reg * 4); 382 } 383 384 trace_amdgpu_device_rreg(adev->pdev->device, reg, ret); 385 386 return ret; 387 } 388 389 /* 390 * MMIO register read with bytes helper functions 391 * @offset:bytes offset from MMIO start 392 */ 393 394 /** 395 * amdgpu_mm_rreg8 - read a memory mapped IO register 396 * 397 * @adev: amdgpu_device pointer 398 * @offset: byte aligned register offset 399 * 400 * Returns the 8 bit value from the offset specified. 401 */ 402 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset) 403 { 404 if (amdgpu_device_skip_hw_access(adev)) 405 return 0; 406 407 if (offset < adev->rmmio_size) 408 return (readb(adev->rmmio + offset)); 409 BUG(); 410 } 411 412 /** 413 * amdgpu_device_xcc_rreg - read a memory mapped IO or indirect register with specific XCC 414 * 415 * @adev: amdgpu_device pointer 416 * @reg: dword aligned register offset 417 * @acc_flags: access flags which require special behavior 418 * @xcc_id: xcc accelerated compute core id 419 * 420 * Returns the 32 bit value from the offset specified. 421 */ 422 uint32_t amdgpu_device_xcc_rreg(struct amdgpu_device *adev, uint32_t reg, 423 uint32_t acc_flags, uint32_t xcc_id) 424 { 425 uint32_t ret, rlcg_flag; 426 427 if (amdgpu_device_skip_hw_access(adev)) 428 return 0; 429 430 if ((reg * 4) < adev->rmmio_size) { 431 if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_runtime(adev) && 432 adev->gfx.rlc.rlcg_reg_access_supported && 433 amdgpu_virt_get_rlcg_reg_access_flag( 434 adev, acc_flags, GC_HWIP, false, &rlcg_flag)) { 435 ret = amdgpu_virt_rlcg_reg_rw(adev, reg, 0, rlcg_flag, 436 GET_INST(GC, xcc_id)); 437 } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && 438 amdgpu_sriov_runtime(adev) && 439 down_read_trylock(&adev->reset_domain->sem)) { 440 ret = amdgpu_kiq_rreg(adev, reg, xcc_id); 441 up_read(&adev->reset_domain->sem); 442 } else { 443 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); 444 } 445 } else { 446 ret = amdgpu_reg_pcie_rd32(adev, reg * 4); 447 } 448 449 return ret; 450 } 451 452 /* 453 * MMIO register write with bytes helper functions 454 * @offset:bytes offset from MMIO start 455 * @value: the value want to be written to the register 456 */ 457 458 /** 459 * amdgpu_mm_wreg8 - read a memory mapped IO register 460 * 461 * @adev: amdgpu_device pointer 462 * @offset: byte aligned register offset 463 * @value: 8 bit value to write 464 * 465 * Writes the value specified to the offset specified. 466 */ 467 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value) 468 { 469 if (amdgpu_device_skip_hw_access(adev)) 470 return; 471 472 if (offset < adev->rmmio_size) 473 writeb(value, adev->rmmio + offset); 474 else 475 BUG(); 476 } 477 478 /** 479 * amdgpu_device_wreg - write to a memory mapped IO or indirect register 480 * 481 * @adev: amdgpu_device pointer 482 * @reg: dword aligned register offset 483 * @v: 32 bit value to write to the register 484 * @acc_flags: access flags which require special behavior 485 * 486 * Writes the value specified to the offset specified. 487 */ 488 void amdgpu_device_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, 489 uint32_t acc_flags) 490 { 491 if (amdgpu_device_skip_hw_access(adev)) 492 return; 493 494 if ((reg * 4) < adev->rmmio_size) { 495 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && 496 amdgpu_sriov_runtime(adev) && 497 down_read_trylock(&adev->reset_domain->sem)) { 498 amdgpu_kiq_wreg(adev, reg, v, 0); 499 up_read(&adev->reset_domain->sem); 500 } else { 501 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); 502 } 503 } else { 504 amdgpu_reg_pcie_wr32(adev, reg * 4, v); 505 } 506 507 trace_amdgpu_device_wreg(adev->pdev->device, reg, v); 508 } 509 510 /** 511 * amdgpu_mm_wreg_mmio_rlc - write register either with direct/indirect mmio or with RLC path if in range 512 * 513 * @adev: amdgpu_device pointer 514 * @reg: mmio/rlc register 515 * @v: value to write 516 * @xcc_id: xcc accelerated compute core id 517 * 518 * this function is invoked only for the debugfs register access 519 */ 520 void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev, uint32_t reg, 521 uint32_t v, uint32_t xcc_id) 522 { 523 if (amdgpu_device_skip_hw_access(adev)) 524 return; 525 526 if (amdgpu_sriov_fullaccess(adev) && adev->gfx.rlc.funcs && 527 adev->gfx.rlc.funcs->is_rlcg_access_range) { 528 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg)) 529 return amdgpu_sriov_wreg(adev, reg, v, 0, 0, xcc_id); 530 } else if ((reg * 4) >= adev->rmmio_size) { 531 amdgpu_reg_pcie_wr32(adev, reg * 4, v); 532 } else { 533 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); 534 } 535 } 536 537 /** 538 * amdgpu_device_xcc_wreg - write to a memory mapped IO or indirect register with specific XCC 539 * 540 * @adev: amdgpu_device pointer 541 * @reg: dword aligned register offset 542 * @v: 32 bit value to write to the register 543 * @acc_flags: access flags which require special behavior 544 * @xcc_id: xcc accelerated compute core id 545 * 546 * Writes the value specified to the offset specified. 547 */ 548 void amdgpu_device_xcc_wreg(struct amdgpu_device *adev, uint32_t reg, 549 uint32_t v, uint32_t acc_flags, uint32_t xcc_id) 550 { 551 uint32_t rlcg_flag; 552 553 if (amdgpu_device_skip_hw_access(adev)) 554 return; 555 556 if ((reg * 4) < adev->rmmio_size) { 557 if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_runtime(adev) && 558 adev->gfx.rlc.rlcg_reg_access_supported && 559 amdgpu_virt_get_rlcg_reg_access_flag( 560 adev, acc_flags, GC_HWIP, true, &rlcg_flag)) { 561 amdgpu_virt_rlcg_reg_rw(adev, reg, v, rlcg_flag, 562 GET_INST(GC, xcc_id)); 563 } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && 564 amdgpu_sriov_runtime(adev) && 565 down_read_trylock(&adev->reset_domain->sem)) { 566 amdgpu_kiq_wreg(adev, reg, v, xcc_id); 567 up_read(&adev->reset_domain->sem); 568 } else { 569 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); 570 } 571 } else { 572 amdgpu_reg_pcie_wr32(adev, reg * 4, v); 573 } 574 } 575 576 /** 577 * amdgpu_device_indirect_rreg - read an indirect register 578 * 579 * @adev: amdgpu_device pointer 580 * @reg_addr: indirect register address to read from 581 * 582 * Returns the value of indirect register @reg_addr 583 */ 584 u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev, u32 reg_addr) 585 { 586 unsigned long flags, pcie_index, pcie_data; 587 void __iomem *pcie_index_offset; 588 void __iomem *pcie_data_offset; 589 u32 r; 590 591 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 592 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 593 594 spin_lock_irqsave(&adev->reg.pcie.lock, flags); 595 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 596 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 597 598 writel(reg_addr, pcie_index_offset); 599 readl(pcie_index_offset); 600 r = readl(pcie_data_offset); 601 spin_unlock_irqrestore(&adev->reg.pcie.lock, flags); 602 603 return r; 604 } 605 606 u32 amdgpu_device_indirect_rreg_ext(struct amdgpu_device *adev, u64 reg_addr) 607 { 608 unsigned long flags, pcie_index, pcie_index_hi, pcie_data; 609 u32 r; 610 void __iomem *pcie_index_offset; 611 void __iomem *pcie_index_hi_offset; 612 void __iomem *pcie_data_offset; 613 614 if (unlikely(!adev->nbio.funcs)) { 615 pcie_index = AMDGPU_PCIE_INDEX_FALLBACK; 616 pcie_data = AMDGPU_PCIE_DATA_FALLBACK; 617 } else { 618 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 619 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 620 } 621 622 if (reg_addr >> 32) { 623 if (unlikely(!adev->nbio.funcs)) 624 pcie_index_hi = AMDGPU_PCIE_INDEX_HI_FALLBACK; 625 else 626 pcie_index_hi = 627 adev->nbio.funcs->get_pcie_index_hi_offset( 628 adev); 629 } else { 630 pcie_index_hi = 0; 631 } 632 633 spin_lock_irqsave(&adev->reg.pcie.lock, flags); 634 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 635 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 636 if (pcie_index_hi != 0) 637 pcie_index_hi_offset = 638 (void __iomem *)adev->rmmio + pcie_index_hi * 4; 639 640 writel(reg_addr, pcie_index_offset); 641 readl(pcie_index_offset); 642 if (pcie_index_hi != 0) { 643 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); 644 readl(pcie_index_hi_offset); 645 } 646 r = readl(pcie_data_offset); 647 648 /* clear the high bits */ 649 if (pcie_index_hi != 0) { 650 writel(0, pcie_index_hi_offset); 651 readl(pcie_index_hi_offset); 652 } 653 654 spin_unlock_irqrestore(&adev->reg.pcie.lock, flags); 655 656 return r; 657 } 658 659 /** 660 * amdgpu_device_indirect_rreg64 - read a 64bits indirect register 661 * 662 * @adev: amdgpu_device pointer 663 * @reg_addr: indirect register address to read from 664 * 665 * Returns the value of indirect register @reg_addr 666 */ 667 u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev, u32 reg_addr) 668 { 669 unsigned long flags, pcie_index, pcie_data; 670 void __iomem *pcie_index_offset; 671 void __iomem *pcie_data_offset; 672 u64 r; 673 674 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 675 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 676 677 spin_lock_irqsave(&adev->reg.pcie.lock, flags); 678 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 679 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 680 681 /* read low 32 bits */ 682 writel(reg_addr, pcie_index_offset); 683 readl(pcie_index_offset); 684 r = readl(pcie_data_offset); 685 /* read high 32 bits */ 686 writel(reg_addr + 4, pcie_index_offset); 687 readl(pcie_index_offset); 688 r |= ((u64)readl(pcie_data_offset) << 32); 689 spin_unlock_irqrestore(&adev->reg.pcie.lock, flags); 690 691 return r; 692 } 693 694 u64 amdgpu_device_indirect_rreg64_ext(struct amdgpu_device *adev, u64 reg_addr) 695 { 696 unsigned long flags, pcie_index, pcie_data; 697 unsigned long pcie_index_hi = 0; 698 void __iomem *pcie_index_offset; 699 void __iomem *pcie_index_hi_offset; 700 void __iomem *pcie_data_offset; 701 u64 r; 702 703 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 704 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 705 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset)) 706 pcie_index_hi = 707 adev->nbio.funcs->get_pcie_index_hi_offset(adev); 708 709 spin_lock_irqsave(&adev->reg.pcie.lock, flags); 710 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 711 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 712 if (pcie_index_hi != 0) 713 pcie_index_hi_offset = 714 (void __iomem *)adev->rmmio + pcie_index_hi * 4; 715 716 /* read low 32 bits */ 717 writel(reg_addr, pcie_index_offset); 718 readl(pcie_index_offset); 719 if (pcie_index_hi != 0) { 720 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); 721 readl(pcie_index_hi_offset); 722 } 723 r = readl(pcie_data_offset); 724 /* read high 32 bits */ 725 writel(reg_addr + 4, pcie_index_offset); 726 readl(pcie_index_offset); 727 if (pcie_index_hi != 0) { 728 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); 729 readl(pcie_index_hi_offset); 730 } 731 r |= ((u64)readl(pcie_data_offset) << 32); 732 733 /* clear the high bits */ 734 if (pcie_index_hi != 0) { 735 writel(0, pcie_index_hi_offset); 736 readl(pcie_index_hi_offset); 737 } 738 739 spin_unlock_irqrestore(&adev->reg.pcie.lock, flags); 740 741 return r; 742 } 743 744 /** 745 * amdgpu_device_indirect_wreg - write an indirect register address 746 * 747 * @adev: amdgpu_device pointer 748 * @reg_addr: indirect register offset 749 * @reg_data: indirect register data 750 * 751 */ 752 void amdgpu_device_indirect_wreg(struct amdgpu_device *adev, u32 reg_addr, 753 u32 reg_data) 754 { 755 unsigned long flags, pcie_index, pcie_data; 756 void __iomem *pcie_index_offset; 757 void __iomem *pcie_data_offset; 758 759 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 760 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 761 762 spin_lock_irqsave(&adev->reg.pcie.lock, flags); 763 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 764 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 765 766 writel(reg_addr, pcie_index_offset); 767 readl(pcie_index_offset); 768 writel(reg_data, pcie_data_offset); 769 readl(pcie_data_offset); 770 spin_unlock_irqrestore(&adev->reg.pcie.lock, flags); 771 } 772 773 void amdgpu_device_indirect_wreg_ext(struct amdgpu_device *adev, u64 reg_addr, 774 u32 reg_data) 775 { 776 unsigned long flags, pcie_index, pcie_index_hi, pcie_data; 777 void __iomem *pcie_index_offset; 778 void __iomem *pcie_index_hi_offset; 779 void __iomem *pcie_data_offset; 780 781 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 782 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 783 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset)) 784 pcie_index_hi = 785 adev->nbio.funcs->get_pcie_index_hi_offset(adev); 786 else 787 pcie_index_hi = 0; 788 789 spin_lock_irqsave(&adev->reg.pcie.lock, flags); 790 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 791 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 792 if (pcie_index_hi != 0) 793 pcie_index_hi_offset = 794 (void __iomem *)adev->rmmio + pcie_index_hi * 4; 795 796 writel(reg_addr, pcie_index_offset); 797 readl(pcie_index_offset); 798 if (pcie_index_hi != 0) { 799 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); 800 readl(pcie_index_hi_offset); 801 } 802 writel(reg_data, pcie_data_offset); 803 readl(pcie_data_offset); 804 805 /* clear the high bits */ 806 if (pcie_index_hi != 0) { 807 writel(0, pcie_index_hi_offset); 808 readl(pcie_index_hi_offset); 809 } 810 811 spin_unlock_irqrestore(&adev->reg.pcie.lock, flags); 812 } 813 814 /** 815 * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address 816 * 817 * @adev: amdgpu_device pointer 818 * @reg_addr: indirect register offset 819 * @reg_data: indirect register data 820 * 821 */ 822 void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev, u32 reg_addr, 823 u64 reg_data) 824 { 825 unsigned long flags, pcie_index, pcie_data; 826 void __iomem *pcie_index_offset; 827 void __iomem *pcie_data_offset; 828 829 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 830 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 831 832 spin_lock_irqsave(&adev->reg.pcie.lock, flags); 833 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 834 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 835 836 /* write low 32 bits */ 837 writel(reg_addr, pcie_index_offset); 838 readl(pcie_index_offset); 839 writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset); 840 readl(pcie_data_offset); 841 /* write high 32 bits */ 842 writel(reg_addr + 4, pcie_index_offset); 843 readl(pcie_index_offset); 844 writel((u32)(reg_data >> 32), pcie_data_offset); 845 readl(pcie_data_offset); 846 spin_unlock_irqrestore(&adev->reg.pcie.lock, flags); 847 } 848 849 void amdgpu_device_indirect_wreg64_ext(struct amdgpu_device *adev, u64 reg_addr, 850 u64 reg_data) 851 { 852 unsigned long flags, pcie_index, pcie_data; 853 unsigned long pcie_index_hi = 0; 854 void __iomem *pcie_index_offset; 855 void __iomem *pcie_index_hi_offset; 856 void __iomem *pcie_data_offset; 857 858 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 859 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 860 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset)) 861 pcie_index_hi = 862 adev->nbio.funcs->get_pcie_index_hi_offset(adev); 863 864 spin_lock_irqsave(&adev->reg.pcie.lock, flags); 865 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 866 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 867 if (pcie_index_hi != 0) 868 pcie_index_hi_offset = 869 (void __iomem *)adev->rmmio + pcie_index_hi * 4; 870 871 /* write low 32 bits */ 872 writel(reg_addr, pcie_index_offset); 873 readl(pcie_index_offset); 874 if (pcie_index_hi != 0) { 875 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); 876 readl(pcie_index_hi_offset); 877 } 878 writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset); 879 readl(pcie_data_offset); 880 /* write high 32 bits */ 881 writel(reg_addr + 4, pcie_index_offset); 882 readl(pcie_index_offset); 883 if (pcie_index_hi != 0) { 884 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); 885 readl(pcie_index_hi_offset); 886 } 887 writel((u32)(reg_data >> 32), pcie_data_offset); 888 readl(pcie_data_offset); 889 890 /* clear the high bits */ 891 if (pcie_index_hi != 0) { 892 writel(0, pcie_index_hi_offset); 893 readl(pcie_index_hi_offset); 894 } 895 896 spin_unlock_irqrestore(&adev->reg.pcie.lock, flags); 897 } 898 899 u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev, u32 reg) 900 { 901 unsigned long flags, address, data; 902 u32 r; 903 904 address = adev->nbio.funcs->get_pcie_port_index_offset(adev); 905 data = adev->nbio.funcs->get_pcie_port_data_offset(adev); 906 907 spin_lock_irqsave(&adev->reg.pcie.lock, flags); 908 WREG32(address, reg * 4); 909 (void)RREG32(address); 910 r = RREG32(data); 911 spin_unlock_irqrestore(&adev->reg.pcie.lock, flags); 912 return r; 913 } 914 915 void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev, u32 reg, u32 v) 916 { 917 unsigned long flags, address, data; 918 919 address = adev->nbio.funcs->get_pcie_port_index_offset(adev); 920 data = adev->nbio.funcs->get_pcie_port_data_offset(adev); 921 922 spin_lock_irqsave(&adev->reg.pcie.lock, flags); 923 WREG32(address, reg * 4); 924 (void)RREG32(address); 925 WREG32(data, v); 926 (void)RREG32(data); 927 spin_unlock_irqrestore(&adev->reg.pcie.lock, flags); 928 } 929 930 uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev, uint32_t inst, 931 uint32_t reg_addr, char reg_name[], 932 uint32_t expected_value, uint32_t mask) 933 { 934 uint32_t ret = 0; 935 uint32_t old_ = 0; 936 uint32_t tmp_ = RREG32(reg_addr); 937 uint32_t loop = adev->usec_timeout; 938 939 while ((tmp_ & (mask)) != (expected_value)) { 940 if (old_ != tmp_) { 941 loop = adev->usec_timeout; 942 old_ = tmp_; 943 } else 944 udelay(1); 945 tmp_ = RREG32(reg_addr); 946 loop--; 947 if (!loop) { 948 dev_warn( 949 adev->dev, 950 "Register(%d) [%s] failed to reach value 0x%08x != 0x%08xn", 951 inst, reg_name, (uint32_t)expected_value, 952 (uint32_t)(tmp_ & (mask))); 953 ret = -ETIMEDOUT; 954 break; 955 } 956 } 957 return ret; 958 } 959