1 /* 2 * Copyright 2008 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: Dave Airlie 25 * Alex Deucher 26 * Jerome Glisse 27 */ 28 29 #include <linux/aperture.h> 30 #include <linux/power_supply.h> 31 #include <linux/kthread.h> 32 #include <linux/module.h> 33 #include <linux/console.h> 34 #include <linux/slab.h> 35 #include <linux/iommu.h> 36 #include <linux/pci.h> 37 #include <linux/pci-p2pdma.h> 38 #include <linux/apple-gmux.h> 39 40 #include <drm/drm_atomic_helper.h> 41 #include <drm/drm_client_event.h> 42 #include <drm/drm_crtc_helper.h> 43 #include <drm/drm_probe_helper.h> 44 #include <drm/amdgpu_drm.h> 45 #include <linux/device.h> 46 #include <linux/vgaarb.h> 47 #include <linux/vga_switcheroo.h> 48 #include <linux/efi.h> 49 #include "amdgpu.h" 50 #include "amdgpu_trace.h" 51 #include "amdgpu_i2c.h" 52 #include "atom.h" 53 #include "amdgpu_atombios.h" 54 #include "amdgpu_atomfirmware.h" 55 #include "amd_pcie.h" 56 #ifdef CONFIG_DRM_AMDGPU_SI 57 #include "si.h" 58 #endif 59 #ifdef CONFIG_DRM_AMDGPU_CIK 60 #include "cik.h" 61 #endif 62 #include "vi.h" 63 #include "soc15.h" 64 #include "nv.h" 65 #include "bif/bif_4_1_d.h" 66 #include <linux/firmware.h> 67 #include "amdgpu_vf_error.h" 68 69 #include "amdgpu_amdkfd.h" 70 #include "amdgpu_pm.h" 71 72 #include "amdgpu_xgmi.h" 73 #include "amdgpu_ras.h" 74 #include "amdgpu_pmu.h" 75 #include "amdgpu_fru_eeprom.h" 76 #include "amdgpu_reset.h" 77 #include "amdgpu_virt.h" 78 #include "amdgpu_dev_coredump.h" 79 80 #include <linux/suspend.h> 81 #include <drm/task_barrier.h> 82 #include <linux/pm_runtime.h> 83 84 #include <drm/drm_drv.h> 85 86 #if IS_ENABLED(CONFIG_X86) 87 #include <asm/intel-family.h> 88 #endif 89 90 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin"); 91 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin"); 92 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin"); 93 MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin"); 94 MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin"); 95 MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin"); 96 MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin"); 97 98 #define AMDGPU_RESUME_MS 2000 99 #define AMDGPU_MAX_RETRY_LIMIT 2 100 #define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL) 101 #define AMDGPU_PCIE_INDEX_FALLBACK (0x38 >> 2) 102 #define AMDGPU_PCIE_INDEX_HI_FALLBACK (0x44 >> 2) 103 #define AMDGPU_PCIE_DATA_FALLBACK (0x3C >> 2) 104 105 #define AMDGPU_VBIOS_SKIP (1U << 0) 106 #define AMDGPU_VBIOS_OPTIONAL (1U << 1) 107 108 static const struct drm_driver amdgpu_kms_driver; 109 110 const char *amdgpu_asic_name[] = { 111 "TAHITI", 112 "PITCAIRN", 113 "VERDE", 114 "OLAND", 115 "HAINAN", 116 "BONAIRE", 117 "KAVERI", 118 "KABINI", 119 "HAWAII", 120 "MULLINS", 121 "TOPAZ", 122 "TONGA", 123 "FIJI", 124 "CARRIZO", 125 "STONEY", 126 "POLARIS10", 127 "POLARIS11", 128 "POLARIS12", 129 "VEGAM", 130 "VEGA10", 131 "VEGA12", 132 "VEGA20", 133 "RAVEN", 134 "ARCTURUS", 135 "RENOIR", 136 "ALDEBARAN", 137 "NAVI10", 138 "CYAN_SKILLFISH", 139 "NAVI14", 140 "NAVI12", 141 "SIENNA_CICHLID", 142 "NAVY_FLOUNDER", 143 "VANGOGH", 144 "DIMGREY_CAVEFISH", 145 "BEIGE_GOBY", 146 "YELLOW_CARP", 147 "IP DISCOVERY", 148 "LAST", 149 }; 150 151 #define AMDGPU_IP_BLK_MASK_ALL GENMASK(AMD_IP_BLOCK_TYPE_NUM - 1, 0) 152 /* 153 * Default init level where all blocks are expected to be initialized. This is 154 * the level of initialization expected by default and also after a full reset 155 * of the device. 156 */ 157 struct amdgpu_init_level amdgpu_init_default = { 158 .level = AMDGPU_INIT_LEVEL_DEFAULT, 159 .hwini_ip_block_mask = AMDGPU_IP_BLK_MASK_ALL, 160 }; 161 162 struct amdgpu_init_level amdgpu_init_recovery = { 163 .level = AMDGPU_INIT_LEVEL_RESET_RECOVERY, 164 .hwini_ip_block_mask = AMDGPU_IP_BLK_MASK_ALL, 165 }; 166 167 /* 168 * Minimal blocks needed to be initialized before a XGMI hive can be reset. This 169 * is used for cases like reset on initialization where the entire hive needs to 170 * be reset before first use. 171 */ 172 struct amdgpu_init_level amdgpu_init_minimal_xgmi = { 173 .level = AMDGPU_INIT_LEVEL_MINIMAL_XGMI, 174 .hwini_ip_block_mask = 175 BIT(AMD_IP_BLOCK_TYPE_GMC) | BIT(AMD_IP_BLOCK_TYPE_SMC) | 176 BIT(AMD_IP_BLOCK_TYPE_COMMON) | BIT(AMD_IP_BLOCK_TYPE_IH) | 177 BIT(AMD_IP_BLOCK_TYPE_PSP) 178 }; 179 180 static inline bool amdgpu_ip_member_of_hwini(struct amdgpu_device *adev, 181 enum amd_ip_block_type block) 182 { 183 return (adev->init_lvl->hwini_ip_block_mask & (1U << block)) != 0; 184 } 185 186 void amdgpu_set_init_level(struct amdgpu_device *adev, 187 enum amdgpu_init_lvl_id lvl) 188 { 189 switch (lvl) { 190 case AMDGPU_INIT_LEVEL_MINIMAL_XGMI: 191 adev->init_lvl = &amdgpu_init_minimal_xgmi; 192 break; 193 case AMDGPU_INIT_LEVEL_RESET_RECOVERY: 194 adev->init_lvl = &amdgpu_init_recovery; 195 break; 196 case AMDGPU_INIT_LEVEL_DEFAULT: 197 fallthrough; 198 default: 199 adev->init_lvl = &amdgpu_init_default; 200 break; 201 } 202 } 203 204 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev); 205 static int amdgpu_device_pm_notifier(struct notifier_block *nb, unsigned long mode, 206 void *data); 207 208 /** 209 * DOC: pcie_replay_count 210 * 211 * The amdgpu driver provides a sysfs API for reporting the total number 212 * of PCIe replays (NAKs). 213 * The file pcie_replay_count is used for this and returns the total 214 * number of replays as a sum of the NAKs generated and NAKs received. 215 */ 216 217 static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev, 218 struct device_attribute *attr, char *buf) 219 { 220 struct drm_device *ddev = dev_get_drvdata(dev); 221 struct amdgpu_device *adev = drm_to_adev(ddev); 222 uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev); 223 224 return sysfs_emit(buf, "%llu\n", cnt); 225 } 226 227 static DEVICE_ATTR(pcie_replay_count, 0444, 228 amdgpu_device_get_pcie_replay_count, NULL); 229 230 static ssize_t amdgpu_sysfs_reg_state_get(struct file *f, struct kobject *kobj, 231 struct bin_attribute *attr, char *buf, 232 loff_t ppos, size_t count) 233 { 234 struct device *dev = kobj_to_dev(kobj); 235 struct drm_device *ddev = dev_get_drvdata(dev); 236 struct amdgpu_device *adev = drm_to_adev(ddev); 237 ssize_t bytes_read; 238 239 switch (ppos) { 240 case AMDGPU_SYS_REG_STATE_XGMI: 241 bytes_read = amdgpu_asic_get_reg_state( 242 adev, AMDGPU_REG_STATE_TYPE_XGMI, buf, count); 243 break; 244 case AMDGPU_SYS_REG_STATE_WAFL: 245 bytes_read = amdgpu_asic_get_reg_state( 246 adev, AMDGPU_REG_STATE_TYPE_WAFL, buf, count); 247 break; 248 case AMDGPU_SYS_REG_STATE_PCIE: 249 bytes_read = amdgpu_asic_get_reg_state( 250 adev, AMDGPU_REG_STATE_TYPE_PCIE, buf, count); 251 break; 252 case AMDGPU_SYS_REG_STATE_USR: 253 bytes_read = amdgpu_asic_get_reg_state( 254 adev, AMDGPU_REG_STATE_TYPE_USR, buf, count); 255 break; 256 case AMDGPU_SYS_REG_STATE_USR_1: 257 bytes_read = amdgpu_asic_get_reg_state( 258 adev, AMDGPU_REG_STATE_TYPE_USR_1, buf, count); 259 break; 260 default: 261 return -EINVAL; 262 } 263 264 return bytes_read; 265 } 266 267 BIN_ATTR(reg_state, 0444, amdgpu_sysfs_reg_state_get, NULL, 268 AMDGPU_SYS_REG_STATE_END); 269 270 int amdgpu_reg_state_sysfs_init(struct amdgpu_device *adev) 271 { 272 int ret; 273 274 if (!amdgpu_asic_get_reg_state_supported(adev)) 275 return 0; 276 277 ret = sysfs_create_bin_file(&adev->dev->kobj, &bin_attr_reg_state); 278 279 return ret; 280 } 281 282 void amdgpu_reg_state_sysfs_fini(struct amdgpu_device *adev) 283 { 284 if (!amdgpu_asic_get_reg_state_supported(adev)) 285 return; 286 sysfs_remove_bin_file(&adev->dev->kobj, &bin_attr_reg_state); 287 } 288 289 int amdgpu_ip_block_suspend(struct amdgpu_ip_block *ip_block) 290 { 291 int r; 292 293 if (ip_block->version->funcs->suspend) { 294 r = ip_block->version->funcs->suspend(ip_block); 295 if (r) { 296 dev_err(ip_block->adev->dev, 297 "suspend of IP block <%s> failed %d\n", 298 ip_block->version->funcs->name, r); 299 return r; 300 } 301 } 302 303 ip_block->status.hw = false; 304 return 0; 305 } 306 307 int amdgpu_ip_block_resume(struct amdgpu_ip_block *ip_block) 308 { 309 int r; 310 311 if (ip_block->version->funcs->resume) { 312 r = ip_block->version->funcs->resume(ip_block); 313 if (r) { 314 dev_err(ip_block->adev->dev, 315 "resume of IP block <%s> failed %d\n", 316 ip_block->version->funcs->name, r); 317 return r; 318 } 319 } 320 321 ip_block->status.hw = true; 322 return 0; 323 } 324 325 /** 326 * DOC: board_info 327 * 328 * The amdgpu driver provides a sysfs API for giving board related information. 329 * It provides the form factor information in the format 330 * 331 * type : form factor 332 * 333 * Possible form factor values 334 * 335 * - "cem" - PCIE CEM card 336 * - "oam" - Open Compute Accelerator Module 337 * - "unknown" - Not known 338 * 339 */ 340 341 static ssize_t amdgpu_device_get_board_info(struct device *dev, 342 struct device_attribute *attr, 343 char *buf) 344 { 345 struct drm_device *ddev = dev_get_drvdata(dev); 346 struct amdgpu_device *adev = drm_to_adev(ddev); 347 enum amdgpu_pkg_type pkg_type = AMDGPU_PKG_TYPE_CEM; 348 const char *pkg; 349 350 if (adev->smuio.funcs && adev->smuio.funcs->get_pkg_type) 351 pkg_type = adev->smuio.funcs->get_pkg_type(adev); 352 353 switch (pkg_type) { 354 case AMDGPU_PKG_TYPE_CEM: 355 pkg = "cem"; 356 break; 357 case AMDGPU_PKG_TYPE_OAM: 358 pkg = "oam"; 359 break; 360 default: 361 pkg = "unknown"; 362 break; 363 } 364 365 return sysfs_emit(buf, "%s : %s\n", "type", pkg); 366 } 367 368 static DEVICE_ATTR(board_info, 0444, amdgpu_device_get_board_info, NULL); 369 370 static struct attribute *amdgpu_board_attrs[] = { 371 &dev_attr_board_info.attr, 372 NULL, 373 }; 374 375 static umode_t amdgpu_board_attrs_is_visible(struct kobject *kobj, 376 struct attribute *attr, int n) 377 { 378 struct device *dev = kobj_to_dev(kobj); 379 struct drm_device *ddev = dev_get_drvdata(dev); 380 struct amdgpu_device *adev = drm_to_adev(ddev); 381 382 if (adev->flags & AMD_IS_APU) 383 return 0; 384 385 return attr->mode; 386 } 387 388 static const struct attribute_group amdgpu_board_attrs_group = { 389 .attrs = amdgpu_board_attrs, 390 .is_visible = amdgpu_board_attrs_is_visible 391 }; 392 393 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev); 394 395 396 /** 397 * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control 398 * 399 * @dev: drm_device pointer 400 * 401 * Returns true if the device is a dGPU with ATPX power control, 402 * otherwise return false. 403 */ 404 bool amdgpu_device_supports_px(struct drm_device *dev) 405 { 406 struct amdgpu_device *adev = drm_to_adev(dev); 407 408 if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid()) 409 return true; 410 return false; 411 } 412 413 /** 414 * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources 415 * 416 * @dev: drm_device pointer 417 * 418 * Returns true if the device is a dGPU with ACPI power control, 419 * otherwise return false. 420 */ 421 bool amdgpu_device_supports_boco(struct drm_device *dev) 422 { 423 struct amdgpu_device *adev = drm_to_adev(dev); 424 425 if (!IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE)) 426 return false; 427 428 if (adev->has_pr3 || 429 ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid())) 430 return true; 431 return false; 432 } 433 434 /** 435 * amdgpu_device_supports_baco - Does the device support BACO 436 * 437 * @dev: drm_device pointer 438 * 439 * Return: 440 * 1 if the device supports BACO; 441 * 3 if the device supports MACO (only works if BACO is supported) 442 * otherwise return 0. 443 */ 444 int amdgpu_device_supports_baco(struct drm_device *dev) 445 { 446 struct amdgpu_device *adev = drm_to_adev(dev); 447 448 return amdgpu_asic_supports_baco(adev); 449 } 450 451 void amdgpu_device_detect_runtime_pm_mode(struct amdgpu_device *adev) 452 { 453 struct drm_device *dev; 454 int bamaco_support; 455 456 dev = adev_to_drm(adev); 457 458 adev->pm.rpm_mode = AMDGPU_RUNPM_NONE; 459 bamaco_support = amdgpu_device_supports_baco(dev); 460 461 switch (amdgpu_runtime_pm) { 462 case 2: 463 if (bamaco_support & MACO_SUPPORT) { 464 adev->pm.rpm_mode = AMDGPU_RUNPM_BAMACO; 465 dev_info(adev->dev, "Forcing BAMACO for runtime pm\n"); 466 } else if (bamaco_support == BACO_SUPPORT) { 467 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO; 468 dev_info(adev->dev, "Requested mode BAMACO not available,fallback to use BACO\n"); 469 } 470 break; 471 case 1: 472 if (bamaco_support & BACO_SUPPORT) { 473 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO; 474 dev_info(adev->dev, "Forcing BACO for runtime pm\n"); 475 } 476 break; 477 case -1: 478 case -2: 479 if (amdgpu_device_supports_px(dev)) { /* enable PX as runtime mode */ 480 adev->pm.rpm_mode = AMDGPU_RUNPM_PX; 481 dev_info(adev->dev, "Using ATPX for runtime pm\n"); 482 } else if (amdgpu_device_supports_boco(dev)) { /* enable boco as runtime mode */ 483 adev->pm.rpm_mode = AMDGPU_RUNPM_BOCO; 484 dev_info(adev->dev, "Using BOCO for runtime pm\n"); 485 } else { 486 if (!bamaco_support) 487 goto no_runtime_pm; 488 489 switch (adev->asic_type) { 490 case CHIP_VEGA20: 491 case CHIP_ARCTURUS: 492 /* BACO are not supported on vega20 and arctrus */ 493 break; 494 case CHIP_VEGA10: 495 /* enable BACO as runpm mode if noretry=0 */ 496 if (!adev->gmc.noretry) 497 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO; 498 break; 499 default: 500 /* enable BACO as runpm mode on CI+ */ 501 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO; 502 break; 503 } 504 505 if (adev->pm.rpm_mode == AMDGPU_RUNPM_BACO) { 506 if (bamaco_support & MACO_SUPPORT) { 507 adev->pm.rpm_mode = AMDGPU_RUNPM_BAMACO; 508 dev_info(adev->dev, "Using BAMACO for runtime pm\n"); 509 } else { 510 dev_info(adev->dev, "Using BACO for runtime pm\n"); 511 } 512 } 513 } 514 break; 515 case 0: 516 dev_info(adev->dev, "runtime pm is manually disabled\n"); 517 break; 518 default: 519 break; 520 } 521 522 no_runtime_pm: 523 if (adev->pm.rpm_mode == AMDGPU_RUNPM_NONE) 524 dev_info(adev->dev, "Runtime PM not available\n"); 525 } 526 /** 527 * amdgpu_device_supports_smart_shift - Is the device dGPU with 528 * smart shift support 529 * 530 * @dev: drm_device pointer 531 * 532 * Returns true if the device is a dGPU with Smart Shift support, 533 * otherwise returns false. 534 */ 535 bool amdgpu_device_supports_smart_shift(struct drm_device *dev) 536 { 537 return (amdgpu_device_supports_boco(dev) && 538 amdgpu_acpi_is_power_shift_control_supported()); 539 } 540 541 /* 542 * VRAM access helper functions 543 */ 544 545 /** 546 * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA 547 * 548 * @adev: amdgpu_device pointer 549 * @pos: offset of the buffer in vram 550 * @buf: virtual address of the buffer in system memory 551 * @size: read/write size, sizeof(@buf) must > @size 552 * @write: true - write to vram, otherwise - read from vram 553 */ 554 void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos, 555 void *buf, size_t size, bool write) 556 { 557 unsigned long flags; 558 uint32_t hi = ~0, tmp = 0; 559 uint32_t *data = buf; 560 uint64_t last; 561 int idx; 562 563 if (!drm_dev_enter(adev_to_drm(adev), &idx)) 564 return; 565 566 BUG_ON(!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4)); 567 568 spin_lock_irqsave(&adev->mmio_idx_lock, flags); 569 for (last = pos + size; pos < last; pos += 4) { 570 tmp = pos >> 31; 571 572 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000); 573 if (tmp != hi) { 574 WREG32_NO_KIQ(mmMM_INDEX_HI, tmp); 575 hi = tmp; 576 } 577 if (write) 578 WREG32_NO_KIQ(mmMM_DATA, *data++); 579 else 580 *data++ = RREG32_NO_KIQ(mmMM_DATA); 581 } 582 583 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); 584 drm_dev_exit(idx); 585 } 586 587 /** 588 * amdgpu_device_aper_access - access vram by vram aperture 589 * 590 * @adev: amdgpu_device pointer 591 * @pos: offset of the buffer in vram 592 * @buf: virtual address of the buffer in system memory 593 * @size: read/write size, sizeof(@buf) must > @size 594 * @write: true - write to vram, otherwise - read from vram 595 * 596 * The return value means how many bytes have been transferred. 597 */ 598 size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos, 599 void *buf, size_t size, bool write) 600 { 601 #ifdef CONFIG_64BIT 602 void __iomem *addr; 603 size_t count = 0; 604 uint64_t last; 605 606 if (!adev->mman.aper_base_kaddr) 607 return 0; 608 609 last = min(pos + size, adev->gmc.visible_vram_size); 610 if (last > pos) { 611 addr = adev->mman.aper_base_kaddr + pos; 612 count = last - pos; 613 614 if (write) { 615 memcpy_toio(addr, buf, count); 616 /* Make sure HDP write cache flush happens without any reordering 617 * after the system memory contents are sent over PCIe device 618 */ 619 mb(); 620 amdgpu_device_flush_hdp(adev, NULL); 621 } else { 622 amdgpu_device_invalidate_hdp(adev, NULL); 623 /* Make sure HDP read cache is invalidated before issuing a read 624 * to the PCIe device 625 */ 626 mb(); 627 memcpy_fromio(buf, addr, count); 628 } 629 630 } 631 632 return count; 633 #else 634 return 0; 635 #endif 636 } 637 638 /** 639 * amdgpu_device_vram_access - read/write a buffer in vram 640 * 641 * @adev: amdgpu_device pointer 642 * @pos: offset of the buffer in vram 643 * @buf: virtual address of the buffer in system memory 644 * @size: read/write size, sizeof(@buf) must > @size 645 * @write: true - write to vram, otherwise - read from vram 646 */ 647 void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos, 648 void *buf, size_t size, bool write) 649 { 650 size_t count; 651 652 /* try to using vram apreature to access vram first */ 653 count = amdgpu_device_aper_access(adev, pos, buf, size, write); 654 size -= count; 655 if (size) { 656 /* using MM to access rest vram */ 657 pos += count; 658 buf += count; 659 amdgpu_device_mm_access(adev, pos, buf, size, write); 660 } 661 } 662 663 /* 664 * register access helper functions. 665 */ 666 667 /* Check if hw access should be skipped because of hotplug or device error */ 668 bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev) 669 { 670 if (adev->no_hw_access) 671 return true; 672 673 #ifdef CONFIG_LOCKDEP 674 /* 675 * This is a bit complicated to understand, so worth a comment. What we assert 676 * here is that the GPU reset is not running on another thread in parallel. 677 * 678 * For this we trylock the read side of the reset semaphore, if that succeeds 679 * we know that the reset is not running in parallel. 680 * 681 * If the trylock fails we assert that we are either already holding the read 682 * side of the lock or are the reset thread itself and hold the write side of 683 * the lock. 684 */ 685 if (in_task()) { 686 if (down_read_trylock(&adev->reset_domain->sem)) 687 up_read(&adev->reset_domain->sem); 688 else 689 lockdep_assert_held(&adev->reset_domain->sem); 690 } 691 #endif 692 return false; 693 } 694 695 /** 696 * amdgpu_device_rreg - read a memory mapped IO or indirect register 697 * 698 * @adev: amdgpu_device pointer 699 * @reg: dword aligned register offset 700 * @acc_flags: access flags which require special behavior 701 * 702 * Returns the 32 bit value from the offset specified. 703 */ 704 uint32_t amdgpu_device_rreg(struct amdgpu_device *adev, 705 uint32_t reg, uint32_t acc_flags) 706 { 707 uint32_t ret; 708 709 if (amdgpu_device_skip_hw_access(adev)) 710 return 0; 711 712 if ((reg * 4) < adev->rmmio_size) { 713 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && 714 amdgpu_sriov_runtime(adev) && 715 down_read_trylock(&adev->reset_domain->sem)) { 716 ret = amdgpu_kiq_rreg(adev, reg, 0); 717 up_read(&adev->reset_domain->sem); 718 } else { 719 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); 720 } 721 } else { 722 ret = adev->pcie_rreg(adev, reg * 4); 723 } 724 725 trace_amdgpu_device_rreg(adev->pdev->device, reg, ret); 726 727 return ret; 728 } 729 730 /* 731 * MMIO register read with bytes helper functions 732 * @offset:bytes offset from MMIO start 733 */ 734 735 /** 736 * amdgpu_mm_rreg8 - read a memory mapped IO register 737 * 738 * @adev: amdgpu_device pointer 739 * @offset: byte aligned register offset 740 * 741 * Returns the 8 bit value from the offset specified. 742 */ 743 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset) 744 { 745 if (amdgpu_device_skip_hw_access(adev)) 746 return 0; 747 748 if (offset < adev->rmmio_size) 749 return (readb(adev->rmmio + offset)); 750 BUG(); 751 } 752 753 754 /** 755 * amdgpu_device_xcc_rreg - read a memory mapped IO or indirect register with specific XCC 756 * 757 * @adev: amdgpu_device pointer 758 * @reg: dword aligned register offset 759 * @acc_flags: access flags which require special behavior 760 * @xcc_id: xcc accelerated compute core id 761 * 762 * Returns the 32 bit value from the offset specified. 763 */ 764 uint32_t amdgpu_device_xcc_rreg(struct amdgpu_device *adev, 765 uint32_t reg, uint32_t acc_flags, 766 uint32_t xcc_id) 767 { 768 uint32_t ret, rlcg_flag; 769 770 if (amdgpu_device_skip_hw_access(adev)) 771 return 0; 772 773 if ((reg * 4) < adev->rmmio_size) { 774 if (amdgpu_sriov_vf(adev) && 775 !amdgpu_sriov_runtime(adev) && 776 adev->gfx.rlc.rlcg_reg_access_supported && 777 amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags, 778 GC_HWIP, false, 779 &rlcg_flag)) { 780 ret = amdgpu_virt_rlcg_reg_rw(adev, reg, 0, rlcg_flag, GET_INST(GC, xcc_id)); 781 } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && 782 amdgpu_sriov_runtime(adev) && 783 down_read_trylock(&adev->reset_domain->sem)) { 784 ret = amdgpu_kiq_rreg(adev, reg, xcc_id); 785 up_read(&adev->reset_domain->sem); 786 } else { 787 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); 788 } 789 } else { 790 ret = adev->pcie_rreg(adev, reg * 4); 791 } 792 793 return ret; 794 } 795 796 /* 797 * MMIO register write with bytes helper functions 798 * @offset:bytes offset from MMIO start 799 * @value: the value want to be written to the register 800 */ 801 802 /** 803 * amdgpu_mm_wreg8 - read a memory mapped IO register 804 * 805 * @adev: amdgpu_device pointer 806 * @offset: byte aligned register offset 807 * @value: 8 bit value to write 808 * 809 * Writes the value specified to the offset specified. 810 */ 811 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value) 812 { 813 if (amdgpu_device_skip_hw_access(adev)) 814 return; 815 816 if (offset < adev->rmmio_size) 817 writeb(value, adev->rmmio + offset); 818 else 819 BUG(); 820 } 821 822 /** 823 * amdgpu_device_wreg - write to a memory mapped IO or indirect register 824 * 825 * @adev: amdgpu_device pointer 826 * @reg: dword aligned register offset 827 * @v: 32 bit value to write to the register 828 * @acc_flags: access flags which require special behavior 829 * 830 * Writes the value specified to the offset specified. 831 */ 832 void amdgpu_device_wreg(struct amdgpu_device *adev, 833 uint32_t reg, uint32_t v, 834 uint32_t acc_flags) 835 { 836 if (amdgpu_device_skip_hw_access(adev)) 837 return; 838 839 if ((reg * 4) < adev->rmmio_size) { 840 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && 841 amdgpu_sriov_runtime(adev) && 842 down_read_trylock(&adev->reset_domain->sem)) { 843 amdgpu_kiq_wreg(adev, reg, v, 0); 844 up_read(&adev->reset_domain->sem); 845 } else { 846 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); 847 } 848 } else { 849 adev->pcie_wreg(adev, reg * 4, v); 850 } 851 852 trace_amdgpu_device_wreg(adev->pdev->device, reg, v); 853 } 854 855 /** 856 * amdgpu_mm_wreg_mmio_rlc - write register either with direct/indirect mmio or with RLC path if in range 857 * 858 * @adev: amdgpu_device pointer 859 * @reg: mmio/rlc register 860 * @v: value to write 861 * @xcc_id: xcc accelerated compute core id 862 * 863 * this function is invoked only for the debugfs register access 864 */ 865 void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev, 866 uint32_t reg, uint32_t v, 867 uint32_t xcc_id) 868 { 869 if (amdgpu_device_skip_hw_access(adev)) 870 return; 871 872 if (amdgpu_sriov_fullaccess(adev) && 873 adev->gfx.rlc.funcs && 874 adev->gfx.rlc.funcs->is_rlcg_access_range) { 875 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg)) 876 return amdgpu_sriov_wreg(adev, reg, v, 0, 0, xcc_id); 877 } else if ((reg * 4) >= adev->rmmio_size) { 878 adev->pcie_wreg(adev, reg * 4, v); 879 } else { 880 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); 881 } 882 } 883 884 /** 885 * amdgpu_device_xcc_wreg - write to a memory mapped IO or indirect register with specific XCC 886 * 887 * @adev: amdgpu_device pointer 888 * @reg: dword aligned register offset 889 * @v: 32 bit value to write to the register 890 * @acc_flags: access flags which require special behavior 891 * @xcc_id: xcc accelerated compute core id 892 * 893 * Writes the value specified to the offset specified. 894 */ 895 void amdgpu_device_xcc_wreg(struct amdgpu_device *adev, 896 uint32_t reg, uint32_t v, 897 uint32_t acc_flags, uint32_t xcc_id) 898 { 899 uint32_t rlcg_flag; 900 901 if (amdgpu_device_skip_hw_access(adev)) 902 return; 903 904 if ((reg * 4) < adev->rmmio_size) { 905 if (amdgpu_sriov_vf(adev) && 906 !amdgpu_sriov_runtime(adev) && 907 adev->gfx.rlc.rlcg_reg_access_supported && 908 amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags, 909 GC_HWIP, true, 910 &rlcg_flag)) { 911 amdgpu_virt_rlcg_reg_rw(adev, reg, v, rlcg_flag, GET_INST(GC, xcc_id)); 912 } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && 913 amdgpu_sriov_runtime(adev) && 914 down_read_trylock(&adev->reset_domain->sem)) { 915 amdgpu_kiq_wreg(adev, reg, v, xcc_id); 916 up_read(&adev->reset_domain->sem); 917 } else { 918 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); 919 } 920 } else { 921 adev->pcie_wreg(adev, reg * 4, v); 922 } 923 } 924 925 /** 926 * amdgpu_device_indirect_rreg - read an indirect register 927 * 928 * @adev: amdgpu_device pointer 929 * @reg_addr: indirect register address to read from 930 * 931 * Returns the value of indirect register @reg_addr 932 */ 933 u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev, 934 u32 reg_addr) 935 { 936 unsigned long flags, pcie_index, pcie_data; 937 void __iomem *pcie_index_offset; 938 void __iomem *pcie_data_offset; 939 u32 r; 940 941 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 942 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 943 944 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 945 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 946 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 947 948 writel(reg_addr, pcie_index_offset); 949 readl(pcie_index_offset); 950 r = readl(pcie_data_offset); 951 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 952 953 return r; 954 } 955 956 u32 amdgpu_device_indirect_rreg_ext(struct amdgpu_device *adev, 957 u64 reg_addr) 958 { 959 unsigned long flags, pcie_index, pcie_index_hi, pcie_data; 960 u32 r; 961 void __iomem *pcie_index_offset; 962 void __iomem *pcie_index_hi_offset; 963 void __iomem *pcie_data_offset; 964 965 if (unlikely(!adev->nbio.funcs)) { 966 pcie_index = AMDGPU_PCIE_INDEX_FALLBACK; 967 pcie_data = AMDGPU_PCIE_DATA_FALLBACK; 968 } else { 969 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 970 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 971 } 972 973 if (reg_addr >> 32) { 974 if (unlikely(!adev->nbio.funcs)) 975 pcie_index_hi = AMDGPU_PCIE_INDEX_HI_FALLBACK; 976 else 977 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev); 978 } else { 979 pcie_index_hi = 0; 980 } 981 982 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 983 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 984 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 985 if (pcie_index_hi != 0) 986 pcie_index_hi_offset = (void __iomem *)adev->rmmio + 987 pcie_index_hi * 4; 988 989 writel(reg_addr, pcie_index_offset); 990 readl(pcie_index_offset); 991 if (pcie_index_hi != 0) { 992 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); 993 readl(pcie_index_hi_offset); 994 } 995 r = readl(pcie_data_offset); 996 997 /* clear the high bits */ 998 if (pcie_index_hi != 0) { 999 writel(0, pcie_index_hi_offset); 1000 readl(pcie_index_hi_offset); 1001 } 1002 1003 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 1004 1005 return r; 1006 } 1007 1008 /** 1009 * amdgpu_device_indirect_rreg64 - read a 64bits indirect register 1010 * 1011 * @adev: amdgpu_device pointer 1012 * @reg_addr: indirect register address to read from 1013 * 1014 * Returns the value of indirect register @reg_addr 1015 */ 1016 u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev, 1017 u32 reg_addr) 1018 { 1019 unsigned long flags, pcie_index, pcie_data; 1020 void __iomem *pcie_index_offset; 1021 void __iomem *pcie_data_offset; 1022 u64 r; 1023 1024 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 1025 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 1026 1027 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 1028 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 1029 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 1030 1031 /* read low 32 bits */ 1032 writel(reg_addr, pcie_index_offset); 1033 readl(pcie_index_offset); 1034 r = readl(pcie_data_offset); 1035 /* read high 32 bits */ 1036 writel(reg_addr + 4, pcie_index_offset); 1037 readl(pcie_index_offset); 1038 r |= ((u64)readl(pcie_data_offset) << 32); 1039 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 1040 1041 return r; 1042 } 1043 1044 u64 amdgpu_device_indirect_rreg64_ext(struct amdgpu_device *adev, 1045 u64 reg_addr) 1046 { 1047 unsigned long flags, pcie_index, pcie_data; 1048 unsigned long pcie_index_hi = 0; 1049 void __iomem *pcie_index_offset; 1050 void __iomem *pcie_index_hi_offset; 1051 void __iomem *pcie_data_offset; 1052 u64 r; 1053 1054 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 1055 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 1056 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset)) 1057 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev); 1058 1059 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 1060 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 1061 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 1062 if (pcie_index_hi != 0) 1063 pcie_index_hi_offset = (void __iomem *)adev->rmmio + 1064 pcie_index_hi * 4; 1065 1066 /* read low 32 bits */ 1067 writel(reg_addr, pcie_index_offset); 1068 readl(pcie_index_offset); 1069 if (pcie_index_hi != 0) { 1070 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); 1071 readl(pcie_index_hi_offset); 1072 } 1073 r = readl(pcie_data_offset); 1074 /* read high 32 bits */ 1075 writel(reg_addr + 4, pcie_index_offset); 1076 readl(pcie_index_offset); 1077 if (pcie_index_hi != 0) { 1078 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); 1079 readl(pcie_index_hi_offset); 1080 } 1081 r |= ((u64)readl(pcie_data_offset) << 32); 1082 1083 /* clear the high bits */ 1084 if (pcie_index_hi != 0) { 1085 writel(0, pcie_index_hi_offset); 1086 readl(pcie_index_hi_offset); 1087 } 1088 1089 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 1090 1091 return r; 1092 } 1093 1094 /** 1095 * amdgpu_device_indirect_wreg - write an indirect register address 1096 * 1097 * @adev: amdgpu_device pointer 1098 * @reg_addr: indirect register offset 1099 * @reg_data: indirect register data 1100 * 1101 */ 1102 void amdgpu_device_indirect_wreg(struct amdgpu_device *adev, 1103 u32 reg_addr, u32 reg_data) 1104 { 1105 unsigned long flags, pcie_index, pcie_data; 1106 void __iomem *pcie_index_offset; 1107 void __iomem *pcie_data_offset; 1108 1109 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 1110 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 1111 1112 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 1113 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 1114 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 1115 1116 writel(reg_addr, pcie_index_offset); 1117 readl(pcie_index_offset); 1118 writel(reg_data, pcie_data_offset); 1119 readl(pcie_data_offset); 1120 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 1121 } 1122 1123 void amdgpu_device_indirect_wreg_ext(struct amdgpu_device *adev, 1124 u64 reg_addr, u32 reg_data) 1125 { 1126 unsigned long flags, pcie_index, pcie_index_hi, pcie_data; 1127 void __iomem *pcie_index_offset; 1128 void __iomem *pcie_index_hi_offset; 1129 void __iomem *pcie_data_offset; 1130 1131 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 1132 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 1133 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset)) 1134 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev); 1135 else 1136 pcie_index_hi = 0; 1137 1138 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 1139 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 1140 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 1141 if (pcie_index_hi != 0) 1142 pcie_index_hi_offset = (void __iomem *)adev->rmmio + 1143 pcie_index_hi * 4; 1144 1145 writel(reg_addr, pcie_index_offset); 1146 readl(pcie_index_offset); 1147 if (pcie_index_hi != 0) { 1148 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); 1149 readl(pcie_index_hi_offset); 1150 } 1151 writel(reg_data, pcie_data_offset); 1152 readl(pcie_data_offset); 1153 1154 /* clear the high bits */ 1155 if (pcie_index_hi != 0) { 1156 writel(0, pcie_index_hi_offset); 1157 readl(pcie_index_hi_offset); 1158 } 1159 1160 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 1161 } 1162 1163 /** 1164 * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address 1165 * 1166 * @adev: amdgpu_device pointer 1167 * @reg_addr: indirect register offset 1168 * @reg_data: indirect register data 1169 * 1170 */ 1171 void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev, 1172 u32 reg_addr, u64 reg_data) 1173 { 1174 unsigned long flags, pcie_index, pcie_data; 1175 void __iomem *pcie_index_offset; 1176 void __iomem *pcie_data_offset; 1177 1178 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 1179 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 1180 1181 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 1182 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 1183 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 1184 1185 /* write low 32 bits */ 1186 writel(reg_addr, pcie_index_offset); 1187 readl(pcie_index_offset); 1188 writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset); 1189 readl(pcie_data_offset); 1190 /* write high 32 bits */ 1191 writel(reg_addr + 4, pcie_index_offset); 1192 readl(pcie_index_offset); 1193 writel((u32)(reg_data >> 32), pcie_data_offset); 1194 readl(pcie_data_offset); 1195 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 1196 } 1197 1198 void amdgpu_device_indirect_wreg64_ext(struct amdgpu_device *adev, 1199 u64 reg_addr, u64 reg_data) 1200 { 1201 unsigned long flags, pcie_index, pcie_data; 1202 unsigned long pcie_index_hi = 0; 1203 void __iomem *pcie_index_offset; 1204 void __iomem *pcie_index_hi_offset; 1205 void __iomem *pcie_data_offset; 1206 1207 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 1208 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 1209 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset)) 1210 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev); 1211 1212 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 1213 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 1214 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 1215 if (pcie_index_hi != 0) 1216 pcie_index_hi_offset = (void __iomem *)adev->rmmio + 1217 pcie_index_hi * 4; 1218 1219 /* write low 32 bits */ 1220 writel(reg_addr, pcie_index_offset); 1221 readl(pcie_index_offset); 1222 if (pcie_index_hi != 0) { 1223 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); 1224 readl(pcie_index_hi_offset); 1225 } 1226 writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset); 1227 readl(pcie_data_offset); 1228 /* write high 32 bits */ 1229 writel(reg_addr + 4, pcie_index_offset); 1230 readl(pcie_index_offset); 1231 if (pcie_index_hi != 0) { 1232 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); 1233 readl(pcie_index_hi_offset); 1234 } 1235 writel((u32)(reg_data >> 32), pcie_data_offset); 1236 readl(pcie_data_offset); 1237 1238 /* clear the high bits */ 1239 if (pcie_index_hi != 0) { 1240 writel(0, pcie_index_hi_offset); 1241 readl(pcie_index_hi_offset); 1242 } 1243 1244 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 1245 } 1246 1247 /** 1248 * amdgpu_device_get_rev_id - query device rev_id 1249 * 1250 * @adev: amdgpu_device pointer 1251 * 1252 * Return device rev_id 1253 */ 1254 u32 amdgpu_device_get_rev_id(struct amdgpu_device *adev) 1255 { 1256 return adev->nbio.funcs->get_rev_id(adev); 1257 } 1258 1259 /** 1260 * amdgpu_invalid_rreg - dummy reg read function 1261 * 1262 * @adev: amdgpu_device pointer 1263 * @reg: offset of register 1264 * 1265 * Dummy register read function. Used for register blocks 1266 * that certain asics don't have (all asics). 1267 * Returns the value in the register. 1268 */ 1269 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg) 1270 { 1271 DRM_ERROR("Invalid callback to read register 0x%04X\n", reg); 1272 BUG(); 1273 return 0; 1274 } 1275 1276 static uint32_t amdgpu_invalid_rreg_ext(struct amdgpu_device *adev, uint64_t reg) 1277 { 1278 DRM_ERROR("Invalid callback to read register 0x%llX\n", reg); 1279 BUG(); 1280 return 0; 1281 } 1282 1283 /** 1284 * amdgpu_invalid_wreg - dummy reg write function 1285 * 1286 * @adev: amdgpu_device pointer 1287 * @reg: offset of register 1288 * @v: value to write to the register 1289 * 1290 * Dummy register read function. Used for register blocks 1291 * that certain asics don't have (all asics). 1292 */ 1293 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) 1294 { 1295 DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n", 1296 reg, v); 1297 BUG(); 1298 } 1299 1300 static void amdgpu_invalid_wreg_ext(struct amdgpu_device *adev, uint64_t reg, uint32_t v) 1301 { 1302 DRM_ERROR("Invalid callback to write register 0x%llX with 0x%08X\n", 1303 reg, v); 1304 BUG(); 1305 } 1306 1307 /** 1308 * amdgpu_invalid_rreg64 - dummy 64 bit reg read function 1309 * 1310 * @adev: amdgpu_device pointer 1311 * @reg: offset of register 1312 * 1313 * Dummy register read function. Used for register blocks 1314 * that certain asics don't have (all asics). 1315 * Returns the value in the register. 1316 */ 1317 static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg) 1318 { 1319 DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg); 1320 BUG(); 1321 return 0; 1322 } 1323 1324 static uint64_t amdgpu_invalid_rreg64_ext(struct amdgpu_device *adev, uint64_t reg) 1325 { 1326 DRM_ERROR("Invalid callback to read register 0x%llX\n", reg); 1327 BUG(); 1328 return 0; 1329 } 1330 1331 /** 1332 * amdgpu_invalid_wreg64 - dummy reg write function 1333 * 1334 * @adev: amdgpu_device pointer 1335 * @reg: offset of register 1336 * @v: value to write to the register 1337 * 1338 * Dummy register read function. Used for register blocks 1339 * that certain asics don't have (all asics). 1340 */ 1341 static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v) 1342 { 1343 DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n", 1344 reg, v); 1345 BUG(); 1346 } 1347 1348 static void amdgpu_invalid_wreg64_ext(struct amdgpu_device *adev, uint64_t reg, uint64_t v) 1349 { 1350 DRM_ERROR("Invalid callback to write 64 bit register 0x%llX with 0x%08llX\n", 1351 reg, v); 1352 BUG(); 1353 } 1354 1355 /** 1356 * amdgpu_block_invalid_rreg - dummy reg read function 1357 * 1358 * @adev: amdgpu_device pointer 1359 * @block: offset of instance 1360 * @reg: offset of register 1361 * 1362 * Dummy register read function. Used for register blocks 1363 * that certain asics don't have (all asics). 1364 * Returns the value in the register. 1365 */ 1366 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev, 1367 uint32_t block, uint32_t reg) 1368 { 1369 DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n", 1370 reg, block); 1371 BUG(); 1372 return 0; 1373 } 1374 1375 /** 1376 * amdgpu_block_invalid_wreg - dummy reg write function 1377 * 1378 * @adev: amdgpu_device pointer 1379 * @block: offset of instance 1380 * @reg: offset of register 1381 * @v: value to write to the register 1382 * 1383 * Dummy register read function. Used for register blocks 1384 * that certain asics don't have (all asics). 1385 */ 1386 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev, 1387 uint32_t block, 1388 uint32_t reg, uint32_t v) 1389 { 1390 DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n", 1391 reg, block, v); 1392 BUG(); 1393 } 1394 1395 static uint32_t amdgpu_device_get_vbios_flags(struct amdgpu_device *adev) 1396 { 1397 if (hweight32(adev->aid_mask) && (adev->flags & AMD_IS_APU)) 1398 return AMDGPU_VBIOS_SKIP; 1399 1400 if (hweight32(adev->aid_mask) && amdgpu_passthrough(adev)) 1401 return AMDGPU_VBIOS_OPTIONAL; 1402 1403 return 0; 1404 } 1405 1406 /** 1407 * amdgpu_device_asic_init - Wrapper for atom asic_init 1408 * 1409 * @adev: amdgpu_device pointer 1410 * 1411 * Does any asic specific work and then calls atom asic init. 1412 */ 1413 static int amdgpu_device_asic_init(struct amdgpu_device *adev) 1414 { 1415 uint32_t flags; 1416 bool optional; 1417 int ret; 1418 1419 amdgpu_asic_pre_asic_init(adev); 1420 flags = amdgpu_device_get_vbios_flags(adev); 1421 optional = !!(flags & (AMDGPU_VBIOS_OPTIONAL | AMDGPU_VBIOS_SKIP)); 1422 1423 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) || 1424 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4) || 1425 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0) || 1426 amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0)) { 1427 amdgpu_psp_wait_for_bootloader(adev); 1428 if (optional && !adev->bios) 1429 return 0; 1430 1431 ret = amdgpu_atomfirmware_asic_init(adev, true); 1432 return ret; 1433 } else { 1434 if (optional && !adev->bios) 1435 return 0; 1436 1437 return amdgpu_atom_asic_init(adev->mode_info.atom_context); 1438 } 1439 1440 return 0; 1441 } 1442 1443 /** 1444 * amdgpu_device_mem_scratch_init - allocate the VRAM scratch page 1445 * 1446 * @adev: amdgpu_device pointer 1447 * 1448 * Allocates a scratch page of VRAM for use by various things in the 1449 * driver. 1450 */ 1451 static int amdgpu_device_mem_scratch_init(struct amdgpu_device *adev) 1452 { 1453 return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, PAGE_SIZE, 1454 AMDGPU_GEM_DOMAIN_VRAM | 1455 AMDGPU_GEM_DOMAIN_GTT, 1456 &adev->mem_scratch.robj, 1457 &adev->mem_scratch.gpu_addr, 1458 (void **)&adev->mem_scratch.ptr); 1459 } 1460 1461 /** 1462 * amdgpu_device_mem_scratch_fini - Free the VRAM scratch page 1463 * 1464 * @adev: amdgpu_device pointer 1465 * 1466 * Frees the VRAM scratch page. 1467 */ 1468 static void amdgpu_device_mem_scratch_fini(struct amdgpu_device *adev) 1469 { 1470 amdgpu_bo_free_kernel(&adev->mem_scratch.robj, NULL, NULL); 1471 } 1472 1473 /** 1474 * amdgpu_device_program_register_sequence - program an array of registers. 1475 * 1476 * @adev: amdgpu_device pointer 1477 * @registers: pointer to the register array 1478 * @array_size: size of the register array 1479 * 1480 * Programs an array or registers with and or masks. 1481 * This is a helper for setting golden registers. 1482 */ 1483 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev, 1484 const u32 *registers, 1485 const u32 array_size) 1486 { 1487 u32 tmp, reg, and_mask, or_mask; 1488 int i; 1489 1490 if (array_size % 3) 1491 return; 1492 1493 for (i = 0; i < array_size; i += 3) { 1494 reg = registers[i + 0]; 1495 and_mask = registers[i + 1]; 1496 or_mask = registers[i + 2]; 1497 1498 if (and_mask == 0xffffffff) { 1499 tmp = or_mask; 1500 } else { 1501 tmp = RREG32(reg); 1502 tmp &= ~and_mask; 1503 if (adev->family >= AMDGPU_FAMILY_AI) 1504 tmp |= (or_mask & and_mask); 1505 else 1506 tmp |= or_mask; 1507 } 1508 WREG32(reg, tmp); 1509 } 1510 } 1511 1512 /** 1513 * amdgpu_device_pci_config_reset - reset the GPU 1514 * 1515 * @adev: amdgpu_device pointer 1516 * 1517 * Resets the GPU using the pci config reset sequence. 1518 * Only applicable to asics prior to vega10. 1519 */ 1520 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev) 1521 { 1522 pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA); 1523 } 1524 1525 /** 1526 * amdgpu_device_pci_reset - reset the GPU using generic PCI means 1527 * 1528 * @adev: amdgpu_device pointer 1529 * 1530 * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.). 1531 */ 1532 int amdgpu_device_pci_reset(struct amdgpu_device *adev) 1533 { 1534 return pci_reset_function(adev->pdev); 1535 } 1536 1537 /* 1538 * amdgpu_device_wb_*() 1539 * Writeback is the method by which the GPU updates special pages in memory 1540 * with the status of certain GPU events (fences, ring pointers,etc.). 1541 */ 1542 1543 /** 1544 * amdgpu_device_wb_fini - Disable Writeback and free memory 1545 * 1546 * @adev: amdgpu_device pointer 1547 * 1548 * Disables Writeback and frees the Writeback memory (all asics). 1549 * Used at driver shutdown. 1550 */ 1551 static void amdgpu_device_wb_fini(struct amdgpu_device *adev) 1552 { 1553 if (adev->wb.wb_obj) { 1554 amdgpu_bo_free_kernel(&adev->wb.wb_obj, 1555 &adev->wb.gpu_addr, 1556 (void **)&adev->wb.wb); 1557 adev->wb.wb_obj = NULL; 1558 } 1559 } 1560 1561 /** 1562 * amdgpu_device_wb_init - Init Writeback driver info and allocate memory 1563 * 1564 * @adev: amdgpu_device pointer 1565 * 1566 * Initializes writeback and allocates writeback memory (all asics). 1567 * Used at driver startup. 1568 * Returns 0 on success or an -error on failure. 1569 */ 1570 static int amdgpu_device_wb_init(struct amdgpu_device *adev) 1571 { 1572 int r; 1573 1574 if (adev->wb.wb_obj == NULL) { 1575 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */ 1576 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8, 1577 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 1578 &adev->wb.wb_obj, &adev->wb.gpu_addr, 1579 (void **)&adev->wb.wb); 1580 if (r) { 1581 dev_warn(adev->dev, "(%d) create WB bo failed\n", r); 1582 return r; 1583 } 1584 1585 adev->wb.num_wb = AMDGPU_MAX_WB; 1586 memset(&adev->wb.used, 0, sizeof(adev->wb.used)); 1587 1588 /* clear wb memory */ 1589 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8); 1590 } 1591 1592 return 0; 1593 } 1594 1595 /** 1596 * amdgpu_device_wb_get - Allocate a wb entry 1597 * 1598 * @adev: amdgpu_device pointer 1599 * @wb: wb index 1600 * 1601 * Allocate a wb slot for use by the driver (all asics). 1602 * Returns 0 on success or -EINVAL on failure. 1603 */ 1604 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb) 1605 { 1606 unsigned long flags, offset; 1607 1608 spin_lock_irqsave(&adev->wb.lock, flags); 1609 offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb); 1610 if (offset < adev->wb.num_wb) { 1611 __set_bit(offset, adev->wb.used); 1612 spin_unlock_irqrestore(&adev->wb.lock, flags); 1613 *wb = offset << 3; /* convert to dw offset */ 1614 return 0; 1615 } else { 1616 spin_unlock_irqrestore(&adev->wb.lock, flags); 1617 return -EINVAL; 1618 } 1619 } 1620 1621 /** 1622 * amdgpu_device_wb_free - Free a wb entry 1623 * 1624 * @adev: amdgpu_device pointer 1625 * @wb: wb index 1626 * 1627 * Free a wb slot allocated for use by the driver (all asics) 1628 */ 1629 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb) 1630 { 1631 unsigned long flags; 1632 1633 wb >>= 3; 1634 spin_lock_irqsave(&adev->wb.lock, flags); 1635 if (wb < adev->wb.num_wb) 1636 __clear_bit(wb, adev->wb.used); 1637 spin_unlock_irqrestore(&adev->wb.lock, flags); 1638 } 1639 1640 /** 1641 * amdgpu_device_resize_fb_bar - try to resize FB BAR 1642 * 1643 * @adev: amdgpu_device pointer 1644 * 1645 * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not 1646 * to fail, but if any of the BARs is not accessible after the size we abort 1647 * driver loading by returning -ENODEV. 1648 */ 1649 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev) 1650 { 1651 int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size); 1652 struct pci_bus *root; 1653 struct resource *res; 1654 unsigned int i; 1655 u16 cmd; 1656 int r; 1657 1658 if (!IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT)) 1659 return 0; 1660 1661 /* Bypass for VF */ 1662 if (amdgpu_sriov_vf(adev)) 1663 return 0; 1664 1665 /* PCI_EXT_CAP_ID_VNDR extended capability is located at 0x100 */ 1666 if (!pci_find_ext_capability(adev->pdev, PCI_EXT_CAP_ID_VNDR)) 1667 DRM_WARN("System can't access extended configuration space, please check!!\n"); 1668 1669 /* skip if the bios has already enabled large BAR */ 1670 if (adev->gmc.real_vram_size && 1671 (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size)) 1672 return 0; 1673 1674 /* Check if the root BUS has 64bit memory resources */ 1675 root = adev->pdev->bus; 1676 while (root->parent) 1677 root = root->parent; 1678 1679 pci_bus_for_each_resource(root, res, i) { 1680 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) && 1681 res->start > 0x100000000ull) 1682 break; 1683 } 1684 1685 /* Trying to resize is pointless without a root hub window above 4GB */ 1686 if (!res) 1687 return 0; 1688 1689 /* Limit the BAR size to what is available */ 1690 rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1, 1691 rbar_size); 1692 1693 /* Disable memory decoding while we change the BAR addresses and size */ 1694 pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd); 1695 pci_write_config_word(adev->pdev, PCI_COMMAND, 1696 cmd & ~PCI_COMMAND_MEMORY); 1697 1698 /* Free the VRAM and doorbell BAR, we most likely need to move both. */ 1699 amdgpu_doorbell_fini(adev); 1700 if (adev->asic_type >= CHIP_BONAIRE) 1701 pci_release_resource(adev->pdev, 2); 1702 1703 pci_release_resource(adev->pdev, 0); 1704 1705 r = pci_resize_resource(adev->pdev, 0, rbar_size); 1706 if (r == -ENOSPC) 1707 DRM_INFO("Not enough PCI address space for a large BAR."); 1708 else if (r && r != -ENOTSUPP) 1709 DRM_ERROR("Problem resizing BAR0 (%d).", r); 1710 1711 pci_assign_unassigned_bus_resources(adev->pdev->bus); 1712 1713 /* When the doorbell or fb BAR isn't available we have no chance of 1714 * using the device. 1715 */ 1716 r = amdgpu_doorbell_init(adev); 1717 if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET)) 1718 return -ENODEV; 1719 1720 pci_write_config_word(adev->pdev, PCI_COMMAND, cmd); 1721 1722 return 0; 1723 } 1724 1725 /* 1726 * GPU helpers function. 1727 */ 1728 /** 1729 * amdgpu_device_need_post - check if the hw need post or not 1730 * 1731 * @adev: amdgpu_device pointer 1732 * 1733 * Check if the asic has been initialized (all asics) at driver startup 1734 * or post is needed if hw reset is performed. 1735 * Returns true if need or false if not. 1736 */ 1737 bool amdgpu_device_need_post(struct amdgpu_device *adev) 1738 { 1739 uint32_t reg, flags; 1740 1741 if (amdgpu_sriov_vf(adev)) 1742 return false; 1743 1744 flags = amdgpu_device_get_vbios_flags(adev); 1745 if (flags & AMDGPU_VBIOS_SKIP) 1746 return false; 1747 if ((flags & AMDGPU_VBIOS_OPTIONAL) && !adev->bios) 1748 return false; 1749 1750 if (amdgpu_passthrough(adev)) { 1751 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot 1752 * some old smc fw still need driver do vPost otherwise gpu hang, while 1753 * those smc fw version above 22.15 doesn't have this flaw, so we force 1754 * vpost executed for smc version below 22.15 1755 */ 1756 if (adev->asic_type == CHIP_FIJI) { 1757 int err; 1758 uint32_t fw_ver; 1759 1760 err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev); 1761 /* force vPost if error occurred */ 1762 if (err) 1763 return true; 1764 1765 fw_ver = *((uint32_t *)adev->pm.fw->data + 69); 1766 release_firmware(adev->pm.fw); 1767 if (fw_ver < 0x00160e00) 1768 return true; 1769 } 1770 } 1771 1772 /* Don't post if we need to reset whole hive on init */ 1773 if (adev->init_lvl->level == AMDGPU_INIT_LEVEL_MINIMAL_XGMI) 1774 return false; 1775 1776 if (adev->has_hw_reset) { 1777 adev->has_hw_reset = false; 1778 return true; 1779 } 1780 1781 /* bios scratch used on CIK+ */ 1782 if (adev->asic_type >= CHIP_BONAIRE) 1783 return amdgpu_atombios_scratch_need_asic_init(adev); 1784 1785 /* check MEM_SIZE for older asics */ 1786 reg = amdgpu_asic_get_config_memsize(adev); 1787 1788 if ((reg != 0) && (reg != 0xffffffff)) 1789 return false; 1790 1791 return true; 1792 } 1793 1794 /* 1795 * Check whether seamless boot is supported. 1796 * 1797 * So far we only support seamless boot on DCE 3.0 or later. 1798 * If users report that it works on older ASICS as well, we may 1799 * loosen this. 1800 */ 1801 bool amdgpu_device_seamless_boot_supported(struct amdgpu_device *adev) 1802 { 1803 switch (amdgpu_seamless) { 1804 case -1: 1805 break; 1806 case 1: 1807 return true; 1808 case 0: 1809 return false; 1810 default: 1811 DRM_ERROR("Invalid value for amdgpu.seamless: %d\n", 1812 amdgpu_seamless); 1813 return false; 1814 } 1815 1816 if (!(adev->flags & AMD_IS_APU)) 1817 return false; 1818 1819 if (adev->mman.keep_stolen_vga_memory) 1820 return false; 1821 1822 return amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(3, 0, 0); 1823 } 1824 1825 /* 1826 * Intel hosts such as Rocket Lake, Alder Lake, Raptor Lake and Sapphire Rapids 1827 * don't support dynamic speed switching. Until we have confirmation from Intel 1828 * that a specific host supports it, it's safer that we keep it disabled for all. 1829 * 1830 * https://edc.intel.com/content/www/us/en/design/products/platforms/details/raptor-lake-s/13th-generation-core-processors-datasheet-volume-1-of-2/005/pci-express-support/ 1831 * https://gitlab.freedesktop.org/drm/amd/-/issues/2663 1832 */ 1833 static bool amdgpu_device_pcie_dynamic_switching_supported(struct amdgpu_device *adev) 1834 { 1835 #if IS_ENABLED(CONFIG_X86) 1836 struct cpuinfo_x86 *c = &cpu_data(0); 1837 1838 /* eGPU change speeds based on USB4 fabric conditions */ 1839 if (dev_is_removable(adev->dev)) 1840 return true; 1841 1842 if (c->x86_vendor == X86_VENDOR_INTEL) 1843 return false; 1844 #endif 1845 return true; 1846 } 1847 1848 /** 1849 * amdgpu_device_should_use_aspm - check if the device should program ASPM 1850 * 1851 * @adev: amdgpu_device pointer 1852 * 1853 * Confirm whether the module parameter and pcie bridge agree that ASPM should 1854 * be set for this device. 1855 * 1856 * Returns true if it should be used or false if not. 1857 */ 1858 bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev) 1859 { 1860 switch (amdgpu_aspm) { 1861 case -1: 1862 break; 1863 case 0: 1864 return false; 1865 case 1: 1866 return true; 1867 default: 1868 return false; 1869 } 1870 if (adev->flags & AMD_IS_APU) 1871 return false; 1872 if (!(adev->pm.pp_feature & PP_PCIE_DPM_MASK)) 1873 return false; 1874 return pcie_aspm_enabled(adev->pdev); 1875 } 1876 1877 /* if we get transitioned to only one device, take VGA back */ 1878 /** 1879 * amdgpu_device_vga_set_decode - enable/disable vga decode 1880 * 1881 * @pdev: PCI device pointer 1882 * @state: enable/disable vga decode 1883 * 1884 * Enable/disable vga decode (all asics). 1885 * Returns VGA resource flags. 1886 */ 1887 static unsigned int amdgpu_device_vga_set_decode(struct pci_dev *pdev, 1888 bool state) 1889 { 1890 struct amdgpu_device *adev = drm_to_adev(pci_get_drvdata(pdev)); 1891 1892 amdgpu_asic_set_vga_state(adev, state); 1893 if (state) 1894 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM | 1895 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 1896 else 1897 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 1898 } 1899 1900 /** 1901 * amdgpu_device_check_block_size - validate the vm block size 1902 * 1903 * @adev: amdgpu_device pointer 1904 * 1905 * Validates the vm block size specified via module parameter. 1906 * The vm block size defines number of bits in page table versus page directory, 1907 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the 1908 * page table and the remaining bits are in the page directory. 1909 */ 1910 static void amdgpu_device_check_block_size(struct amdgpu_device *adev) 1911 { 1912 /* defines number of bits in page table versus page directory, 1913 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the 1914 * page table and the remaining bits are in the page directory 1915 */ 1916 if (amdgpu_vm_block_size == -1) 1917 return; 1918 1919 if (amdgpu_vm_block_size < 9) { 1920 dev_warn(adev->dev, "VM page table size (%d) too small\n", 1921 amdgpu_vm_block_size); 1922 amdgpu_vm_block_size = -1; 1923 } 1924 } 1925 1926 /** 1927 * amdgpu_device_check_vm_size - validate the vm size 1928 * 1929 * @adev: amdgpu_device pointer 1930 * 1931 * Validates the vm size in GB specified via module parameter. 1932 * The VM size is the size of the GPU virtual memory space in GB. 1933 */ 1934 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev) 1935 { 1936 /* no need to check the default value */ 1937 if (amdgpu_vm_size == -1) 1938 return; 1939 1940 if (amdgpu_vm_size < 1) { 1941 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n", 1942 amdgpu_vm_size); 1943 amdgpu_vm_size = -1; 1944 } 1945 } 1946 1947 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev) 1948 { 1949 struct sysinfo si; 1950 bool is_os_64 = (sizeof(void *) == 8); 1951 uint64_t total_memory; 1952 uint64_t dram_size_seven_GB = 0x1B8000000; 1953 uint64_t dram_size_three_GB = 0xB8000000; 1954 1955 if (amdgpu_smu_memory_pool_size == 0) 1956 return; 1957 1958 if (!is_os_64) { 1959 DRM_WARN("Not 64-bit OS, feature not supported\n"); 1960 goto def_value; 1961 } 1962 si_meminfo(&si); 1963 total_memory = (uint64_t)si.totalram * si.mem_unit; 1964 1965 if ((amdgpu_smu_memory_pool_size == 1) || 1966 (amdgpu_smu_memory_pool_size == 2)) { 1967 if (total_memory < dram_size_three_GB) 1968 goto def_value1; 1969 } else if ((amdgpu_smu_memory_pool_size == 4) || 1970 (amdgpu_smu_memory_pool_size == 8)) { 1971 if (total_memory < dram_size_seven_GB) 1972 goto def_value1; 1973 } else { 1974 DRM_WARN("Smu memory pool size not supported\n"); 1975 goto def_value; 1976 } 1977 adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28; 1978 1979 return; 1980 1981 def_value1: 1982 DRM_WARN("No enough system memory\n"); 1983 def_value: 1984 adev->pm.smu_prv_buffer_size = 0; 1985 } 1986 1987 static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev) 1988 { 1989 if (!(adev->flags & AMD_IS_APU) || 1990 adev->asic_type < CHIP_RAVEN) 1991 return 0; 1992 1993 switch (adev->asic_type) { 1994 case CHIP_RAVEN: 1995 if (adev->pdev->device == 0x15dd) 1996 adev->apu_flags |= AMD_APU_IS_RAVEN; 1997 if (adev->pdev->device == 0x15d8) 1998 adev->apu_flags |= AMD_APU_IS_PICASSO; 1999 break; 2000 case CHIP_RENOIR: 2001 if ((adev->pdev->device == 0x1636) || 2002 (adev->pdev->device == 0x164c)) 2003 adev->apu_flags |= AMD_APU_IS_RENOIR; 2004 else 2005 adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE; 2006 break; 2007 case CHIP_VANGOGH: 2008 adev->apu_flags |= AMD_APU_IS_VANGOGH; 2009 break; 2010 case CHIP_YELLOW_CARP: 2011 break; 2012 case CHIP_CYAN_SKILLFISH: 2013 if ((adev->pdev->device == 0x13FE) || 2014 (adev->pdev->device == 0x143F)) 2015 adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2; 2016 break; 2017 default: 2018 break; 2019 } 2020 2021 return 0; 2022 } 2023 2024 /** 2025 * amdgpu_device_check_arguments - validate module params 2026 * 2027 * @adev: amdgpu_device pointer 2028 * 2029 * Validates certain module parameters and updates 2030 * the associated values used by the driver (all asics). 2031 */ 2032 static int amdgpu_device_check_arguments(struct amdgpu_device *adev) 2033 { 2034 int i; 2035 2036 if (amdgpu_sched_jobs < 4) { 2037 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n", 2038 amdgpu_sched_jobs); 2039 amdgpu_sched_jobs = 4; 2040 } else if (!is_power_of_2(amdgpu_sched_jobs)) { 2041 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n", 2042 amdgpu_sched_jobs); 2043 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs); 2044 } 2045 2046 if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) { 2047 /* gart size must be greater or equal to 32M */ 2048 dev_warn(adev->dev, "gart size (%d) too small\n", 2049 amdgpu_gart_size); 2050 amdgpu_gart_size = -1; 2051 } 2052 2053 if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) { 2054 /* gtt size must be greater or equal to 32M */ 2055 dev_warn(adev->dev, "gtt size (%d) too small\n", 2056 amdgpu_gtt_size); 2057 amdgpu_gtt_size = -1; 2058 } 2059 2060 /* valid range is between 4 and 9 inclusive */ 2061 if (amdgpu_vm_fragment_size != -1 && 2062 (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) { 2063 dev_warn(adev->dev, "valid range is between 4 and 9\n"); 2064 amdgpu_vm_fragment_size = -1; 2065 } 2066 2067 if (amdgpu_sched_hw_submission < 2) { 2068 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n", 2069 amdgpu_sched_hw_submission); 2070 amdgpu_sched_hw_submission = 2; 2071 } else if (!is_power_of_2(amdgpu_sched_hw_submission)) { 2072 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n", 2073 amdgpu_sched_hw_submission); 2074 amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission); 2075 } 2076 2077 if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) { 2078 dev_warn(adev->dev, "invalid option for reset method, reverting to default\n"); 2079 amdgpu_reset_method = -1; 2080 } 2081 2082 amdgpu_device_check_smu_prv_buffer_size(adev); 2083 2084 amdgpu_device_check_vm_size(adev); 2085 2086 amdgpu_device_check_block_size(adev); 2087 2088 adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type); 2089 2090 for (i = 0; i < MAX_XCP; i++) 2091 adev->enforce_isolation[i] = !!enforce_isolation; 2092 2093 return 0; 2094 } 2095 2096 /** 2097 * amdgpu_switcheroo_set_state - set switcheroo state 2098 * 2099 * @pdev: pci dev pointer 2100 * @state: vga_switcheroo state 2101 * 2102 * Callback for the switcheroo driver. Suspends or resumes 2103 * the asics before or after it is powered up using ACPI methods. 2104 */ 2105 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, 2106 enum vga_switcheroo_state state) 2107 { 2108 struct drm_device *dev = pci_get_drvdata(pdev); 2109 int r; 2110 2111 if (amdgpu_device_supports_px(dev) && state == VGA_SWITCHEROO_OFF) 2112 return; 2113 2114 if (state == VGA_SWITCHEROO_ON) { 2115 pr_info("switched on\n"); 2116 /* don't suspend or resume card normally */ 2117 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; 2118 2119 pci_set_power_state(pdev, PCI_D0); 2120 amdgpu_device_load_pci_state(pdev); 2121 r = pci_enable_device(pdev); 2122 if (r) 2123 DRM_WARN("pci_enable_device failed (%d)\n", r); 2124 amdgpu_device_resume(dev, true); 2125 2126 dev->switch_power_state = DRM_SWITCH_POWER_ON; 2127 } else { 2128 pr_info("switched off\n"); 2129 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; 2130 amdgpu_device_prepare(dev); 2131 amdgpu_device_suspend(dev, true); 2132 amdgpu_device_cache_pci_state(pdev); 2133 /* Shut down the device */ 2134 pci_disable_device(pdev); 2135 pci_set_power_state(pdev, PCI_D3cold); 2136 dev->switch_power_state = DRM_SWITCH_POWER_OFF; 2137 } 2138 } 2139 2140 /** 2141 * amdgpu_switcheroo_can_switch - see if switcheroo state can change 2142 * 2143 * @pdev: pci dev pointer 2144 * 2145 * Callback for the switcheroo driver. Check of the switcheroo 2146 * state can be changed. 2147 * Returns true if the state can be changed, false if not. 2148 */ 2149 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev) 2150 { 2151 struct drm_device *dev = pci_get_drvdata(pdev); 2152 2153 /* 2154 * FIXME: open_count is protected by drm_global_mutex but that would lead to 2155 * locking inversion with the driver load path. And the access here is 2156 * completely racy anyway. So don't bother with locking for now. 2157 */ 2158 return atomic_read(&dev->open_count) == 0; 2159 } 2160 2161 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = { 2162 .set_gpu_state = amdgpu_switcheroo_set_state, 2163 .reprobe = NULL, 2164 .can_switch = amdgpu_switcheroo_can_switch, 2165 }; 2166 2167 /** 2168 * amdgpu_device_ip_set_clockgating_state - set the CG state 2169 * 2170 * @dev: amdgpu_device pointer 2171 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 2172 * @state: clockgating state (gate or ungate) 2173 * 2174 * Sets the requested clockgating state for all instances of 2175 * the hardware IP specified. 2176 * Returns the error code from the last instance. 2177 */ 2178 int amdgpu_device_ip_set_clockgating_state(void *dev, 2179 enum amd_ip_block_type block_type, 2180 enum amd_clockgating_state state) 2181 { 2182 struct amdgpu_device *adev = dev; 2183 int i, r = 0; 2184 2185 for (i = 0; i < adev->num_ip_blocks; i++) { 2186 if (!adev->ip_blocks[i].status.valid) 2187 continue; 2188 if (adev->ip_blocks[i].version->type != block_type) 2189 continue; 2190 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state) 2191 continue; 2192 r = adev->ip_blocks[i].version->funcs->set_clockgating_state( 2193 &adev->ip_blocks[i], state); 2194 if (r) 2195 DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n", 2196 adev->ip_blocks[i].version->funcs->name, r); 2197 } 2198 return r; 2199 } 2200 2201 /** 2202 * amdgpu_device_ip_set_powergating_state - set the PG state 2203 * 2204 * @dev: amdgpu_device pointer 2205 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 2206 * @state: powergating state (gate or ungate) 2207 * 2208 * Sets the requested powergating state for all instances of 2209 * the hardware IP specified. 2210 * Returns the error code from the last instance. 2211 */ 2212 int amdgpu_device_ip_set_powergating_state(void *dev, 2213 enum amd_ip_block_type block_type, 2214 enum amd_powergating_state state) 2215 { 2216 struct amdgpu_device *adev = dev; 2217 int i, r = 0; 2218 2219 for (i = 0; i < adev->num_ip_blocks; i++) { 2220 if (!adev->ip_blocks[i].status.valid) 2221 continue; 2222 if (adev->ip_blocks[i].version->type != block_type) 2223 continue; 2224 if (!adev->ip_blocks[i].version->funcs->set_powergating_state) 2225 continue; 2226 r = adev->ip_blocks[i].version->funcs->set_powergating_state( 2227 &adev->ip_blocks[i], state); 2228 if (r) 2229 DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n", 2230 adev->ip_blocks[i].version->funcs->name, r); 2231 } 2232 return r; 2233 } 2234 2235 /** 2236 * amdgpu_device_ip_get_clockgating_state - get the CG state 2237 * 2238 * @adev: amdgpu_device pointer 2239 * @flags: clockgating feature flags 2240 * 2241 * Walks the list of IPs on the device and updates the clockgating 2242 * flags for each IP. 2243 * Updates @flags with the feature flags for each hardware IP where 2244 * clockgating is enabled. 2245 */ 2246 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev, 2247 u64 *flags) 2248 { 2249 int i; 2250 2251 for (i = 0; i < adev->num_ip_blocks; i++) { 2252 if (!adev->ip_blocks[i].status.valid) 2253 continue; 2254 if (adev->ip_blocks[i].version->funcs->get_clockgating_state) 2255 adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags); 2256 } 2257 } 2258 2259 /** 2260 * amdgpu_device_ip_wait_for_idle - wait for idle 2261 * 2262 * @adev: amdgpu_device pointer 2263 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 2264 * 2265 * Waits for the request hardware IP to be idle. 2266 * Returns 0 for success or a negative error code on failure. 2267 */ 2268 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev, 2269 enum amd_ip_block_type block_type) 2270 { 2271 int i, r; 2272 2273 for (i = 0; i < adev->num_ip_blocks; i++) { 2274 if (!adev->ip_blocks[i].status.valid) 2275 continue; 2276 if (adev->ip_blocks[i].version->type == block_type) { 2277 if (adev->ip_blocks[i].version->funcs->wait_for_idle) { 2278 r = adev->ip_blocks[i].version->funcs->wait_for_idle( 2279 &adev->ip_blocks[i]); 2280 if (r) 2281 return r; 2282 } 2283 break; 2284 } 2285 } 2286 return 0; 2287 2288 } 2289 2290 /** 2291 * amdgpu_device_ip_is_valid - is the hardware IP enabled 2292 * 2293 * @adev: amdgpu_device pointer 2294 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 2295 * 2296 * Check if the hardware IP is enable or not. 2297 * Returns true if it the IP is enable, false if not. 2298 */ 2299 bool amdgpu_device_ip_is_valid(struct amdgpu_device *adev, 2300 enum amd_ip_block_type block_type) 2301 { 2302 int i; 2303 2304 for (i = 0; i < adev->num_ip_blocks; i++) { 2305 if (adev->ip_blocks[i].version->type == block_type) 2306 return adev->ip_blocks[i].status.valid; 2307 } 2308 return false; 2309 2310 } 2311 2312 /** 2313 * amdgpu_device_ip_get_ip_block - get a hw IP pointer 2314 * 2315 * @adev: amdgpu_device pointer 2316 * @type: Type of hardware IP (SMU, GFX, UVD, etc.) 2317 * 2318 * Returns a pointer to the hardware IP block structure 2319 * if it exists for the asic, otherwise NULL. 2320 */ 2321 struct amdgpu_ip_block * 2322 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev, 2323 enum amd_ip_block_type type) 2324 { 2325 int i; 2326 2327 for (i = 0; i < adev->num_ip_blocks; i++) 2328 if (adev->ip_blocks[i].version->type == type) 2329 return &adev->ip_blocks[i]; 2330 2331 return NULL; 2332 } 2333 2334 /** 2335 * amdgpu_device_ip_block_version_cmp 2336 * 2337 * @adev: amdgpu_device pointer 2338 * @type: enum amd_ip_block_type 2339 * @major: major version 2340 * @minor: minor version 2341 * 2342 * return 0 if equal or greater 2343 * return 1 if smaller or the ip_block doesn't exist 2344 */ 2345 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev, 2346 enum amd_ip_block_type type, 2347 u32 major, u32 minor) 2348 { 2349 struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type); 2350 2351 if (ip_block && ((ip_block->version->major > major) || 2352 ((ip_block->version->major == major) && 2353 (ip_block->version->minor >= minor)))) 2354 return 0; 2355 2356 return 1; 2357 } 2358 2359 /** 2360 * amdgpu_device_ip_block_add 2361 * 2362 * @adev: amdgpu_device pointer 2363 * @ip_block_version: pointer to the IP to add 2364 * 2365 * Adds the IP block driver information to the collection of IPs 2366 * on the asic. 2367 */ 2368 int amdgpu_device_ip_block_add(struct amdgpu_device *adev, 2369 const struct amdgpu_ip_block_version *ip_block_version) 2370 { 2371 if (!ip_block_version) 2372 return -EINVAL; 2373 2374 switch (ip_block_version->type) { 2375 case AMD_IP_BLOCK_TYPE_VCN: 2376 if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK) 2377 return 0; 2378 break; 2379 case AMD_IP_BLOCK_TYPE_JPEG: 2380 if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK) 2381 return 0; 2382 break; 2383 default: 2384 break; 2385 } 2386 2387 dev_info(adev->dev, "detected ip block number %d <%s>\n", 2388 adev->num_ip_blocks, ip_block_version->funcs->name); 2389 2390 adev->ip_blocks[adev->num_ip_blocks].adev = adev; 2391 2392 adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version; 2393 2394 return 0; 2395 } 2396 2397 /** 2398 * amdgpu_device_enable_virtual_display - enable virtual display feature 2399 * 2400 * @adev: amdgpu_device pointer 2401 * 2402 * Enabled the virtual display feature if the user has enabled it via 2403 * the module parameter virtual_display. This feature provides a virtual 2404 * display hardware on headless boards or in virtualized environments. 2405 * This function parses and validates the configuration string specified by 2406 * the user and configures the virtual display configuration (number of 2407 * virtual connectors, crtcs, etc.) specified. 2408 */ 2409 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev) 2410 { 2411 adev->enable_virtual_display = false; 2412 2413 if (amdgpu_virtual_display) { 2414 const char *pci_address_name = pci_name(adev->pdev); 2415 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname; 2416 2417 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL); 2418 pciaddstr_tmp = pciaddstr; 2419 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) { 2420 pciaddname = strsep(&pciaddname_tmp, ","); 2421 if (!strcmp("all", pciaddname) 2422 || !strcmp(pci_address_name, pciaddname)) { 2423 long num_crtc; 2424 int res = -1; 2425 2426 adev->enable_virtual_display = true; 2427 2428 if (pciaddname_tmp) 2429 res = kstrtol(pciaddname_tmp, 10, 2430 &num_crtc); 2431 2432 if (!res) { 2433 if (num_crtc < 1) 2434 num_crtc = 1; 2435 if (num_crtc > 6) 2436 num_crtc = 6; 2437 adev->mode_info.num_crtc = num_crtc; 2438 } else { 2439 adev->mode_info.num_crtc = 1; 2440 } 2441 break; 2442 } 2443 } 2444 2445 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n", 2446 amdgpu_virtual_display, pci_address_name, 2447 adev->enable_virtual_display, adev->mode_info.num_crtc); 2448 2449 kfree(pciaddstr); 2450 } 2451 } 2452 2453 void amdgpu_device_set_sriov_virtual_display(struct amdgpu_device *adev) 2454 { 2455 if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) { 2456 adev->mode_info.num_crtc = 1; 2457 adev->enable_virtual_display = true; 2458 DRM_INFO("virtual_display:%d, num_crtc:%d\n", 2459 adev->enable_virtual_display, adev->mode_info.num_crtc); 2460 } 2461 } 2462 2463 /** 2464 * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware 2465 * 2466 * @adev: amdgpu_device pointer 2467 * 2468 * Parses the asic configuration parameters specified in the gpu info 2469 * firmware and makes them available to the driver for use in configuring 2470 * the asic. 2471 * Returns 0 on success, -EINVAL on failure. 2472 */ 2473 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev) 2474 { 2475 const char *chip_name; 2476 int err; 2477 const struct gpu_info_firmware_header_v1_0 *hdr; 2478 2479 adev->firmware.gpu_info_fw = NULL; 2480 2481 if (adev->mman.discovery_bin) 2482 return 0; 2483 2484 switch (adev->asic_type) { 2485 default: 2486 return 0; 2487 case CHIP_VEGA10: 2488 chip_name = "vega10"; 2489 break; 2490 case CHIP_VEGA12: 2491 chip_name = "vega12"; 2492 break; 2493 case CHIP_RAVEN: 2494 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 2495 chip_name = "raven2"; 2496 else if (adev->apu_flags & AMD_APU_IS_PICASSO) 2497 chip_name = "picasso"; 2498 else 2499 chip_name = "raven"; 2500 break; 2501 case CHIP_ARCTURUS: 2502 chip_name = "arcturus"; 2503 break; 2504 case CHIP_NAVI12: 2505 chip_name = "navi12"; 2506 break; 2507 } 2508 2509 err = amdgpu_ucode_request(adev, &adev->firmware.gpu_info_fw, 2510 AMDGPU_UCODE_OPTIONAL, 2511 "amdgpu/%s_gpu_info.bin", chip_name); 2512 if (err) { 2513 dev_err(adev->dev, 2514 "Failed to get gpu_info firmware \"%s_gpu_info.bin\"\n", 2515 chip_name); 2516 goto out; 2517 } 2518 2519 hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data; 2520 amdgpu_ucode_print_gpu_info_hdr(&hdr->header); 2521 2522 switch (hdr->version_major) { 2523 case 1: 2524 { 2525 const struct gpu_info_firmware_v1_0 *gpu_info_fw = 2526 (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data + 2527 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 2528 2529 /* 2530 * Should be dropped when DAL no longer needs it. 2531 */ 2532 if (adev->asic_type == CHIP_NAVI12) 2533 goto parse_soc_bounding_box; 2534 2535 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se); 2536 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh); 2537 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se); 2538 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se); 2539 adev->gfx.config.max_texture_channel_caches = 2540 le32_to_cpu(gpu_info_fw->gc_num_tccs); 2541 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs); 2542 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds); 2543 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth); 2544 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth); 2545 adev->gfx.config.double_offchip_lds_buf = 2546 le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer); 2547 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size); 2548 adev->gfx.cu_info.max_waves_per_simd = 2549 le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd); 2550 adev->gfx.cu_info.max_scratch_slots_per_cu = 2551 le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu); 2552 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size); 2553 if (hdr->version_minor >= 1) { 2554 const struct gpu_info_firmware_v1_1 *gpu_info_fw = 2555 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data + 2556 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 2557 adev->gfx.config.num_sc_per_sh = 2558 le32_to_cpu(gpu_info_fw->num_sc_per_sh); 2559 adev->gfx.config.num_packer_per_sc = 2560 le32_to_cpu(gpu_info_fw->num_packer_per_sc); 2561 } 2562 2563 parse_soc_bounding_box: 2564 /* 2565 * soc bounding box info is not integrated in disocovery table, 2566 * we always need to parse it from gpu info firmware if needed. 2567 */ 2568 if (hdr->version_minor == 2) { 2569 const struct gpu_info_firmware_v1_2 *gpu_info_fw = 2570 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data + 2571 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 2572 adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box; 2573 } 2574 break; 2575 } 2576 default: 2577 dev_err(adev->dev, 2578 "Unsupported gpu_info table %d\n", hdr->header.ucode_version); 2579 err = -EINVAL; 2580 goto out; 2581 } 2582 out: 2583 return err; 2584 } 2585 2586 /** 2587 * amdgpu_device_ip_early_init - run early init for hardware IPs 2588 * 2589 * @adev: amdgpu_device pointer 2590 * 2591 * Early initialization pass for hardware IPs. The hardware IPs that make 2592 * up each asic are discovered each IP's early_init callback is run. This 2593 * is the first stage in initializing the asic. 2594 * Returns 0 on success, negative error code on failure. 2595 */ 2596 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev) 2597 { 2598 struct amdgpu_ip_block *ip_block; 2599 struct pci_dev *parent; 2600 bool total, skip_bios; 2601 uint32_t bios_flags; 2602 int i, r; 2603 2604 amdgpu_device_enable_virtual_display(adev); 2605 2606 if (amdgpu_sriov_vf(adev)) { 2607 r = amdgpu_virt_request_full_gpu(adev, true); 2608 if (r) 2609 return r; 2610 } 2611 2612 switch (adev->asic_type) { 2613 #ifdef CONFIG_DRM_AMDGPU_SI 2614 case CHIP_VERDE: 2615 case CHIP_TAHITI: 2616 case CHIP_PITCAIRN: 2617 case CHIP_OLAND: 2618 case CHIP_HAINAN: 2619 adev->family = AMDGPU_FAMILY_SI; 2620 r = si_set_ip_blocks(adev); 2621 if (r) 2622 return r; 2623 break; 2624 #endif 2625 #ifdef CONFIG_DRM_AMDGPU_CIK 2626 case CHIP_BONAIRE: 2627 case CHIP_HAWAII: 2628 case CHIP_KAVERI: 2629 case CHIP_KABINI: 2630 case CHIP_MULLINS: 2631 if (adev->flags & AMD_IS_APU) 2632 adev->family = AMDGPU_FAMILY_KV; 2633 else 2634 adev->family = AMDGPU_FAMILY_CI; 2635 2636 r = cik_set_ip_blocks(adev); 2637 if (r) 2638 return r; 2639 break; 2640 #endif 2641 case CHIP_TOPAZ: 2642 case CHIP_TONGA: 2643 case CHIP_FIJI: 2644 case CHIP_POLARIS10: 2645 case CHIP_POLARIS11: 2646 case CHIP_POLARIS12: 2647 case CHIP_VEGAM: 2648 case CHIP_CARRIZO: 2649 case CHIP_STONEY: 2650 if (adev->flags & AMD_IS_APU) 2651 adev->family = AMDGPU_FAMILY_CZ; 2652 else 2653 adev->family = AMDGPU_FAMILY_VI; 2654 2655 r = vi_set_ip_blocks(adev); 2656 if (r) 2657 return r; 2658 break; 2659 default: 2660 r = amdgpu_discovery_set_ip_blocks(adev); 2661 if (r) 2662 return r; 2663 break; 2664 } 2665 2666 if (amdgpu_has_atpx() && 2667 (amdgpu_is_atpx_hybrid() || 2668 amdgpu_has_atpx_dgpu_power_cntl()) && 2669 ((adev->flags & AMD_IS_APU) == 0) && 2670 !dev_is_removable(&adev->pdev->dev)) 2671 adev->flags |= AMD_IS_PX; 2672 2673 if (!(adev->flags & AMD_IS_APU)) { 2674 parent = pcie_find_root_port(adev->pdev); 2675 adev->has_pr3 = parent ? pci_pr3_present(parent) : false; 2676 } 2677 2678 2679 adev->pm.pp_feature = amdgpu_pp_feature_mask; 2680 if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS) 2681 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 2682 if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID) 2683 adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK; 2684 if (!amdgpu_device_pcie_dynamic_switching_supported(adev)) 2685 adev->pm.pp_feature &= ~PP_PCIE_DPM_MASK; 2686 2687 total = true; 2688 for (i = 0; i < adev->num_ip_blocks; i++) { 2689 ip_block = &adev->ip_blocks[i]; 2690 2691 if ((amdgpu_ip_block_mask & (1 << i)) == 0) { 2692 DRM_WARN("disabled ip block: %d <%s>\n", 2693 i, adev->ip_blocks[i].version->funcs->name); 2694 adev->ip_blocks[i].status.valid = false; 2695 } else if (ip_block->version->funcs->early_init) { 2696 r = ip_block->version->funcs->early_init(ip_block); 2697 if (r == -ENOENT) { 2698 adev->ip_blocks[i].status.valid = false; 2699 } else if (r) { 2700 DRM_ERROR("early_init of IP block <%s> failed %d\n", 2701 adev->ip_blocks[i].version->funcs->name, r); 2702 total = false; 2703 } else { 2704 adev->ip_blocks[i].status.valid = true; 2705 } 2706 } else { 2707 adev->ip_blocks[i].status.valid = true; 2708 } 2709 /* get the vbios after the asic_funcs are set up */ 2710 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) { 2711 r = amdgpu_device_parse_gpu_info_fw(adev); 2712 if (r) 2713 return r; 2714 2715 bios_flags = amdgpu_device_get_vbios_flags(adev); 2716 skip_bios = !!(bios_flags & AMDGPU_VBIOS_SKIP); 2717 /* Read BIOS */ 2718 if (!skip_bios) { 2719 bool optional = 2720 !!(bios_flags & AMDGPU_VBIOS_OPTIONAL); 2721 if (!amdgpu_get_bios(adev) && !optional) 2722 return -EINVAL; 2723 2724 if (optional && !adev->bios) 2725 dev_info( 2726 adev->dev, 2727 "VBIOS image optional, proceeding without VBIOS image"); 2728 2729 if (adev->bios) { 2730 r = amdgpu_atombios_init(adev); 2731 if (r) { 2732 dev_err(adev->dev, 2733 "amdgpu_atombios_init failed\n"); 2734 amdgpu_vf_error_put( 2735 adev, 2736 AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 2737 0, 0); 2738 return r; 2739 } 2740 } 2741 } 2742 2743 /*get pf2vf msg info at it's earliest time*/ 2744 if (amdgpu_sriov_vf(adev)) 2745 amdgpu_virt_init_data_exchange(adev); 2746 2747 } 2748 } 2749 if (!total) 2750 return -ENODEV; 2751 2752 ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX); 2753 if (ip_block->status.valid != false) 2754 amdgpu_amdkfd_device_probe(adev); 2755 2756 adev->cg_flags &= amdgpu_cg_mask; 2757 adev->pg_flags &= amdgpu_pg_mask; 2758 2759 return 0; 2760 } 2761 2762 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev) 2763 { 2764 int i, r; 2765 2766 for (i = 0; i < adev->num_ip_blocks; i++) { 2767 if (!adev->ip_blocks[i].status.sw) 2768 continue; 2769 if (adev->ip_blocks[i].status.hw) 2770 continue; 2771 if (!amdgpu_ip_member_of_hwini( 2772 adev, adev->ip_blocks[i].version->type)) 2773 continue; 2774 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || 2775 (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) || 2776 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) { 2777 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]); 2778 if (r) { 2779 DRM_ERROR("hw_init of IP block <%s> failed %d\n", 2780 adev->ip_blocks[i].version->funcs->name, r); 2781 return r; 2782 } 2783 adev->ip_blocks[i].status.hw = true; 2784 } 2785 } 2786 2787 return 0; 2788 } 2789 2790 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev) 2791 { 2792 int i, r; 2793 2794 for (i = 0; i < adev->num_ip_blocks; i++) { 2795 if (!adev->ip_blocks[i].status.sw) 2796 continue; 2797 if (adev->ip_blocks[i].status.hw) 2798 continue; 2799 if (!amdgpu_ip_member_of_hwini( 2800 adev, adev->ip_blocks[i].version->type)) 2801 continue; 2802 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]); 2803 if (r) { 2804 DRM_ERROR("hw_init of IP block <%s> failed %d\n", 2805 adev->ip_blocks[i].version->funcs->name, r); 2806 return r; 2807 } 2808 adev->ip_blocks[i].status.hw = true; 2809 } 2810 2811 return 0; 2812 } 2813 2814 static int amdgpu_device_fw_loading(struct amdgpu_device *adev) 2815 { 2816 int r = 0; 2817 int i; 2818 uint32_t smu_version; 2819 2820 if (adev->asic_type >= CHIP_VEGA10) { 2821 for (i = 0; i < adev->num_ip_blocks; i++) { 2822 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP) 2823 continue; 2824 2825 if (!amdgpu_ip_member_of_hwini(adev, 2826 AMD_IP_BLOCK_TYPE_PSP)) 2827 break; 2828 2829 if (!adev->ip_blocks[i].status.sw) 2830 continue; 2831 2832 /* no need to do the fw loading again if already done*/ 2833 if (adev->ip_blocks[i].status.hw == true) 2834 break; 2835 2836 if (amdgpu_in_reset(adev) || adev->in_suspend) { 2837 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]); 2838 if (r) 2839 return r; 2840 } else { 2841 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]); 2842 if (r) { 2843 DRM_ERROR("hw_init of IP block <%s> failed %d\n", 2844 adev->ip_blocks[i].version->funcs->name, r); 2845 return r; 2846 } 2847 adev->ip_blocks[i].status.hw = true; 2848 } 2849 break; 2850 } 2851 } 2852 2853 if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA) 2854 r = amdgpu_pm_load_smu_firmware(adev, &smu_version); 2855 2856 return r; 2857 } 2858 2859 static int amdgpu_device_init_schedulers(struct amdgpu_device *adev) 2860 { 2861 long timeout; 2862 int r, i; 2863 2864 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 2865 struct amdgpu_ring *ring = adev->rings[i]; 2866 2867 /* No need to setup the GPU scheduler for rings that don't need it */ 2868 if (!ring || ring->no_scheduler) 2869 continue; 2870 2871 switch (ring->funcs->type) { 2872 case AMDGPU_RING_TYPE_GFX: 2873 timeout = adev->gfx_timeout; 2874 break; 2875 case AMDGPU_RING_TYPE_COMPUTE: 2876 timeout = adev->compute_timeout; 2877 break; 2878 case AMDGPU_RING_TYPE_SDMA: 2879 timeout = adev->sdma_timeout; 2880 break; 2881 default: 2882 timeout = adev->video_timeout; 2883 break; 2884 } 2885 2886 r = drm_sched_init(&ring->sched, &amdgpu_sched_ops, NULL, 2887 DRM_SCHED_PRIORITY_COUNT, 2888 ring->num_hw_submission, 0, 2889 timeout, adev->reset_domain->wq, 2890 ring->sched_score, ring->name, 2891 adev->dev); 2892 if (r) { 2893 DRM_ERROR("Failed to create scheduler on ring %s.\n", 2894 ring->name); 2895 return r; 2896 } 2897 r = amdgpu_uvd_entity_init(adev, ring); 2898 if (r) { 2899 DRM_ERROR("Failed to create UVD scheduling entity on ring %s.\n", 2900 ring->name); 2901 return r; 2902 } 2903 r = amdgpu_vce_entity_init(adev, ring); 2904 if (r) { 2905 DRM_ERROR("Failed to create VCE scheduling entity on ring %s.\n", 2906 ring->name); 2907 return r; 2908 } 2909 } 2910 2911 amdgpu_xcp_update_partition_sched_list(adev); 2912 2913 return 0; 2914 } 2915 2916 2917 /** 2918 * amdgpu_device_ip_init - run init for hardware IPs 2919 * 2920 * @adev: amdgpu_device pointer 2921 * 2922 * Main initialization pass for hardware IPs. The list of all the hardware 2923 * IPs that make up the asic is walked and the sw_init and hw_init callbacks 2924 * are run. sw_init initializes the software state associated with each IP 2925 * and hw_init initializes the hardware associated with each IP. 2926 * Returns 0 on success, negative error code on failure. 2927 */ 2928 static int amdgpu_device_ip_init(struct amdgpu_device *adev) 2929 { 2930 bool init_badpage; 2931 int i, r; 2932 2933 r = amdgpu_ras_init(adev); 2934 if (r) 2935 return r; 2936 2937 for (i = 0; i < adev->num_ip_blocks; i++) { 2938 if (!adev->ip_blocks[i].status.valid) 2939 continue; 2940 if (adev->ip_blocks[i].version->funcs->sw_init) { 2941 r = adev->ip_blocks[i].version->funcs->sw_init(&adev->ip_blocks[i]); 2942 if (r) { 2943 DRM_ERROR("sw_init of IP block <%s> failed %d\n", 2944 adev->ip_blocks[i].version->funcs->name, r); 2945 goto init_failed; 2946 } 2947 } 2948 adev->ip_blocks[i].status.sw = true; 2949 2950 if (!amdgpu_ip_member_of_hwini( 2951 adev, adev->ip_blocks[i].version->type)) 2952 continue; 2953 2954 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) { 2955 /* need to do common hw init early so everything is set up for gmc */ 2956 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]); 2957 if (r) { 2958 DRM_ERROR("hw_init %d failed %d\n", i, r); 2959 goto init_failed; 2960 } 2961 adev->ip_blocks[i].status.hw = true; 2962 } else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { 2963 /* need to do gmc hw init early so we can allocate gpu mem */ 2964 /* Try to reserve bad pages early */ 2965 if (amdgpu_sriov_vf(adev)) 2966 amdgpu_virt_exchange_data(adev); 2967 2968 r = amdgpu_device_mem_scratch_init(adev); 2969 if (r) { 2970 DRM_ERROR("amdgpu_mem_scratch_init failed %d\n", r); 2971 goto init_failed; 2972 } 2973 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]); 2974 if (r) { 2975 DRM_ERROR("hw_init %d failed %d\n", i, r); 2976 goto init_failed; 2977 } 2978 r = amdgpu_device_wb_init(adev); 2979 if (r) { 2980 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r); 2981 goto init_failed; 2982 } 2983 adev->ip_blocks[i].status.hw = true; 2984 2985 /* right after GMC hw init, we create CSA */ 2986 if (adev->gfx.mcbp) { 2987 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj, 2988 AMDGPU_GEM_DOMAIN_VRAM | 2989 AMDGPU_GEM_DOMAIN_GTT, 2990 AMDGPU_CSA_SIZE); 2991 if (r) { 2992 DRM_ERROR("allocate CSA failed %d\n", r); 2993 goto init_failed; 2994 } 2995 } 2996 2997 r = amdgpu_seq64_init(adev); 2998 if (r) { 2999 DRM_ERROR("allocate seq64 failed %d\n", r); 3000 goto init_failed; 3001 } 3002 } 3003 } 3004 3005 if (amdgpu_sriov_vf(adev)) 3006 amdgpu_virt_init_data_exchange(adev); 3007 3008 r = amdgpu_ib_pool_init(adev); 3009 if (r) { 3010 dev_err(adev->dev, "IB initialization failed (%d).\n", r); 3011 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r); 3012 goto init_failed; 3013 } 3014 3015 r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/ 3016 if (r) 3017 goto init_failed; 3018 3019 r = amdgpu_device_ip_hw_init_phase1(adev); 3020 if (r) 3021 goto init_failed; 3022 3023 r = amdgpu_device_fw_loading(adev); 3024 if (r) 3025 goto init_failed; 3026 3027 r = amdgpu_device_ip_hw_init_phase2(adev); 3028 if (r) 3029 goto init_failed; 3030 3031 /* 3032 * retired pages will be loaded from eeprom and reserved here, 3033 * it should be called after amdgpu_device_ip_hw_init_phase2 since 3034 * for some ASICs the RAS EEPROM code relies on SMU fully functioning 3035 * for I2C communication which only true at this point. 3036 * 3037 * amdgpu_ras_recovery_init may fail, but the upper only cares the 3038 * failure from bad gpu situation and stop amdgpu init process 3039 * accordingly. For other failed cases, it will still release all 3040 * the resource and print error message, rather than returning one 3041 * negative value to upper level. 3042 * 3043 * Note: theoretically, this should be called before all vram allocations 3044 * to protect retired page from abusing 3045 */ 3046 init_badpage = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI); 3047 r = amdgpu_ras_recovery_init(adev, init_badpage); 3048 if (r) 3049 goto init_failed; 3050 3051 /** 3052 * In case of XGMI grab extra reference for reset domain for this device 3053 */ 3054 if (adev->gmc.xgmi.num_physical_nodes > 1) { 3055 if (amdgpu_xgmi_add_device(adev) == 0) { 3056 if (!amdgpu_sriov_vf(adev)) { 3057 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 3058 3059 if (WARN_ON(!hive)) { 3060 r = -ENOENT; 3061 goto init_failed; 3062 } 3063 3064 if (!hive->reset_domain || 3065 !amdgpu_reset_get_reset_domain(hive->reset_domain)) { 3066 r = -ENOENT; 3067 amdgpu_put_xgmi_hive(hive); 3068 goto init_failed; 3069 } 3070 3071 /* Drop the early temporary reset domain we created for device */ 3072 amdgpu_reset_put_reset_domain(adev->reset_domain); 3073 adev->reset_domain = hive->reset_domain; 3074 amdgpu_put_xgmi_hive(hive); 3075 } 3076 } 3077 } 3078 3079 r = amdgpu_device_init_schedulers(adev); 3080 if (r) 3081 goto init_failed; 3082 3083 if (adev->mman.buffer_funcs_ring->sched.ready) 3084 amdgpu_ttm_set_buffer_funcs_status(adev, true); 3085 3086 /* Don't init kfd if whole hive need to be reset during init */ 3087 if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) { 3088 kgd2kfd_init_zone_device(adev); 3089 amdgpu_amdkfd_device_init(adev); 3090 } 3091 3092 amdgpu_fru_get_product_info(adev); 3093 3094 init_failed: 3095 3096 return r; 3097 } 3098 3099 /** 3100 * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer 3101 * 3102 * @adev: amdgpu_device pointer 3103 * 3104 * Writes a reset magic value to the gart pointer in VRAM. The driver calls 3105 * this function before a GPU reset. If the value is retained after a 3106 * GPU reset, VRAM has not been lost. Some GPU resets may destroy VRAM contents. 3107 */ 3108 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev) 3109 { 3110 memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM); 3111 } 3112 3113 /** 3114 * amdgpu_device_check_vram_lost - check if vram is valid 3115 * 3116 * @adev: amdgpu_device pointer 3117 * 3118 * Checks the reset magic value written to the gart pointer in VRAM. 3119 * The driver calls this after a GPU reset to see if the contents of 3120 * VRAM is lost or now. 3121 * returns true if vram is lost, false if not. 3122 */ 3123 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev) 3124 { 3125 if (memcmp(adev->gart.ptr, adev->reset_magic, 3126 AMDGPU_RESET_MAGIC_NUM)) 3127 return true; 3128 3129 if (!amdgpu_in_reset(adev)) 3130 return false; 3131 3132 /* 3133 * For all ASICs with baco/mode1 reset, the VRAM is 3134 * always assumed to be lost. 3135 */ 3136 switch (amdgpu_asic_reset_method(adev)) { 3137 case AMD_RESET_METHOD_BACO: 3138 case AMD_RESET_METHOD_MODE1: 3139 return true; 3140 default: 3141 return false; 3142 } 3143 } 3144 3145 /** 3146 * amdgpu_device_set_cg_state - set clockgating for amdgpu device 3147 * 3148 * @adev: amdgpu_device pointer 3149 * @state: clockgating state (gate or ungate) 3150 * 3151 * The list of all the hardware IPs that make up the asic is walked and the 3152 * set_clockgating_state callbacks are run. 3153 * Late initialization pass enabling clockgating for hardware IPs. 3154 * Fini or suspend, pass disabling clockgating for hardware IPs. 3155 * Returns 0 on success, negative error code on failure. 3156 */ 3157 3158 int amdgpu_device_set_cg_state(struct amdgpu_device *adev, 3159 enum amd_clockgating_state state) 3160 { 3161 int i, j, r; 3162 3163 if (amdgpu_emu_mode == 1) 3164 return 0; 3165 3166 for (j = 0; j < adev->num_ip_blocks; j++) { 3167 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; 3168 if (!adev->ip_blocks[i].status.late_initialized) 3169 continue; 3170 /* skip CG for GFX, SDMA on S0ix */ 3171 if (adev->in_s0ix && 3172 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX || 3173 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA)) 3174 continue; 3175 /* skip CG for VCE/UVD, it's handled specially */ 3176 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && 3177 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && 3178 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && 3179 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG && 3180 adev->ip_blocks[i].version->funcs->set_clockgating_state) { 3181 /* enable clockgating to save power */ 3182 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(&adev->ip_blocks[i], 3183 state); 3184 if (r) { 3185 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n", 3186 adev->ip_blocks[i].version->funcs->name, r); 3187 return r; 3188 } 3189 } 3190 } 3191 3192 return 0; 3193 } 3194 3195 int amdgpu_device_set_pg_state(struct amdgpu_device *adev, 3196 enum amd_powergating_state state) 3197 { 3198 int i, j, r; 3199 3200 if (amdgpu_emu_mode == 1) 3201 return 0; 3202 3203 for (j = 0; j < adev->num_ip_blocks; j++) { 3204 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; 3205 if (!adev->ip_blocks[i].status.late_initialized) 3206 continue; 3207 /* skip PG for GFX, SDMA on S0ix */ 3208 if (adev->in_s0ix && 3209 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX || 3210 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA)) 3211 continue; 3212 /* skip CG for VCE/UVD, it's handled specially */ 3213 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && 3214 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && 3215 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && 3216 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG && 3217 adev->ip_blocks[i].version->funcs->set_powergating_state) { 3218 /* enable powergating to save power */ 3219 r = adev->ip_blocks[i].version->funcs->set_powergating_state(&adev->ip_blocks[i], 3220 state); 3221 if (r) { 3222 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n", 3223 adev->ip_blocks[i].version->funcs->name, r); 3224 return r; 3225 } 3226 } 3227 } 3228 return 0; 3229 } 3230 3231 static int amdgpu_device_enable_mgpu_fan_boost(void) 3232 { 3233 struct amdgpu_gpu_instance *gpu_ins; 3234 struct amdgpu_device *adev; 3235 int i, ret = 0; 3236 3237 mutex_lock(&mgpu_info.mutex); 3238 3239 /* 3240 * MGPU fan boost feature should be enabled 3241 * only when there are two or more dGPUs in 3242 * the system 3243 */ 3244 if (mgpu_info.num_dgpu < 2) 3245 goto out; 3246 3247 for (i = 0; i < mgpu_info.num_dgpu; i++) { 3248 gpu_ins = &(mgpu_info.gpu_ins[i]); 3249 adev = gpu_ins->adev; 3250 if (!(adev->flags & AMD_IS_APU) && 3251 !gpu_ins->mgpu_fan_enabled) { 3252 ret = amdgpu_dpm_enable_mgpu_fan_boost(adev); 3253 if (ret) 3254 break; 3255 3256 gpu_ins->mgpu_fan_enabled = 1; 3257 } 3258 } 3259 3260 out: 3261 mutex_unlock(&mgpu_info.mutex); 3262 3263 return ret; 3264 } 3265 3266 /** 3267 * amdgpu_device_ip_late_init - run late init for hardware IPs 3268 * 3269 * @adev: amdgpu_device pointer 3270 * 3271 * Late initialization pass for hardware IPs. The list of all the hardware 3272 * IPs that make up the asic is walked and the late_init callbacks are run. 3273 * late_init covers any special initialization that an IP requires 3274 * after all of the have been initialized or something that needs to happen 3275 * late in the init process. 3276 * Returns 0 on success, negative error code on failure. 3277 */ 3278 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev) 3279 { 3280 struct amdgpu_gpu_instance *gpu_instance; 3281 int i = 0, r; 3282 3283 for (i = 0; i < adev->num_ip_blocks; i++) { 3284 if (!adev->ip_blocks[i].status.hw) 3285 continue; 3286 if (adev->ip_blocks[i].version->funcs->late_init) { 3287 r = adev->ip_blocks[i].version->funcs->late_init(&adev->ip_blocks[i]); 3288 if (r) { 3289 DRM_ERROR("late_init of IP block <%s> failed %d\n", 3290 adev->ip_blocks[i].version->funcs->name, r); 3291 return r; 3292 } 3293 } 3294 adev->ip_blocks[i].status.late_initialized = true; 3295 } 3296 3297 r = amdgpu_ras_late_init(adev); 3298 if (r) { 3299 DRM_ERROR("amdgpu_ras_late_init failed %d", r); 3300 return r; 3301 } 3302 3303 if (!amdgpu_reset_in_recovery(adev)) 3304 amdgpu_ras_set_error_query_ready(adev, true); 3305 3306 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE); 3307 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE); 3308 3309 amdgpu_device_fill_reset_magic(adev); 3310 3311 r = amdgpu_device_enable_mgpu_fan_boost(); 3312 if (r) 3313 DRM_ERROR("enable mgpu fan boost failed (%d).\n", r); 3314 3315 /* For passthrough configuration on arcturus and aldebaran, enable special handling SBR */ 3316 if (amdgpu_passthrough(adev) && 3317 ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1) || 3318 adev->asic_type == CHIP_ALDEBARAN)) 3319 amdgpu_dpm_handle_passthrough_sbr(adev, true); 3320 3321 if (adev->gmc.xgmi.num_physical_nodes > 1) { 3322 mutex_lock(&mgpu_info.mutex); 3323 3324 /* 3325 * Reset device p-state to low as this was booted with high. 3326 * 3327 * This should be performed only after all devices from the same 3328 * hive get initialized. 3329 * 3330 * However, it's unknown how many device in the hive in advance. 3331 * As this is counted one by one during devices initializations. 3332 * 3333 * So, we wait for all XGMI interlinked devices initialized. 3334 * This may bring some delays as those devices may come from 3335 * different hives. But that should be OK. 3336 */ 3337 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) { 3338 for (i = 0; i < mgpu_info.num_gpu; i++) { 3339 gpu_instance = &(mgpu_info.gpu_ins[i]); 3340 if (gpu_instance->adev->flags & AMD_IS_APU) 3341 continue; 3342 3343 r = amdgpu_xgmi_set_pstate(gpu_instance->adev, 3344 AMDGPU_XGMI_PSTATE_MIN); 3345 if (r) { 3346 DRM_ERROR("pstate setting failed (%d).\n", r); 3347 break; 3348 } 3349 } 3350 } 3351 3352 mutex_unlock(&mgpu_info.mutex); 3353 } 3354 3355 return 0; 3356 } 3357 3358 static void amdgpu_ip_block_hw_fini(struct amdgpu_ip_block *ip_block) 3359 { 3360 int r; 3361 3362 if (!ip_block->version->funcs->hw_fini) { 3363 DRM_ERROR("hw_fini of IP block <%s> not defined\n", 3364 ip_block->version->funcs->name); 3365 } else { 3366 r = ip_block->version->funcs->hw_fini(ip_block); 3367 /* XXX handle errors */ 3368 if (r) { 3369 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n", 3370 ip_block->version->funcs->name, r); 3371 } 3372 } 3373 3374 ip_block->status.hw = false; 3375 } 3376 3377 /** 3378 * amdgpu_device_smu_fini_early - smu hw_fini wrapper 3379 * 3380 * @adev: amdgpu_device pointer 3381 * 3382 * For ASICs need to disable SMC first 3383 */ 3384 static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev) 3385 { 3386 int i; 3387 3388 if (amdgpu_ip_version(adev, GC_HWIP, 0) > IP_VERSION(9, 0, 0)) 3389 return; 3390 3391 for (i = 0; i < adev->num_ip_blocks; i++) { 3392 if (!adev->ip_blocks[i].status.hw) 3393 continue; 3394 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) { 3395 amdgpu_ip_block_hw_fini(&adev->ip_blocks[i]); 3396 break; 3397 } 3398 } 3399 } 3400 3401 static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev) 3402 { 3403 int i, r; 3404 3405 for (i = 0; i < adev->num_ip_blocks; i++) { 3406 if (!adev->ip_blocks[i].version->funcs->early_fini) 3407 continue; 3408 3409 r = adev->ip_blocks[i].version->funcs->early_fini(&adev->ip_blocks[i]); 3410 if (r) { 3411 DRM_DEBUG("early_fini of IP block <%s> failed %d\n", 3412 adev->ip_blocks[i].version->funcs->name, r); 3413 } 3414 } 3415 3416 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); 3417 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); 3418 3419 amdgpu_amdkfd_suspend(adev, false); 3420 3421 /* Workaround for ASICs need to disable SMC first */ 3422 amdgpu_device_smu_fini_early(adev); 3423 3424 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 3425 if (!adev->ip_blocks[i].status.hw) 3426 continue; 3427 3428 amdgpu_ip_block_hw_fini(&adev->ip_blocks[i]); 3429 } 3430 3431 if (amdgpu_sriov_vf(adev)) { 3432 if (amdgpu_virt_release_full_gpu(adev, false)) 3433 DRM_ERROR("failed to release exclusive mode on fini\n"); 3434 } 3435 3436 return 0; 3437 } 3438 3439 /** 3440 * amdgpu_device_ip_fini - run fini for hardware IPs 3441 * 3442 * @adev: amdgpu_device pointer 3443 * 3444 * Main teardown pass for hardware IPs. The list of all the hardware 3445 * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks 3446 * are run. hw_fini tears down the hardware associated with each IP 3447 * and sw_fini tears down any software state associated with each IP. 3448 * Returns 0 on success, negative error code on failure. 3449 */ 3450 static int amdgpu_device_ip_fini(struct amdgpu_device *adev) 3451 { 3452 int i, r; 3453 3454 if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done) 3455 amdgpu_virt_release_ras_err_handler_data(adev); 3456 3457 if (adev->gmc.xgmi.num_physical_nodes > 1) 3458 amdgpu_xgmi_remove_device(adev); 3459 3460 amdgpu_amdkfd_device_fini_sw(adev); 3461 3462 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 3463 if (!adev->ip_blocks[i].status.sw) 3464 continue; 3465 3466 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { 3467 amdgpu_ucode_free_bo(adev); 3468 amdgpu_free_static_csa(&adev->virt.csa_obj); 3469 amdgpu_device_wb_fini(adev); 3470 amdgpu_device_mem_scratch_fini(adev); 3471 amdgpu_ib_pool_fini(adev); 3472 amdgpu_seq64_fini(adev); 3473 } 3474 if (adev->ip_blocks[i].version->funcs->sw_fini) { 3475 r = adev->ip_blocks[i].version->funcs->sw_fini(&adev->ip_blocks[i]); 3476 /* XXX handle errors */ 3477 if (r) { 3478 DRM_DEBUG("sw_fini of IP block <%s> failed %d\n", 3479 adev->ip_blocks[i].version->funcs->name, r); 3480 } 3481 } 3482 adev->ip_blocks[i].status.sw = false; 3483 adev->ip_blocks[i].status.valid = false; 3484 } 3485 3486 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 3487 if (!adev->ip_blocks[i].status.late_initialized) 3488 continue; 3489 if (adev->ip_blocks[i].version->funcs->late_fini) 3490 adev->ip_blocks[i].version->funcs->late_fini(&adev->ip_blocks[i]); 3491 adev->ip_blocks[i].status.late_initialized = false; 3492 } 3493 3494 amdgpu_ras_fini(adev); 3495 3496 return 0; 3497 } 3498 3499 /** 3500 * amdgpu_device_delayed_init_work_handler - work handler for IB tests 3501 * 3502 * @work: work_struct. 3503 */ 3504 static void amdgpu_device_delayed_init_work_handler(struct work_struct *work) 3505 { 3506 struct amdgpu_device *adev = 3507 container_of(work, struct amdgpu_device, delayed_init_work.work); 3508 int r; 3509 3510 r = amdgpu_ib_ring_tests(adev); 3511 if (r) 3512 DRM_ERROR("ib ring test failed (%d).\n", r); 3513 } 3514 3515 static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work) 3516 { 3517 struct amdgpu_device *adev = 3518 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work); 3519 3520 WARN_ON_ONCE(adev->gfx.gfx_off_state); 3521 WARN_ON_ONCE(adev->gfx.gfx_off_req_count); 3522 3523 if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true, 0)) 3524 adev->gfx.gfx_off_state = true; 3525 } 3526 3527 /** 3528 * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1) 3529 * 3530 * @adev: amdgpu_device pointer 3531 * 3532 * Main suspend function for hardware IPs. The list of all the hardware 3533 * IPs that make up the asic is walked, clockgating is disabled and the 3534 * suspend callbacks are run. suspend puts the hardware and software state 3535 * in each IP into a state suitable for suspend. 3536 * Returns 0 on success, negative error code on failure. 3537 */ 3538 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev) 3539 { 3540 int i, r; 3541 3542 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); 3543 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); 3544 3545 /* 3546 * Per PMFW team's suggestion, driver needs to handle gfxoff 3547 * and df cstate features disablement for gpu reset(e.g. Mode1Reset) 3548 * scenario. Add the missing df cstate disablement here. 3549 */ 3550 if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW)) 3551 dev_warn(adev->dev, "Failed to disallow df cstate"); 3552 3553 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 3554 if (!adev->ip_blocks[i].status.valid) 3555 continue; 3556 3557 /* displays are handled separately */ 3558 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE) 3559 continue; 3560 3561 /* XXX handle errors */ 3562 r = amdgpu_ip_block_suspend(&adev->ip_blocks[i]); 3563 if (r) 3564 return r; 3565 } 3566 3567 return 0; 3568 } 3569 3570 /** 3571 * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2) 3572 * 3573 * @adev: amdgpu_device pointer 3574 * 3575 * Main suspend function for hardware IPs. The list of all the hardware 3576 * IPs that make up the asic is walked, clockgating is disabled and the 3577 * suspend callbacks are run. suspend puts the hardware and software state 3578 * in each IP into a state suitable for suspend. 3579 * Returns 0 on success, negative error code on failure. 3580 */ 3581 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev) 3582 { 3583 int i, r; 3584 3585 if (adev->in_s0ix) 3586 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D3Entry); 3587 3588 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 3589 if (!adev->ip_blocks[i].status.valid) 3590 continue; 3591 /* displays are handled in phase1 */ 3592 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) 3593 continue; 3594 /* PSP lost connection when err_event_athub occurs */ 3595 if (amdgpu_ras_intr_triggered() && 3596 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) { 3597 adev->ip_blocks[i].status.hw = false; 3598 continue; 3599 } 3600 3601 /* skip unnecessary suspend if we do not initialize them yet */ 3602 if (!amdgpu_ip_member_of_hwini( 3603 adev, adev->ip_blocks[i].version->type)) 3604 continue; 3605 3606 /* skip suspend of gfx/mes and psp for S0ix 3607 * gfx is in gfxoff state, so on resume it will exit gfxoff just 3608 * like at runtime. PSP is also part of the always on hardware 3609 * so no need to suspend it. 3610 */ 3611 if (adev->in_s0ix && 3612 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP || 3613 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX || 3614 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_MES)) 3615 continue; 3616 3617 /* SDMA 5.x+ is part of GFX power domain so it's covered by GFXOFF */ 3618 if (adev->in_s0ix && 3619 (amdgpu_ip_version(adev, SDMA0_HWIP, 0) >= 3620 IP_VERSION(5, 0, 0)) && 3621 (adev->ip_blocks[i].version->type == 3622 AMD_IP_BLOCK_TYPE_SDMA)) 3623 continue; 3624 3625 /* Once swPSP provides the IMU, RLC FW binaries to TOS during cold-boot. 3626 * These are in TMR, hence are expected to be reused by PSP-TOS to reload 3627 * from this location and RLC Autoload automatically also gets loaded 3628 * from here based on PMFW -> PSP message during re-init sequence. 3629 * Therefore, the psp suspend & resume should be skipped to avoid destroy 3630 * the TMR and reload FWs again for IMU enabled APU ASICs. 3631 */ 3632 if (amdgpu_in_reset(adev) && 3633 (adev->flags & AMD_IS_APU) && adev->gfx.imu.funcs && 3634 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) 3635 continue; 3636 3637 /* XXX handle errors */ 3638 r = amdgpu_ip_block_suspend(&adev->ip_blocks[i]); 3639 adev->ip_blocks[i].status.hw = false; 3640 3641 /* handle putting the SMC in the appropriate state */ 3642 if (!amdgpu_sriov_vf(adev)) { 3643 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) { 3644 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state); 3645 if (r) { 3646 DRM_ERROR("SMC failed to set mp1 state %d, %d\n", 3647 adev->mp1_state, r); 3648 return r; 3649 } 3650 } 3651 } 3652 } 3653 3654 return 0; 3655 } 3656 3657 /** 3658 * amdgpu_device_ip_suspend - run suspend for hardware IPs 3659 * 3660 * @adev: amdgpu_device pointer 3661 * 3662 * Main suspend function for hardware IPs. The list of all the hardware 3663 * IPs that make up the asic is walked, clockgating is disabled and the 3664 * suspend callbacks are run. suspend puts the hardware and software state 3665 * in each IP into a state suitable for suspend. 3666 * Returns 0 on success, negative error code on failure. 3667 */ 3668 int amdgpu_device_ip_suspend(struct amdgpu_device *adev) 3669 { 3670 int r; 3671 3672 if (amdgpu_sriov_vf(adev)) { 3673 amdgpu_virt_fini_data_exchange(adev); 3674 amdgpu_virt_request_full_gpu(adev, false); 3675 } 3676 3677 amdgpu_ttm_set_buffer_funcs_status(adev, false); 3678 3679 r = amdgpu_device_ip_suspend_phase1(adev); 3680 if (r) 3681 return r; 3682 r = amdgpu_device_ip_suspend_phase2(adev); 3683 3684 if (amdgpu_sriov_vf(adev)) 3685 amdgpu_virt_release_full_gpu(adev, false); 3686 3687 return r; 3688 } 3689 3690 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev) 3691 { 3692 int i, r; 3693 3694 static enum amd_ip_block_type ip_order[] = { 3695 AMD_IP_BLOCK_TYPE_COMMON, 3696 AMD_IP_BLOCK_TYPE_GMC, 3697 AMD_IP_BLOCK_TYPE_PSP, 3698 AMD_IP_BLOCK_TYPE_IH, 3699 }; 3700 3701 for (i = 0; i < adev->num_ip_blocks; i++) { 3702 int j; 3703 struct amdgpu_ip_block *block; 3704 3705 block = &adev->ip_blocks[i]; 3706 block->status.hw = false; 3707 3708 for (j = 0; j < ARRAY_SIZE(ip_order); j++) { 3709 3710 if (block->version->type != ip_order[j] || 3711 !block->status.valid) 3712 continue; 3713 3714 r = block->version->funcs->hw_init(&adev->ip_blocks[i]); 3715 if (r) { 3716 dev_err(adev->dev, "RE-INIT-early: %s failed\n", 3717 block->version->funcs->name); 3718 return r; 3719 } 3720 block->status.hw = true; 3721 } 3722 } 3723 3724 return 0; 3725 } 3726 3727 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev) 3728 { 3729 struct amdgpu_ip_block *block; 3730 int i, r = 0; 3731 3732 static enum amd_ip_block_type ip_order[] = { 3733 AMD_IP_BLOCK_TYPE_SMC, 3734 AMD_IP_BLOCK_TYPE_DCE, 3735 AMD_IP_BLOCK_TYPE_GFX, 3736 AMD_IP_BLOCK_TYPE_SDMA, 3737 AMD_IP_BLOCK_TYPE_MES, 3738 AMD_IP_BLOCK_TYPE_UVD, 3739 AMD_IP_BLOCK_TYPE_VCE, 3740 AMD_IP_BLOCK_TYPE_VCN, 3741 AMD_IP_BLOCK_TYPE_JPEG 3742 }; 3743 3744 for (i = 0; i < ARRAY_SIZE(ip_order); i++) { 3745 block = amdgpu_device_ip_get_ip_block(adev, ip_order[i]); 3746 3747 if (!block) 3748 continue; 3749 3750 if (block->status.valid && !block->status.hw) { 3751 if (block->version->type == AMD_IP_BLOCK_TYPE_SMC) { 3752 r = amdgpu_ip_block_resume(block); 3753 } else { 3754 r = block->version->funcs->hw_init(block); 3755 } 3756 3757 if (r) { 3758 dev_err(adev->dev, "RE-INIT-late: %s failed\n", 3759 block->version->funcs->name); 3760 break; 3761 } 3762 block->status.hw = true; 3763 } 3764 } 3765 3766 return r; 3767 } 3768 3769 /** 3770 * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs 3771 * 3772 * @adev: amdgpu_device pointer 3773 * 3774 * First resume function for hardware IPs. The list of all the hardware 3775 * IPs that make up the asic is walked and the resume callbacks are run for 3776 * COMMON, GMC, and IH. resume puts the hardware into a functional state 3777 * after a suspend and updates the software state as necessary. This 3778 * function is also used for restoring the GPU after a GPU reset. 3779 * Returns 0 on success, negative error code on failure. 3780 */ 3781 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev) 3782 { 3783 int i, r; 3784 3785 for (i = 0; i < adev->num_ip_blocks; i++) { 3786 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw) 3787 continue; 3788 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || 3789 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || 3790 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH || 3791 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) { 3792 3793 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]); 3794 if (r) 3795 return r; 3796 } 3797 } 3798 3799 return 0; 3800 } 3801 3802 /** 3803 * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs 3804 * 3805 * @adev: amdgpu_device pointer 3806 * 3807 * Second resume function for hardware IPs. The list of all the hardware 3808 * IPs that make up the asic is walked and the resume callbacks are run for 3809 * all blocks except COMMON, GMC, and IH. resume puts the hardware into a 3810 * functional state after a suspend and updates the software state as 3811 * necessary. This function is also used for restoring the GPU after a GPU 3812 * reset. 3813 * Returns 0 on success, negative error code on failure. 3814 */ 3815 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev) 3816 { 3817 int i, r; 3818 3819 for (i = 0; i < adev->num_ip_blocks; i++) { 3820 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw) 3821 continue; 3822 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || 3823 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || 3824 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH || 3825 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE || 3826 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) 3827 continue; 3828 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]); 3829 if (r) 3830 return r; 3831 } 3832 3833 return 0; 3834 } 3835 3836 /** 3837 * amdgpu_device_ip_resume_phase3 - run resume for hardware IPs 3838 * 3839 * @adev: amdgpu_device pointer 3840 * 3841 * Third resume function for hardware IPs. The list of all the hardware 3842 * IPs that make up the asic is walked and the resume callbacks are run for 3843 * all DCE. resume puts the hardware into a functional state after a suspend 3844 * and updates the software state as necessary. This function is also used 3845 * for restoring the GPU after a GPU reset. 3846 * 3847 * Returns 0 on success, negative error code on failure. 3848 */ 3849 static int amdgpu_device_ip_resume_phase3(struct amdgpu_device *adev) 3850 { 3851 int i, r; 3852 3853 for (i = 0; i < adev->num_ip_blocks; i++) { 3854 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw) 3855 continue; 3856 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) { 3857 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]); 3858 if (r) 3859 return r; 3860 } 3861 } 3862 3863 return 0; 3864 } 3865 3866 /** 3867 * amdgpu_device_ip_resume - run resume for hardware IPs 3868 * 3869 * @adev: amdgpu_device pointer 3870 * 3871 * Main resume function for hardware IPs. The hardware IPs 3872 * are split into two resume functions because they are 3873 * also used in recovering from a GPU reset and some additional 3874 * steps need to be take between them. In this case (S3/S4) they are 3875 * run sequentially. 3876 * Returns 0 on success, negative error code on failure. 3877 */ 3878 static int amdgpu_device_ip_resume(struct amdgpu_device *adev) 3879 { 3880 int r; 3881 3882 r = amdgpu_device_ip_resume_phase1(adev); 3883 if (r) 3884 return r; 3885 3886 r = amdgpu_device_fw_loading(adev); 3887 if (r) 3888 return r; 3889 3890 r = amdgpu_device_ip_resume_phase2(adev); 3891 3892 if (adev->mman.buffer_funcs_ring->sched.ready) 3893 amdgpu_ttm_set_buffer_funcs_status(adev, true); 3894 3895 if (r) 3896 return r; 3897 3898 amdgpu_fence_driver_hw_init(adev); 3899 3900 r = amdgpu_device_ip_resume_phase3(adev); 3901 3902 return r; 3903 } 3904 3905 /** 3906 * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV 3907 * 3908 * @adev: amdgpu_device pointer 3909 * 3910 * Query the VBIOS data tables to determine if the board supports SR-IOV. 3911 */ 3912 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev) 3913 { 3914 if (amdgpu_sriov_vf(adev)) { 3915 if (adev->is_atom_fw) { 3916 if (amdgpu_atomfirmware_gpu_virtualization_supported(adev)) 3917 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; 3918 } else { 3919 if (amdgpu_atombios_has_gpu_virtualization_table(adev)) 3920 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; 3921 } 3922 3923 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS)) 3924 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0); 3925 } 3926 } 3927 3928 /** 3929 * amdgpu_device_asic_has_dc_support - determine if DC supports the asic 3930 * 3931 * @asic_type: AMD asic type 3932 * 3933 * Check if there is DC (new modesetting infrastructre) support for an asic. 3934 * returns true if DC has support, false if not. 3935 */ 3936 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type) 3937 { 3938 switch (asic_type) { 3939 #ifdef CONFIG_DRM_AMDGPU_SI 3940 case CHIP_HAINAN: 3941 #endif 3942 case CHIP_TOPAZ: 3943 /* chips with no display hardware */ 3944 return false; 3945 #if defined(CONFIG_DRM_AMD_DC) 3946 case CHIP_TAHITI: 3947 case CHIP_PITCAIRN: 3948 case CHIP_VERDE: 3949 case CHIP_OLAND: 3950 /* 3951 * We have systems in the wild with these ASICs that require 3952 * LVDS and VGA support which is not supported with DC. 3953 * 3954 * Fallback to the non-DC driver here by default so as not to 3955 * cause regressions. 3956 */ 3957 #if defined(CONFIG_DRM_AMD_DC_SI) 3958 return amdgpu_dc > 0; 3959 #else 3960 return false; 3961 #endif 3962 case CHIP_BONAIRE: 3963 case CHIP_KAVERI: 3964 case CHIP_KABINI: 3965 case CHIP_MULLINS: 3966 /* 3967 * We have systems in the wild with these ASICs that require 3968 * VGA support which is not supported with DC. 3969 * 3970 * Fallback to the non-DC driver here by default so as not to 3971 * cause regressions. 3972 */ 3973 return amdgpu_dc > 0; 3974 default: 3975 return amdgpu_dc != 0; 3976 #else 3977 default: 3978 if (amdgpu_dc > 0) 3979 DRM_INFO_ONCE("Display Core has been requested via kernel parameter but isn't supported by ASIC, ignoring\n"); 3980 return false; 3981 #endif 3982 } 3983 } 3984 3985 /** 3986 * amdgpu_device_has_dc_support - check if dc is supported 3987 * 3988 * @adev: amdgpu_device pointer 3989 * 3990 * Returns true for supported, false for not supported 3991 */ 3992 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev) 3993 { 3994 if (adev->enable_virtual_display || 3995 (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK)) 3996 return false; 3997 3998 return amdgpu_device_asic_has_dc_support(adev->asic_type); 3999 } 4000 4001 static void amdgpu_device_xgmi_reset_func(struct work_struct *__work) 4002 { 4003 struct amdgpu_device *adev = 4004 container_of(__work, struct amdgpu_device, xgmi_reset_work); 4005 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 4006 4007 /* It's a bug to not have a hive within this function */ 4008 if (WARN_ON(!hive)) 4009 return; 4010 4011 /* 4012 * Use task barrier to synchronize all xgmi reset works across the 4013 * hive. task_barrier_enter and task_barrier_exit will block 4014 * until all the threads running the xgmi reset works reach 4015 * those points. task_barrier_full will do both blocks. 4016 */ 4017 if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) { 4018 4019 task_barrier_enter(&hive->tb); 4020 adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev)); 4021 4022 if (adev->asic_reset_res) 4023 goto fail; 4024 4025 task_barrier_exit(&hive->tb); 4026 adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev)); 4027 4028 if (adev->asic_reset_res) 4029 goto fail; 4030 4031 amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__MMHUB); 4032 } else { 4033 4034 task_barrier_full(&hive->tb); 4035 adev->asic_reset_res = amdgpu_asic_reset(adev); 4036 } 4037 4038 fail: 4039 if (adev->asic_reset_res) 4040 DRM_WARN("ASIC reset failed with error, %d for drm dev, %s", 4041 adev->asic_reset_res, adev_to_drm(adev)->unique); 4042 amdgpu_put_xgmi_hive(hive); 4043 } 4044 4045 static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev) 4046 { 4047 char *input = amdgpu_lockup_timeout; 4048 char *timeout_setting = NULL; 4049 int index = 0; 4050 long timeout; 4051 int ret = 0; 4052 4053 /* 4054 * By default timeout for non compute jobs is 10000 4055 * and 60000 for compute jobs. 4056 * In SR-IOV or passthrough mode, timeout for compute 4057 * jobs are 60000 by default. 4058 */ 4059 adev->gfx_timeout = msecs_to_jiffies(10000); 4060 adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout; 4061 if (amdgpu_sriov_vf(adev)) 4062 adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ? 4063 msecs_to_jiffies(60000) : msecs_to_jiffies(10000); 4064 else 4065 adev->compute_timeout = msecs_to_jiffies(60000); 4066 4067 if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) { 4068 while ((timeout_setting = strsep(&input, ",")) && 4069 strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) { 4070 ret = kstrtol(timeout_setting, 0, &timeout); 4071 if (ret) 4072 return ret; 4073 4074 if (timeout == 0) { 4075 index++; 4076 continue; 4077 } else if (timeout < 0) { 4078 timeout = MAX_SCHEDULE_TIMEOUT; 4079 dev_warn(adev->dev, "lockup timeout disabled"); 4080 add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK); 4081 } else { 4082 timeout = msecs_to_jiffies(timeout); 4083 } 4084 4085 switch (index++) { 4086 case 0: 4087 adev->gfx_timeout = timeout; 4088 break; 4089 case 1: 4090 adev->compute_timeout = timeout; 4091 break; 4092 case 2: 4093 adev->sdma_timeout = timeout; 4094 break; 4095 case 3: 4096 adev->video_timeout = timeout; 4097 break; 4098 default: 4099 break; 4100 } 4101 } 4102 /* 4103 * There is only one value specified and 4104 * it should apply to all non-compute jobs. 4105 */ 4106 if (index == 1) { 4107 adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout; 4108 if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev)) 4109 adev->compute_timeout = adev->gfx_timeout; 4110 } 4111 } 4112 4113 return ret; 4114 } 4115 4116 /** 4117 * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU 4118 * 4119 * @adev: amdgpu_device pointer 4120 * 4121 * RAM direct mapped to GPU if IOMMU is not enabled or is pass through mode 4122 */ 4123 static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev) 4124 { 4125 struct iommu_domain *domain; 4126 4127 domain = iommu_get_domain_for_dev(adev->dev); 4128 if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY) 4129 adev->ram_is_direct_mapped = true; 4130 } 4131 4132 #if defined(CONFIG_HSA_AMD_P2P) 4133 /** 4134 * amdgpu_device_check_iommu_remap - Check if DMA remapping is enabled. 4135 * 4136 * @adev: amdgpu_device pointer 4137 * 4138 * return if IOMMU remapping bar address 4139 */ 4140 static bool amdgpu_device_check_iommu_remap(struct amdgpu_device *adev) 4141 { 4142 struct iommu_domain *domain; 4143 4144 domain = iommu_get_domain_for_dev(adev->dev); 4145 if (domain && (domain->type == IOMMU_DOMAIN_DMA || 4146 domain->type == IOMMU_DOMAIN_DMA_FQ)) 4147 return true; 4148 4149 return false; 4150 } 4151 #endif 4152 4153 static const struct attribute *amdgpu_dev_attributes[] = { 4154 &dev_attr_pcie_replay_count.attr, 4155 NULL 4156 }; 4157 4158 static void amdgpu_device_set_mcbp(struct amdgpu_device *adev) 4159 { 4160 if (amdgpu_mcbp == 1) 4161 adev->gfx.mcbp = true; 4162 else if (amdgpu_mcbp == 0) 4163 adev->gfx.mcbp = false; 4164 4165 if (amdgpu_sriov_vf(adev)) 4166 adev->gfx.mcbp = true; 4167 4168 if (adev->gfx.mcbp) 4169 DRM_INFO("MCBP is enabled\n"); 4170 } 4171 4172 /** 4173 * amdgpu_device_init - initialize the driver 4174 * 4175 * @adev: amdgpu_device pointer 4176 * @flags: driver flags 4177 * 4178 * Initializes the driver info and hw (all asics). 4179 * Returns 0 for success or an error on failure. 4180 * Called at driver startup. 4181 */ 4182 int amdgpu_device_init(struct amdgpu_device *adev, 4183 uint32_t flags) 4184 { 4185 struct drm_device *ddev = adev_to_drm(adev); 4186 struct pci_dev *pdev = adev->pdev; 4187 int r, i; 4188 bool px = false; 4189 u32 max_MBps; 4190 int tmp; 4191 4192 adev->shutdown = false; 4193 adev->flags = flags; 4194 4195 if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST) 4196 adev->asic_type = amdgpu_force_asic_type; 4197 else 4198 adev->asic_type = flags & AMD_ASIC_MASK; 4199 4200 adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT; 4201 if (amdgpu_emu_mode == 1) 4202 adev->usec_timeout *= 10; 4203 adev->gmc.gart_size = 512 * 1024 * 1024; 4204 adev->accel_working = false; 4205 adev->num_rings = 0; 4206 RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub()); 4207 adev->mman.buffer_funcs = NULL; 4208 adev->mman.buffer_funcs_ring = NULL; 4209 adev->vm_manager.vm_pte_funcs = NULL; 4210 adev->vm_manager.vm_pte_num_scheds = 0; 4211 adev->gmc.gmc_funcs = NULL; 4212 adev->harvest_ip_mask = 0x0; 4213 adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS); 4214 bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 4215 4216 adev->smc_rreg = &amdgpu_invalid_rreg; 4217 adev->smc_wreg = &amdgpu_invalid_wreg; 4218 adev->pcie_rreg = &amdgpu_invalid_rreg; 4219 adev->pcie_wreg = &amdgpu_invalid_wreg; 4220 adev->pcie_rreg_ext = &amdgpu_invalid_rreg_ext; 4221 adev->pcie_wreg_ext = &amdgpu_invalid_wreg_ext; 4222 adev->pciep_rreg = &amdgpu_invalid_rreg; 4223 adev->pciep_wreg = &amdgpu_invalid_wreg; 4224 adev->pcie_rreg64 = &amdgpu_invalid_rreg64; 4225 adev->pcie_wreg64 = &amdgpu_invalid_wreg64; 4226 adev->pcie_rreg64_ext = &amdgpu_invalid_rreg64_ext; 4227 adev->pcie_wreg64_ext = &amdgpu_invalid_wreg64_ext; 4228 adev->uvd_ctx_rreg = &amdgpu_invalid_rreg; 4229 adev->uvd_ctx_wreg = &amdgpu_invalid_wreg; 4230 adev->didt_rreg = &amdgpu_invalid_rreg; 4231 adev->didt_wreg = &amdgpu_invalid_wreg; 4232 adev->gc_cac_rreg = &amdgpu_invalid_rreg; 4233 adev->gc_cac_wreg = &amdgpu_invalid_wreg; 4234 adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg; 4235 adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg; 4236 4237 DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n", 4238 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device, 4239 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision); 4240 4241 /* mutex initialization are all done here so we 4242 * can recall function without having locking issues 4243 */ 4244 mutex_init(&adev->firmware.mutex); 4245 mutex_init(&adev->pm.mutex); 4246 mutex_init(&adev->gfx.gpu_clock_mutex); 4247 mutex_init(&adev->srbm_mutex); 4248 mutex_init(&adev->gfx.pipe_reserve_mutex); 4249 mutex_init(&adev->gfx.gfx_off_mutex); 4250 mutex_init(&adev->gfx.partition_mutex); 4251 mutex_init(&adev->grbm_idx_mutex); 4252 mutex_init(&adev->mn_lock); 4253 mutex_init(&adev->virt.vf_errors.lock); 4254 mutex_init(&adev->virt.rlcg_reg_lock); 4255 hash_init(adev->mn_hash); 4256 mutex_init(&adev->psp.mutex); 4257 mutex_init(&adev->notifier_lock); 4258 mutex_init(&adev->pm.stable_pstate_ctx_lock); 4259 mutex_init(&adev->benchmark_mutex); 4260 mutex_init(&adev->gfx.reset_sem_mutex); 4261 /* Initialize the mutex for cleaner shader isolation between GFX and compute processes */ 4262 mutex_init(&adev->enforce_isolation_mutex); 4263 mutex_init(&adev->gfx.kfd_sch_mutex); 4264 4265 amdgpu_device_init_apu_flags(adev); 4266 4267 r = amdgpu_device_check_arguments(adev); 4268 if (r) 4269 return r; 4270 4271 spin_lock_init(&adev->mmio_idx_lock); 4272 spin_lock_init(&adev->smc_idx_lock); 4273 spin_lock_init(&adev->pcie_idx_lock); 4274 spin_lock_init(&adev->uvd_ctx_idx_lock); 4275 spin_lock_init(&adev->didt_idx_lock); 4276 spin_lock_init(&adev->gc_cac_idx_lock); 4277 spin_lock_init(&adev->se_cac_idx_lock); 4278 spin_lock_init(&adev->audio_endpt_idx_lock); 4279 spin_lock_init(&adev->mm_stats.lock); 4280 spin_lock_init(&adev->wb.lock); 4281 4282 INIT_LIST_HEAD(&adev->reset_list); 4283 4284 INIT_LIST_HEAD(&adev->ras_list); 4285 4286 INIT_LIST_HEAD(&adev->pm.od_kobj_list); 4287 4288 INIT_DELAYED_WORK(&adev->delayed_init_work, 4289 amdgpu_device_delayed_init_work_handler); 4290 INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work, 4291 amdgpu_device_delay_enable_gfx_off); 4292 /* 4293 * Initialize the enforce_isolation work structures for each XCP 4294 * partition. This work handler is responsible for enforcing shader 4295 * isolation on AMD GPUs. It counts the number of emitted fences for 4296 * each GFX and compute ring. If there are any fences, it schedules 4297 * the `enforce_isolation_work` to be run after a delay. If there are 4298 * no fences, it signals the Kernel Fusion Driver (KFD) to resume the 4299 * runqueue. 4300 */ 4301 for (i = 0; i < MAX_XCP; i++) { 4302 INIT_DELAYED_WORK(&adev->gfx.enforce_isolation[i].work, 4303 amdgpu_gfx_enforce_isolation_handler); 4304 adev->gfx.enforce_isolation[i].adev = adev; 4305 adev->gfx.enforce_isolation[i].xcp_id = i; 4306 } 4307 4308 INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func); 4309 4310 adev->gfx.gfx_off_req_count = 1; 4311 adev->gfx.gfx_off_residency = 0; 4312 adev->gfx.gfx_off_entrycount = 0; 4313 adev->pm.ac_power = power_supply_is_system_supplied() > 0; 4314 4315 atomic_set(&adev->throttling_logging_enabled, 1); 4316 /* 4317 * If throttling continues, logging will be performed every minute 4318 * to avoid log flooding. "-1" is subtracted since the thermal 4319 * throttling interrupt comes every second. Thus, the total logging 4320 * interval is 59 seconds(retelimited printk interval) + 1(waiting 4321 * for throttling interrupt) = 60 seconds. 4322 */ 4323 ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1); 4324 ratelimit_state_init(&adev->virt.ras_telemetry_rs, 5 * HZ, 1); 4325 4326 ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE); 4327 ratelimit_set_flags(&adev->virt.ras_telemetry_rs, RATELIMIT_MSG_ON_RELEASE); 4328 4329 /* Registers mapping */ 4330 /* TODO: block userspace mapping of io register */ 4331 if (adev->asic_type >= CHIP_BONAIRE) { 4332 adev->rmmio_base = pci_resource_start(adev->pdev, 5); 4333 adev->rmmio_size = pci_resource_len(adev->pdev, 5); 4334 } else { 4335 adev->rmmio_base = pci_resource_start(adev->pdev, 2); 4336 adev->rmmio_size = pci_resource_len(adev->pdev, 2); 4337 } 4338 4339 for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++) 4340 atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN); 4341 4342 adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size); 4343 if (!adev->rmmio) 4344 return -ENOMEM; 4345 4346 DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base); 4347 DRM_INFO("register mmio size: %u\n", (unsigned int)adev->rmmio_size); 4348 4349 /* 4350 * Reset domain needs to be present early, before XGMI hive discovered 4351 * (if any) and initialized to use reset sem and in_gpu reset flag 4352 * early on during init and before calling to RREG32. 4353 */ 4354 adev->reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "amdgpu-reset-dev"); 4355 if (!adev->reset_domain) 4356 return -ENOMEM; 4357 4358 /* detect hw virtualization here */ 4359 amdgpu_detect_virtualization(adev); 4360 4361 amdgpu_device_get_pcie_info(adev); 4362 4363 r = amdgpu_device_get_job_timeout_settings(adev); 4364 if (r) { 4365 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n"); 4366 return r; 4367 } 4368 4369 amdgpu_device_set_mcbp(adev); 4370 4371 /* 4372 * By default, use default mode where all blocks are expected to be 4373 * initialized. At present a 'swinit' of blocks is required to be 4374 * completed before the need for a different level is detected. 4375 */ 4376 amdgpu_set_init_level(adev, AMDGPU_INIT_LEVEL_DEFAULT); 4377 /* early init functions */ 4378 r = amdgpu_device_ip_early_init(adev); 4379 if (r) 4380 return r; 4381 4382 /* Get rid of things like offb */ 4383 r = aperture_remove_conflicting_pci_devices(adev->pdev, amdgpu_kms_driver.name); 4384 if (r) 4385 return r; 4386 4387 /* Enable TMZ based on IP_VERSION */ 4388 amdgpu_gmc_tmz_set(adev); 4389 4390 if (amdgpu_sriov_vf(adev) && 4391 amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 3, 0)) 4392 /* VF MMIO access (except mailbox range) from CPU 4393 * will be blocked during sriov runtime 4394 */ 4395 adev->virt.caps |= AMDGPU_VF_MMIO_ACCESS_PROTECT; 4396 4397 amdgpu_gmc_noretry_set(adev); 4398 /* Need to get xgmi info early to decide the reset behavior*/ 4399 if (adev->gmc.xgmi.supported) { 4400 r = adev->gfxhub.funcs->get_xgmi_info(adev); 4401 if (r) 4402 return r; 4403 } 4404 4405 /* enable PCIE atomic ops */ 4406 if (amdgpu_sriov_vf(adev)) { 4407 if (adev->virt.fw_reserve.p_pf2vf) 4408 adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *) 4409 adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags == 4410 (PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64); 4411 /* APUs w/ gfx9 onwards doesn't reply on PCIe atomics, rather it is a 4412 * internal path natively support atomics, set have_atomics_support to true. 4413 */ 4414 } else if ((adev->flags & AMD_IS_APU) && 4415 (amdgpu_ip_version(adev, GC_HWIP, 0) > 4416 IP_VERSION(9, 0, 0))) { 4417 adev->have_atomics_support = true; 4418 } else { 4419 adev->have_atomics_support = 4420 !pci_enable_atomic_ops_to_root(adev->pdev, 4421 PCI_EXP_DEVCAP2_ATOMIC_COMP32 | 4422 PCI_EXP_DEVCAP2_ATOMIC_COMP64); 4423 } 4424 4425 if (!adev->have_atomics_support) 4426 dev_info(adev->dev, "PCIE atomic ops is not supported\n"); 4427 4428 /* doorbell bar mapping and doorbell index init*/ 4429 amdgpu_doorbell_init(adev); 4430 4431 if (amdgpu_emu_mode == 1) { 4432 /* post the asic on emulation mode */ 4433 emu_soc_asic_init(adev); 4434 goto fence_driver_init; 4435 } 4436 4437 amdgpu_reset_init(adev); 4438 4439 /* detect if we are with an SRIOV vbios */ 4440 if (adev->bios) 4441 amdgpu_device_detect_sriov_bios(adev); 4442 4443 /* check if we need to reset the asic 4444 * E.g., driver was not cleanly unloaded previously, etc. 4445 */ 4446 if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) { 4447 if (adev->gmc.xgmi.num_physical_nodes) { 4448 dev_info(adev->dev, "Pending hive reset.\n"); 4449 amdgpu_set_init_level(adev, 4450 AMDGPU_INIT_LEVEL_MINIMAL_XGMI); 4451 } else if (amdgpu_ip_version(adev, MP1_HWIP, 0) == IP_VERSION(13, 0, 10) && 4452 !amdgpu_device_has_display_hardware(adev)) { 4453 r = psp_gpu_reset(adev); 4454 } else { 4455 tmp = amdgpu_reset_method; 4456 /* It should do a default reset when loading or reloading the driver, 4457 * regardless of the module parameter reset_method. 4458 */ 4459 amdgpu_reset_method = AMD_RESET_METHOD_NONE; 4460 r = amdgpu_asic_reset(adev); 4461 amdgpu_reset_method = tmp; 4462 } 4463 4464 if (r) { 4465 dev_err(adev->dev, "asic reset on init failed\n"); 4466 goto failed; 4467 } 4468 } 4469 4470 /* Post card if necessary */ 4471 if (amdgpu_device_need_post(adev)) { 4472 if (!adev->bios) { 4473 dev_err(adev->dev, "no vBIOS found\n"); 4474 r = -EINVAL; 4475 goto failed; 4476 } 4477 DRM_INFO("GPU posting now...\n"); 4478 r = amdgpu_device_asic_init(adev); 4479 if (r) { 4480 dev_err(adev->dev, "gpu post error!\n"); 4481 goto failed; 4482 } 4483 } 4484 4485 if (adev->bios) { 4486 if (adev->is_atom_fw) { 4487 /* Initialize clocks */ 4488 r = amdgpu_atomfirmware_get_clock_info(adev); 4489 if (r) { 4490 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n"); 4491 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0); 4492 goto failed; 4493 } 4494 } else { 4495 /* Initialize clocks */ 4496 r = amdgpu_atombios_get_clock_info(adev); 4497 if (r) { 4498 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n"); 4499 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0); 4500 goto failed; 4501 } 4502 /* init i2c buses */ 4503 amdgpu_i2c_init(adev); 4504 } 4505 } 4506 4507 fence_driver_init: 4508 /* Fence driver */ 4509 r = amdgpu_fence_driver_sw_init(adev); 4510 if (r) { 4511 dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n"); 4512 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0); 4513 goto failed; 4514 } 4515 4516 /* init the mode config */ 4517 drm_mode_config_init(adev_to_drm(adev)); 4518 4519 r = amdgpu_device_ip_init(adev); 4520 if (r) { 4521 dev_err(adev->dev, "amdgpu_device_ip_init failed\n"); 4522 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0); 4523 goto release_ras_con; 4524 } 4525 4526 amdgpu_fence_driver_hw_init(adev); 4527 4528 dev_info(adev->dev, 4529 "SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n", 4530 adev->gfx.config.max_shader_engines, 4531 adev->gfx.config.max_sh_per_se, 4532 adev->gfx.config.max_cu_per_sh, 4533 adev->gfx.cu_info.number); 4534 4535 adev->accel_working = true; 4536 4537 amdgpu_vm_check_compute_bug(adev); 4538 4539 /* Initialize the buffer migration limit. */ 4540 if (amdgpu_moverate >= 0) 4541 max_MBps = amdgpu_moverate; 4542 else 4543 max_MBps = 8; /* Allow 8 MB/s. */ 4544 /* Get a log2 for easy divisions. */ 4545 adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps)); 4546 4547 /* 4548 * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost. 4549 * Otherwise the mgpu fan boost feature will be skipped due to the 4550 * gpu instance is counted less. 4551 */ 4552 amdgpu_register_gpu_instance(adev); 4553 4554 /* enable clockgating, etc. after ib tests, etc. since some blocks require 4555 * explicit gating rather than handling it automatically. 4556 */ 4557 if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) { 4558 r = amdgpu_device_ip_late_init(adev); 4559 if (r) { 4560 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n"); 4561 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r); 4562 goto release_ras_con; 4563 } 4564 /* must succeed. */ 4565 amdgpu_ras_resume(adev); 4566 queue_delayed_work(system_wq, &adev->delayed_init_work, 4567 msecs_to_jiffies(AMDGPU_RESUME_MS)); 4568 } 4569 4570 if (amdgpu_sriov_vf(adev)) { 4571 amdgpu_virt_release_full_gpu(adev, true); 4572 flush_delayed_work(&adev->delayed_init_work); 4573 } 4574 4575 /* 4576 * Place those sysfs registering after `late_init`. As some of those 4577 * operations performed in `late_init` might affect the sysfs 4578 * interfaces creating. 4579 */ 4580 r = amdgpu_atombios_sysfs_init(adev); 4581 if (r) 4582 drm_err(&adev->ddev, 4583 "registering atombios sysfs failed (%d).\n", r); 4584 4585 r = amdgpu_pm_sysfs_init(adev); 4586 if (r) 4587 DRM_ERROR("registering pm sysfs failed (%d).\n", r); 4588 4589 r = amdgpu_ucode_sysfs_init(adev); 4590 if (r) { 4591 adev->ucode_sysfs_en = false; 4592 DRM_ERROR("Creating firmware sysfs failed (%d).\n", r); 4593 } else 4594 adev->ucode_sysfs_en = true; 4595 4596 r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes); 4597 if (r) 4598 dev_err(adev->dev, "Could not create amdgpu device attr\n"); 4599 4600 r = devm_device_add_group(adev->dev, &amdgpu_board_attrs_group); 4601 if (r) 4602 dev_err(adev->dev, 4603 "Could not create amdgpu board attributes\n"); 4604 4605 amdgpu_fru_sysfs_init(adev); 4606 amdgpu_reg_state_sysfs_init(adev); 4607 amdgpu_xcp_cfg_sysfs_init(adev); 4608 4609 if (IS_ENABLED(CONFIG_PERF_EVENTS)) 4610 r = amdgpu_pmu_init(adev); 4611 if (r) 4612 dev_err(adev->dev, "amdgpu_pmu_init failed\n"); 4613 4614 /* Have stored pci confspace at hand for restore in sudden PCI error */ 4615 if (amdgpu_device_cache_pci_state(adev->pdev)) 4616 pci_restore_state(pdev); 4617 4618 /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */ 4619 /* this will fail for cards that aren't VGA class devices, just 4620 * ignore it 4621 */ 4622 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) 4623 vga_client_register(adev->pdev, amdgpu_device_vga_set_decode); 4624 4625 px = amdgpu_device_supports_px(ddev); 4626 4627 if (px || (!dev_is_removable(&adev->pdev->dev) && 4628 apple_gmux_detect(NULL, NULL))) 4629 vga_switcheroo_register_client(adev->pdev, 4630 &amdgpu_switcheroo_ops, px); 4631 4632 if (px) 4633 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain); 4634 4635 if (adev->init_lvl->level == AMDGPU_INIT_LEVEL_MINIMAL_XGMI) 4636 amdgpu_xgmi_reset_on_init(adev); 4637 4638 amdgpu_device_check_iommu_direct_map(adev); 4639 4640 adev->pm_nb.notifier_call = amdgpu_device_pm_notifier; 4641 r = register_pm_notifier(&adev->pm_nb); 4642 if (r) 4643 goto failed; 4644 4645 return 0; 4646 4647 release_ras_con: 4648 if (amdgpu_sriov_vf(adev)) 4649 amdgpu_virt_release_full_gpu(adev, true); 4650 4651 /* failed in exclusive mode due to timeout */ 4652 if (amdgpu_sriov_vf(adev) && 4653 !amdgpu_sriov_runtime(adev) && 4654 amdgpu_virt_mmio_blocked(adev) && 4655 !amdgpu_virt_wait_reset(adev)) { 4656 dev_err(adev->dev, "VF exclusive mode timeout\n"); 4657 /* Don't send request since VF is inactive. */ 4658 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; 4659 adev->virt.ops = NULL; 4660 r = -EAGAIN; 4661 } 4662 amdgpu_release_ras_context(adev); 4663 4664 failed: 4665 amdgpu_vf_error_trans_all(adev); 4666 4667 return r; 4668 } 4669 4670 static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev) 4671 { 4672 4673 /* Clear all CPU mappings pointing to this device */ 4674 unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1); 4675 4676 /* Unmap all mapped bars - Doorbell, registers and VRAM */ 4677 amdgpu_doorbell_fini(adev); 4678 4679 iounmap(adev->rmmio); 4680 adev->rmmio = NULL; 4681 if (adev->mman.aper_base_kaddr) 4682 iounmap(adev->mman.aper_base_kaddr); 4683 adev->mman.aper_base_kaddr = NULL; 4684 4685 /* Memory manager related */ 4686 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { 4687 arch_phys_wc_del(adev->gmc.vram_mtrr); 4688 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); 4689 } 4690 } 4691 4692 /** 4693 * amdgpu_device_fini_hw - tear down the driver 4694 * 4695 * @adev: amdgpu_device pointer 4696 * 4697 * Tear down the driver info (all asics). 4698 * Called at driver shutdown. 4699 */ 4700 void amdgpu_device_fini_hw(struct amdgpu_device *adev) 4701 { 4702 dev_info(adev->dev, "amdgpu: finishing device.\n"); 4703 flush_delayed_work(&adev->delayed_init_work); 4704 4705 if (adev->mman.initialized) 4706 drain_workqueue(adev->mman.bdev.wq); 4707 adev->shutdown = true; 4708 4709 unregister_pm_notifier(&adev->pm_nb); 4710 4711 /* make sure IB test finished before entering exclusive mode 4712 * to avoid preemption on IB test 4713 */ 4714 if (amdgpu_sriov_vf(adev)) { 4715 amdgpu_virt_request_full_gpu(adev, false); 4716 amdgpu_virt_fini_data_exchange(adev); 4717 } 4718 4719 /* disable all interrupts */ 4720 amdgpu_irq_disable_all(adev); 4721 if (adev->mode_info.mode_config_initialized) { 4722 if (!drm_drv_uses_atomic_modeset(adev_to_drm(adev))) 4723 drm_helper_force_disable_all(adev_to_drm(adev)); 4724 else 4725 drm_atomic_helper_shutdown(adev_to_drm(adev)); 4726 } 4727 amdgpu_fence_driver_hw_fini(adev); 4728 4729 if (adev->pm.sysfs_initialized) 4730 amdgpu_pm_sysfs_fini(adev); 4731 if (adev->ucode_sysfs_en) 4732 amdgpu_ucode_sysfs_fini(adev); 4733 sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes); 4734 amdgpu_fru_sysfs_fini(adev); 4735 4736 amdgpu_reg_state_sysfs_fini(adev); 4737 amdgpu_xcp_cfg_sysfs_fini(adev); 4738 4739 /* disable ras feature must before hw fini */ 4740 amdgpu_ras_pre_fini(adev); 4741 4742 amdgpu_ttm_set_buffer_funcs_status(adev, false); 4743 4744 amdgpu_device_ip_fini_early(adev); 4745 4746 amdgpu_irq_fini_hw(adev); 4747 4748 if (adev->mman.initialized) 4749 ttm_device_clear_dma_mappings(&adev->mman.bdev); 4750 4751 amdgpu_gart_dummy_page_fini(adev); 4752 4753 if (drm_dev_is_unplugged(adev_to_drm(adev))) 4754 amdgpu_device_unmap_mmio(adev); 4755 4756 } 4757 4758 void amdgpu_device_fini_sw(struct amdgpu_device *adev) 4759 { 4760 int idx; 4761 bool px; 4762 4763 amdgpu_device_ip_fini(adev); 4764 amdgpu_fence_driver_sw_fini(adev); 4765 amdgpu_ucode_release(&adev->firmware.gpu_info_fw); 4766 adev->accel_working = false; 4767 dma_fence_put(rcu_dereference_protected(adev->gang_submit, true)); 4768 4769 amdgpu_reset_fini(adev); 4770 4771 /* free i2c buses */ 4772 amdgpu_i2c_fini(adev); 4773 4774 if (adev->bios) { 4775 if (amdgpu_emu_mode != 1) 4776 amdgpu_atombios_fini(adev); 4777 amdgpu_bios_release(adev); 4778 } 4779 4780 kfree(adev->fru_info); 4781 adev->fru_info = NULL; 4782 4783 px = amdgpu_device_supports_px(adev_to_drm(adev)); 4784 4785 if (px || (!dev_is_removable(&adev->pdev->dev) && 4786 apple_gmux_detect(NULL, NULL))) 4787 vga_switcheroo_unregister_client(adev->pdev); 4788 4789 if (px) 4790 vga_switcheroo_fini_domain_pm_ops(adev->dev); 4791 4792 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) 4793 vga_client_unregister(adev->pdev); 4794 4795 if (drm_dev_enter(adev_to_drm(adev), &idx)) { 4796 4797 iounmap(adev->rmmio); 4798 adev->rmmio = NULL; 4799 amdgpu_doorbell_fini(adev); 4800 drm_dev_exit(idx); 4801 } 4802 4803 if (IS_ENABLED(CONFIG_PERF_EVENTS)) 4804 amdgpu_pmu_fini(adev); 4805 if (adev->mman.discovery_bin) 4806 amdgpu_discovery_fini(adev); 4807 4808 amdgpu_reset_put_reset_domain(adev->reset_domain); 4809 adev->reset_domain = NULL; 4810 4811 kfree(adev->pci_state); 4812 4813 } 4814 4815 /** 4816 * amdgpu_device_evict_resources - evict device resources 4817 * @adev: amdgpu device object 4818 * 4819 * Evicts all ttm device resources(vram BOs, gart table) from the lru list 4820 * of the vram memory type. Mainly used for evicting device resources 4821 * at suspend time. 4822 * 4823 */ 4824 static int amdgpu_device_evict_resources(struct amdgpu_device *adev) 4825 { 4826 int ret; 4827 4828 /* No need to evict vram on APUs unless going to S4 */ 4829 if (!adev->in_s4 && (adev->flags & AMD_IS_APU)) 4830 return 0; 4831 4832 ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM); 4833 if (ret) 4834 DRM_WARN("evicting device resources failed\n"); 4835 return ret; 4836 } 4837 4838 /* 4839 * Suspend & resume. 4840 */ 4841 /** 4842 * amdgpu_device_pm_notifier - Notification block for Suspend/Hibernate events 4843 * @nb: notifier block 4844 * @mode: suspend mode 4845 * @data: data 4846 * 4847 * This function is called when the system is about to suspend or hibernate. 4848 * It is used to evict resources from the device before the system goes to 4849 * sleep while there is still access to swap. 4850 */ 4851 static int amdgpu_device_pm_notifier(struct notifier_block *nb, unsigned long mode, 4852 void *data) 4853 { 4854 struct amdgpu_device *adev = container_of(nb, struct amdgpu_device, pm_nb); 4855 int r; 4856 4857 switch (mode) { 4858 case PM_HIBERNATION_PREPARE: 4859 adev->in_s4 = true; 4860 fallthrough; 4861 case PM_SUSPEND_PREPARE: 4862 r = amdgpu_device_evict_resources(adev); 4863 /* 4864 * This is considered non-fatal at this time because 4865 * amdgpu_device_prepare() will also fatally evict resources. 4866 * See https://gitlab.freedesktop.org/drm/amd/-/issues/3781 4867 */ 4868 if (r) 4869 drm_warn(adev_to_drm(adev), "Failed to evict resources, freeze active processes if problems occur: %d\n", r); 4870 break; 4871 } 4872 4873 return NOTIFY_DONE; 4874 } 4875 4876 /** 4877 * amdgpu_device_prepare - prepare for device suspend 4878 * 4879 * @dev: drm dev pointer 4880 * 4881 * Prepare to put the hw in the suspend state (all asics). 4882 * Returns 0 for success or an error on failure. 4883 * Called at driver suspend. 4884 */ 4885 int amdgpu_device_prepare(struct drm_device *dev) 4886 { 4887 struct amdgpu_device *adev = drm_to_adev(dev); 4888 int i, r; 4889 4890 amdgpu_choose_low_power_state(adev); 4891 4892 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 4893 return 0; 4894 4895 /* Evict the majority of BOs before starting suspend sequence */ 4896 r = amdgpu_device_evict_resources(adev); 4897 if (r) 4898 goto unprepare; 4899 4900 flush_delayed_work(&adev->gfx.gfx_off_delay_work); 4901 4902 for (i = 0; i < adev->num_ip_blocks; i++) { 4903 if (!adev->ip_blocks[i].status.valid) 4904 continue; 4905 if (!adev->ip_blocks[i].version->funcs->prepare_suspend) 4906 continue; 4907 r = adev->ip_blocks[i].version->funcs->prepare_suspend(&adev->ip_blocks[i]); 4908 if (r) 4909 goto unprepare; 4910 } 4911 4912 return 0; 4913 4914 unprepare: 4915 adev->in_s0ix = adev->in_s3 = adev->in_s4 = false; 4916 4917 return r; 4918 } 4919 4920 /** 4921 * amdgpu_device_suspend - initiate device suspend 4922 * 4923 * @dev: drm dev pointer 4924 * @notify_clients: notify in-kernel DRM clients 4925 * 4926 * Puts the hw in the suspend state (all asics). 4927 * Returns 0 for success or an error on failure. 4928 * Called at driver suspend. 4929 */ 4930 int amdgpu_device_suspend(struct drm_device *dev, bool notify_clients) 4931 { 4932 struct amdgpu_device *adev = drm_to_adev(dev); 4933 int r = 0; 4934 4935 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 4936 return 0; 4937 4938 adev->in_suspend = true; 4939 4940 if (amdgpu_sriov_vf(adev)) { 4941 amdgpu_virt_fini_data_exchange(adev); 4942 r = amdgpu_virt_request_full_gpu(adev, false); 4943 if (r) 4944 return r; 4945 } 4946 4947 if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D3)) 4948 DRM_WARN("smart shift update failed\n"); 4949 4950 if (notify_clients) 4951 drm_client_dev_suspend(adev_to_drm(adev), false); 4952 4953 cancel_delayed_work_sync(&adev->delayed_init_work); 4954 4955 amdgpu_ras_suspend(adev); 4956 4957 amdgpu_device_ip_suspend_phase1(adev); 4958 4959 if (!adev->in_s0ix) 4960 amdgpu_amdkfd_suspend(adev, adev->in_runpm); 4961 4962 r = amdgpu_device_evict_resources(adev); 4963 if (r) 4964 return r; 4965 4966 amdgpu_ttm_set_buffer_funcs_status(adev, false); 4967 4968 amdgpu_fence_driver_hw_fini(adev); 4969 4970 amdgpu_device_ip_suspend_phase2(adev); 4971 4972 if (amdgpu_sriov_vf(adev)) 4973 amdgpu_virt_release_full_gpu(adev, false); 4974 4975 r = amdgpu_dpm_notify_rlc_state(adev, false); 4976 if (r) 4977 return r; 4978 4979 return 0; 4980 } 4981 4982 /** 4983 * amdgpu_device_resume - initiate device resume 4984 * 4985 * @dev: drm dev pointer 4986 * @notify_clients: notify in-kernel DRM clients 4987 * 4988 * Bring the hw back to operating state (all asics). 4989 * Returns 0 for success or an error on failure. 4990 * Called at driver resume. 4991 */ 4992 int amdgpu_device_resume(struct drm_device *dev, bool notify_clients) 4993 { 4994 struct amdgpu_device *adev = drm_to_adev(dev); 4995 int r = 0; 4996 4997 if (amdgpu_sriov_vf(adev)) { 4998 r = amdgpu_virt_request_full_gpu(adev, true); 4999 if (r) 5000 return r; 5001 } 5002 5003 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 5004 return 0; 5005 5006 if (adev->in_s0ix) 5007 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry); 5008 5009 /* post card */ 5010 if (amdgpu_device_need_post(adev)) { 5011 r = amdgpu_device_asic_init(adev); 5012 if (r) 5013 dev_err(adev->dev, "amdgpu asic init failed\n"); 5014 } 5015 5016 r = amdgpu_device_ip_resume(adev); 5017 5018 if (r) { 5019 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r); 5020 goto exit; 5021 } 5022 5023 if (!adev->in_s0ix) { 5024 r = amdgpu_amdkfd_resume(adev, adev->in_runpm); 5025 if (r) 5026 goto exit; 5027 } 5028 5029 r = amdgpu_device_ip_late_init(adev); 5030 if (r) 5031 goto exit; 5032 5033 queue_delayed_work(system_wq, &adev->delayed_init_work, 5034 msecs_to_jiffies(AMDGPU_RESUME_MS)); 5035 exit: 5036 if (amdgpu_sriov_vf(adev)) { 5037 amdgpu_virt_init_data_exchange(adev); 5038 amdgpu_virt_release_full_gpu(adev, true); 5039 } 5040 5041 if (r) 5042 return r; 5043 5044 /* Make sure IB tests flushed */ 5045 flush_delayed_work(&adev->delayed_init_work); 5046 5047 if (notify_clients) 5048 drm_client_dev_resume(adev_to_drm(adev), false); 5049 5050 amdgpu_ras_resume(adev); 5051 5052 if (adev->mode_info.num_crtc) { 5053 /* 5054 * Most of the connector probing functions try to acquire runtime pm 5055 * refs to ensure that the GPU is powered on when connector polling is 5056 * performed. Since we're calling this from a runtime PM callback, 5057 * trying to acquire rpm refs will cause us to deadlock. 5058 * 5059 * Since we're guaranteed to be holding the rpm lock, it's safe to 5060 * temporarily disable the rpm helpers so this doesn't deadlock us. 5061 */ 5062 #ifdef CONFIG_PM 5063 dev->dev->power.disable_depth++; 5064 #endif 5065 if (!adev->dc_enabled) 5066 drm_helper_hpd_irq_event(dev); 5067 else 5068 drm_kms_helper_hotplug_event(dev); 5069 #ifdef CONFIG_PM 5070 dev->dev->power.disable_depth--; 5071 #endif 5072 } 5073 adev->in_suspend = false; 5074 5075 if (adev->enable_mes) 5076 amdgpu_mes_self_test(adev); 5077 5078 if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D0)) 5079 DRM_WARN("smart shift update failed\n"); 5080 5081 return 0; 5082 } 5083 5084 /** 5085 * amdgpu_device_ip_check_soft_reset - did soft reset succeed 5086 * 5087 * @adev: amdgpu_device pointer 5088 * 5089 * The list of all the hardware IPs that make up the asic is walked and 5090 * the check_soft_reset callbacks are run. check_soft_reset determines 5091 * if the asic is still hung or not. 5092 * Returns true if any of the IPs are still in a hung state, false if not. 5093 */ 5094 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev) 5095 { 5096 int i; 5097 bool asic_hang = false; 5098 5099 if (amdgpu_sriov_vf(adev)) 5100 return true; 5101 5102 if (amdgpu_asic_need_full_reset(adev)) 5103 return true; 5104 5105 for (i = 0; i < adev->num_ip_blocks; i++) { 5106 if (!adev->ip_blocks[i].status.valid) 5107 continue; 5108 if (adev->ip_blocks[i].version->funcs->check_soft_reset) 5109 adev->ip_blocks[i].status.hang = 5110 adev->ip_blocks[i].version->funcs->check_soft_reset( 5111 &adev->ip_blocks[i]); 5112 if (adev->ip_blocks[i].status.hang) { 5113 dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name); 5114 asic_hang = true; 5115 } 5116 } 5117 return asic_hang; 5118 } 5119 5120 /** 5121 * amdgpu_device_ip_pre_soft_reset - prepare for soft reset 5122 * 5123 * @adev: amdgpu_device pointer 5124 * 5125 * The list of all the hardware IPs that make up the asic is walked and the 5126 * pre_soft_reset callbacks are run if the block is hung. pre_soft_reset 5127 * handles any IP specific hardware or software state changes that are 5128 * necessary for a soft reset to succeed. 5129 * Returns 0 on success, negative error code on failure. 5130 */ 5131 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev) 5132 { 5133 int i, r = 0; 5134 5135 for (i = 0; i < adev->num_ip_blocks; i++) { 5136 if (!adev->ip_blocks[i].status.valid) 5137 continue; 5138 if (adev->ip_blocks[i].status.hang && 5139 adev->ip_blocks[i].version->funcs->pre_soft_reset) { 5140 r = adev->ip_blocks[i].version->funcs->pre_soft_reset(&adev->ip_blocks[i]); 5141 if (r) 5142 return r; 5143 } 5144 } 5145 5146 return 0; 5147 } 5148 5149 /** 5150 * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed 5151 * 5152 * @adev: amdgpu_device pointer 5153 * 5154 * Some hardware IPs cannot be soft reset. If they are hung, a full gpu 5155 * reset is necessary to recover. 5156 * Returns true if a full asic reset is required, false if not. 5157 */ 5158 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev) 5159 { 5160 int i; 5161 5162 if (amdgpu_asic_need_full_reset(adev)) 5163 return true; 5164 5165 for (i = 0; i < adev->num_ip_blocks; i++) { 5166 if (!adev->ip_blocks[i].status.valid) 5167 continue; 5168 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) || 5169 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) || 5170 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) || 5171 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) || 5172 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) { 5173 if (adev->ip_blocks[i].status.hang) { 5174 dev_info(adev->dev, "Some block need full reset!\n"); 5175 return true; 5176 } 5177 } 5178 } 5179 return false; 5180 } 5181 5182 /** 5183 * amdgpu_device_ip_soft_reset - do a soft reset 5184 * 5185 * @adev: amdgpu_device pointer 5186 * 5187 * The list of all the hardware IPs that make up the asic is walked and the 5188 * soft_reset callbacks are run if the block is hung. soft_reset handles any 5189 * IP specific hardware or software state changes that are necessary to soft 5190 * reset the IP. 5191 * Returns 0 on success, negative error code on failure. 5192 */ 5193 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev) 5194 { 5195 int i, r = 0; 5196 5197 for (i = 0; i < adev->num_ip_blocks; i++) { 5198 if (!adev->ip_blocks[i].status.valid) 5199 continue; 5200 if (adev->ip_blocks[i].status.hang && 5201 adev->ip_blocks[i].version->funcs->soft_reset) { 5202 r = adev->ip_blocks[i].version->funcs->soft_reset(&adev->ip_blocks[i]); 5203 if (r) 5204 return r; 5205 } 5206 } 5207 5208 return 0; 5209 } 5210 5211 /** 5212 * amdgpu_device_ip_post_soft_reset - clean up from soft reset 5213 * 5214 * @adev: amdgpu_device pointer 5215 * 5216 * The list of all the hardware IPs that make up the asic is walked and the 5217 * post_soft_reset callbacks are run if the asic was hung. post_soft_reset 5218 * handles any IP specific hardware or software state changes that are 5219 * necessary after the IP has been soft reset. 5220 * Returns 0 on success, negative error code on failure. 5221 */ 5222 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev) 5223 { 5224 int i, r = 0; 5225 5226 for (i = 0; i < adev->num_ip_blocks; i++) { 5227 if (!adev->ip_blocks[i].status.valid) 5228 continue; 5229 if (adev->ip_blocks[i].status.hang && 5230 adev->ip_blocks[i].version->funcs->post_soft_reset) 5231 r = adev->ip_blocks[i].version->funcs->post_soft_reset(&adev->ip_blocks[i]); 5232 if (r) 5233 return r; 5234 } 5235 5236 return 0; 5237 } 5238 5239 /** 5240 * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf 5241 * 5242 * @adev: amdgpu_device pointer 5243 * @reset_context: amdgpu reset context pointer 5244 * 5245 * do VF FLR and reinitialize Asic 5246 * return 0 means succeeded otherwise failed 5247 */ 5248 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev, 5249 struct amdgpu_reset_context *reset_context) 5250 { 5251 int r; 5252 struct amdgpu_hive_info *hive = NULL; 5253 5254 if (test_bit(AMDGPU_HOST_FLR, &reset_context->flags)) { 5255 if (!amdgpu_ras_get_fed_status(adev)) 5256 amdgpu_virt_ready_to_reset(adev); 5257 amdgpu_virt_wait_reset(adev); 5258 clear_bit(AMDGPU_HOST_FLR, &reset_context->flags); 5259 r = amdgpu_virt_request_full_gpu(adev, true); 5260 } else { 5261 r = amdgpu_virt_reset_gpu(adev); 5262 } 5263 if (r) 5264 return r; 5265 5266 amdgpu_ras_clear_err_state(adev); 5267 amdgpu_irq_gpu_reset_resume_helper(adev); 5268 5269 /* some sw clean up VF needs to do before recover */ 5270 amdgpu_virt_post_reset(adev); 5271 5272 /* Resume IP prior to SMC */ 5273 r = amdgpu_device_ip_reinit_early_sriov(adev); 5274 if (r) 5275 return r; 5276 5277 amdgpu_virt_init_data_exchange(adev); 5278 5279 r = amdgpu_device_fw_loading(adev); 5280 if (r) 5281 return r; 5282 5283 /* now we are okay to resume SMC/CP/SDMA */ 5284 r = amdgpu_device_ip_reinit_late_sriov(adev); 5285 if (r) 5286 return r; 5287 5288 hive = amdgpu_get_xgmi_hive(adev); 5289 /* Update PSP FW topology after reset */ 5290 if (hive && adev->gmc.xgmi.num_physical_nodes > 1) 5291 r = amdgpu_xgmi_update_topology(hive, adev); 5292 if (hive) 5293 amdgpu_put_xgmi_hive(hive); 5294 if (r) 5295 return r; 5296 5297 r = amdgpu_ib_ring_tests(adev); 5298 if (r) 5299 return r; 5300 5301 if (adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) 5302 amdgpu_inc_vram_lost(adev); 5303 5304 /* need to be called during full access so we can't do it later like 5305 * bare-metal does. 5306 */ 5307 amdgpu_amdkfd_post_reset(adev); 5308 amdgpu_virt_release_full_gpu(adev, true); 5309 5310 /* Aldebaran and gfx_11_0_3 support ras in SRIOV, so need resume ras during reset */ 5311 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) || 5312 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) || 5313 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4) || 5314 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3)) 5315 amdgpu_ras_resume(adev); 5316 5317 amdgpu_virt_ras_telemetry_post_reset(adev); 5318 5319 return 0; 5320 } 5321 5322 /** 5323 * amdgpu_device_has_job_running - check if there is any unfinished job 5324 * 5325 * @adev: amdgpu_device pointer 5326 * 5327 * check if there is any job running on the device when guest driver receives 5328 * FLR notification from host driver. If there are still jobs running, then 5329 * the guest driver will not respond the FLR reset. Instead, let the job hit 5330 * the timeout and guest driver then issue the reset request. 5331 */ 5332 bool amdgpu_device_has_job_running(struct amdgpu_device *adev) 5333 { 5334 int i; 5335 5336 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 5337 struct amdgpu_ring *ring = adev->rings[i]; 5338 5339 if (!amdgpu_ring_sched_ready(ring)) 5340 continue; 5341 5342 if (amdgpu_fence_count_emitted(ring)) 5343 return true; 5344 } 5345 return false; 5346 } 5347 5348 /** 5349 * amdgpu_device_should_recover_gpu - check if we should try GPU recovery 5350 * 5351 * @adev: amdgpu_device pointer 5352 * 5353 * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover 5354 * a hung GPU. 5355 */ 5356 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev) 5357 { 5358 5359 if (amdgpu_gpu_recovery == 0) 5360 goto disabled; 5361 5362 /* Skip soft reset check in fatal error mode */ 5363 if (!amdgpu_ras_is_poison_mode_supported(adev)) 5364 return true; 5365 5366 if (amdgpu_sriov_vf(adev)) 5367 return true; 5368 5369 if (amdgpu_gpu_recovery == -1) { 5370 switch (adev->asic_type) { 5371 #ifdef CONFIG_DRM_AMDGPU_SI 5372 case CHIP_VERDE: 5373 case CHIP_TAHITI: 5374 case CHIP_PITCAIRN: 5375 case CHIP_OLAND: 5376 case CHIP_HAINAN: 5377 #endif 5378 #ifdef CONFIG_DRM_AMDGPU_CIK 5379 case CHIP_KAVERI: 5380 case CHIP_KABINI: 5381 case CHIP_MULLINS: 5382 #endif 5383 case CHIP_CARRIZO: 5384 case CHIP_STONEY: 5385 case CHIP_CYAN_SKILLFISH: 5386 goto disabled; 5387 default: 5388 break; 5389 } 5390 } 5391 5392 return true; 5393 5394 disabled: 5395 dev_info(adev->dev, "GPU recovery disabled.\n"); 5396 return false; 5397 } 5398 5399 int amdgpu_device_mode1_reset(struct amdgpu_device *adev) 5400 { 5401 u32 i; 5402 int ret = 0; 5403 5404 if (adev->bios) 5405 amdgpu_atombios_scratch_regs_engine_hung(adev, true); 5406 5407 dev_info(adev->dev, "GPU mode1 reset\n"); 5408 5409 /* Cache the state before bus master disable. The saved config space 5410 * values are used in other cases like restore after mode-2 reset. 5411 */ 5412 amdgpu_device_cache_pci_state(adev->pdev); 5413 5414 /* disable BM */ 5415 pci_clear_master(adev->pdev); 5416 5417 if (amdgpu_dpm_is_mode1_reset_supported(adev)) { 5418 dev_info(adev->dev, "GPU smu mode1 reset\n"); 5419 ret = amdgpu_dpm_mode1_reset(adev); 5420 } else { 5421 dev_info(adev->dev, "GPU psp mode1 reset\n"); 5422 ret = psp_gpu_reset(adev); 5423 } 5424 5425 if (ret) 5426 goto mode1_reset_failed; 5427 5428 amdgpu_device_load_pci_state(adev->pdev); 5429 ret = amdgpu_psp_wait_for_bootloader(adev); 5430 if (ret) 5431 goto mode1_reset_failed; 5432 5433 /* wait for asic to come out of reset */ 5434 for (i = 0; i < adev->usec_timeout; i++) { 5435 u32 memsize = adev->nbio.funcs->get_memsize(adev); 5436 5437 if (memsize != 0xffffffff) 5438 break; 5439 udelay(1); 5440 } 5441 5442 if (i >= adev->usec_timeout) { 5443 ret = -ETIMEDOUT; 5444 goto mode1_reset_failed; 5445 } 5446 5447 if (adev->bios) 5448 amdgpu_atombios_scratch_regs_engine_hung(adev, false); 5449 5450 return 0; 5451 5452 mode1_reset_failed: 5453 dev_err(adev->dev, "GPU mode1 reset failed\n"); 5454 return ret; 5455 } 5456 5457 int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, 5458 struct amdgpu_reset_context *reset_context) 5459 { 5460 int i, r = 0; 5461 struct amdgpu_job *job = NULL; 5462 struct amdgpu_device *tmp_adev = reset_context->reset_req_dev; 5463 bool need_full_reset = 5464 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 5465 5466 if (reset_context->reset_req_dev == adev) 5467 job = reset_context->job; 5468 5469 if (amdgpu_sriov_vf(adev)) 5470 amdgpu_virt_pre_reset(adev); 5471 5472 amdgpu_fence_driver_isr_toggle(adev, true); 5473 5474 /* block all schedulers and reset given job's ring */ 5475 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 5476 struct amdgpu_ring *ring = adev->rings[i]; 5477 5478 if (!amdgpu_ring_sched_ready(ring)) 5479 continue; 5480 5481 /* Clear job fence from fence drv to avoid force_completion 5482 * leave NULL and vm flush fence in fence drv 5483 */ 5484 amdgpu_fence_driver_clear_job_fences(ring); 5485 5486 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */ 5487 amdgpu_fence_driver_force_completion(ring); 5488 } 5489 5490 amdgpu_fence_driver_isr_toggle(adev, false); 5491 5492 if (job && job->vm) 5493 drm_sched_increase_karma(&job->base); 5494 5495 r = amdgpu_reset_prepare_hwcontext(adev, reset_context); 5496 /* If reset handler not implemented, continue; otherwise return */ 5497 if (r == -EOPNOTSUPP) 5498 r = 0; 5499 else 5500 return r; 5501 5502 /* Don't suspend on bare metal if we are not going to HW reset the ASIC */ 5503 if (!amdgpu_sriov_vf(adev)) { 5504 5505 if (!need_full_reset) 5506 need_full_reset = amdgpu_device_ip_need_full_reset(adev); 5507 5508 if (!need_full_reset && amdgpu_gpu_recovery && 5509 amdgpu_device_ip_check_soft_reset(adev)) { 5510 amdgpu_device_ip_pre_soft_reset(adev); 5511 r = amdgpu_device_ip_soft_reset(adev); 5512 amdgpu_device_ip_post_soft_reset(adev); 5513 if (r || amdgpu_device_ip_check_soft_reset(adev)) { 5514 dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n"); 5515 need_full_reset = true; 5516 } 5517 } 5518 5519 if (!test_bit(AMDGPU_SKIP_COREDUMP, &reset_context->flags)) { 5520 dev_info(tmp_adev->dev, "Dumping IP State\n"); 5521 /* Trigger ip dump before we reset the asic */ 5522 for (i = 0; i < tmp_adev->num_ip_blocks; i++) 5523 if (tmp_adev->ip_blocks[i].version->funcs->dump_ip_state) 5524 tmp_adev->ip_blocks[i].version->funcs 5525 ->dump_ip_state((void *)&tmp_adev->ip_blocks[i]); 5526 dev_info(tmp_adev->dev, "Dumping IP State Completed\n"); 5527 } 5528 5529 if (need_full_reset) 5530 r = amdgpu_device_ip_suspend(adev); 5531 if (need_full_reset) 5532 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 5533 else 5534 clear_bit(AMDGPU_NEED_FULL_RESET, 5535 &reset_context->flags); 5536 } 5537 5538 return r; 5539 } 5540 5541 int amdgpu_device_reinit_after_reset(struct amdgpu_reset_context *reset_context) 5542 { 5543 struct list_head *device_list_handle; 5544 bool full_reset, vram_lost = false; 5545 struct amdgpu_device *tmp_adev; 5546 int r, init_level; 5547 5548 device_list_handle = reset_context->reset_device_list; 5549 5550 if (!device_list_handle) 5551 return -EINVAL; 5552 5553 full_reset = test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 5554 5555 /** 5556 * If it's reset on init, it's default init level, otherwise keep level 5557 * as recovery level. 5558 */ 5559 if (reset_context->method == AMD_RESET_METHOD_ON_INIT) 5560 init_level = AMDGPU_INIT_LEVEL_DEFAULT; 5561 else 5562 init_level = AMDGPU_INIT_LEVEL_RESET_RECOVERY; 5563 5564 r = 0; 5565 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 5566 amdgpu_set_init_level(tmp_adev, init_level); 5567 if (full_reset) { 5568 /* post card */ 5569 amdgpu_ras_clear_err_state(tmp_adev); 5570 r = amdgpu_device_asic_init(tmp_adev); 5571 if (r) { 5572 dev_warn(tmp_adev->dev, "asic atom init failed!"); 5573 } else { 5574 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n"); 5575 5576 r = amdgpu_device_ip_resume_phase1(tmp_adev); 5577 if (r) 5578 goto out; 5579 5580 vram_lost = amdgpu_device_check_vram_lost(tmp_adev); 5581 5582 if (!test_bit(AMDGPU_SKIP_COREDUMP, &reset_context->flags)) 5583 amdgpu_coredump(tmp_adev, false, vram_lost, reset_context->job); 5584 5585 if (vram_lost) { 5586 DRM_INFO("VRAM is lost due to GPU reset!\n"); 5587 amdgpu_inc_vram_lost(tmp_adev); 5588 } 5589 5590 r = amdgpu_device_fw_loading(tmp_adev); 5591 if (r) 5592 return r; 5593 5594 r = amdgpu_xcp_restore_partition_mode( 5595 tmp_adev->xcp_mgr); 5596 if (r) 5597 goto out; 5598 5599 r = amdgpu_device_ip_resume_phase2(tmp_adev); 5600 if (r) 5601 goto out; 5602 5603 if (tmp_adev->mman.buffer_funcs_ring->sched.ready) 5604 amdgpu_ttm_set_buffer_funcs_status(tmp_adev, true); 5605 5606 r = amdgpu_device_ip_resume_phase3(tmp_adev); 5607 if (r) 5608 goto out; 5609 5610 if (vram_lost) 5611 amdgpu_device_fill_reset_magic(tmp_adev); 5612 5613 /* 5614 * Add this ASIC as tracked as reset was already 5615 * complete successfully. 5616 */ 5617 amdgpu_register_gpu_instance(tmp_adev); 5618 5619 if (!reset_context->hive && 5620 tmp_adev->gmc.xgmi.num_physical_nodes > 1) 5621 amdgpu_xgmi_add_device(tmp_adev); 5622 5623 r = amdgpu_device_ip_late_init(tmp_adev); 5624 if (r) 5625 goto out; 5626 5627 drm_client_dev_resume(adev_to_drm(tmp_adev), false); 5628 5629 /* 5630 * The GPU enters bad state once faulty pages 5631 * by ECC has reached the threshold, and ras 5632 * recovery is scheduled next. So add one check 5633 * here to break recovery if it indeed exceeds 5634 * bad page threshold, and remind user to 5635 * retire this GPU or setting one bigger 5636 * bad_page_threshold value to fix this once 5637 * probing driver again. 5638 */ 5639 if (!amdgpu_ras_is_rma(tmp_adev)) { 5640 /* must succeed. */ 5641 amdgpu_ras_resume(tmp_adev); 5642 } else { 5643 r = -EINVAL; 5644 goto out; 5645 } 5646 5647 /* Update PSP FW topology after reset */ 5648 if (reset_context->hive && 5649 tmp_adev->gmc.xgmi.num_physical_nodes > 1) 5650 r = amdgpu_xgmi_update_topology( 5651 reset_context->hive, tmp_adev); 5652 } 5653 } 5654 5655 out: 5656 if (!r) { 5657 /* IP init is complete now, set level as default */ 5658 amdgpu_set_init_level(tmp_adev, 5659 AMDGPU_INIT_LEVEL_DEFAULT); 5660 amdgpu_irq_gpu_reset_resume_helper(tmp_adev); 5661 r = amdgpu_ib_ring_tests(tmp_adev); 5662 if (r) { 5663 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r); 5664 r = -EAGAIN; 5665 goto end; 5666 } 5667 } 5668 5669 if (r) 5670 tmp_adev->asic_reset_res = r; 5671 } 5672 5673 end: 5674 return r; 5675 } 5676 5677 int amdgpu_do_asic_reset(struct list_head *device_list_handle, 5678 struct amdgpu_reset_context *reset_context) 5679 { 5680 struct amdgpu_device *tmp_adev = NULL; 5681 bool need_full_reset, skip_hw_reset; 5682 int r = 0; 5683 5684 /* Try reset handler method first */ 5685 tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device, 5686 reset_list); 5687 5688 reset_context->reset_device_list = device_list_handle; 5689 r = amdgpu_reset_perform_reset(tmp_adev, reset_context); 5690 /* If reset handler not implemented, continue; otherwise return */ 5691 if (r == -EOPNOTSUPP) 5692 r = 0; 5693 else 5694 return r; 5695 5696 /* Reset handler not implemented, use the default method */ 5697 need_full_reset = 5698 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 5699 skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags); 5700 5701 /* 5702 * ASIC reset has to be done on all XGMI hive nodes ASAP 5703 * to allow proper links negotiation in FW (within 1 sec) 5704 */ 5705 if (!skip_hw_reset && need_full_reset) { 5706 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 5707 /* For XGMI run all resets in parallel to speed up the process */ 5708 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) { 5709 if (!queue_work(system_unbound_wq, 5710 &tmp_adev->xgmi_reset_work)) 5711 r = -EALREADY; 5712 } else 5713 r = amdgpu_asic_reset(tmp_adev); 5714 5715 if (r) { 5716 dev_err(tmp_adev->dev, 5717 "ASIC reset failed with error, %d for drm dev, %s", 5718 r, adev_to_drm(tmp_adev)->unique); 5719 goto out; 5720 } 5721 } 5722 5723 /* For XGMI wait for all resets to complete before proceed */ 5724 if (!r) { 5725 list_for_each_entry(tmp_adev, device_list_handle, 5726 reset_list) { 5727 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) { 5728 flush_work(&tmp_adev->xgmi_reset_work); 5729 r = tmp_adev->asic_reset_res; 5730 if (r) 5731 break; 5732 } 5733 } 5734 } 5735 } 5736 5737 if (!r && amdgpu_ras_intr_triggered()) { 5738 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 5739 amdgpu_ras_reset_error_count(tmp_adev, 5740 AMDGPU_RAS_BLOCK__MMHUB); 5741 } 5742 5743 amdgpu_ras_intr_cleared(); 5744 } 5745 5746 r = amdgpu_device_reinit_after_reset(reset_context); 5747 if (r == -EAGAIN) 5748 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 5749 else 5750 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 5751 5752 out: 5753 return r; 5754 } 5755 5756 static void amdgpu_device_set_mp1_state(struct amdgpu_device *adev) 5757 { 5758 5759 switch (amdgpu_asic_reset_method(adev)) { 5760 case AMD_RESET_METHOD_MODE1: 5761 adev->mp1_state = PP_MP1_STATE_SHUTDOWN; 5762 break; 5763 case AMD_RESET_METHOD_MODE2: 5764 adev->mp1_state = PP_MP1_STATE_RESET; 5765 break; 5766 default: 5767 adev->mp1_state = PP_MP1_STATE_NONE; 5768 break; 5769 } 5770 } 5771 5772 static void amdgpu_device_unset_mp1_state(struct amdgpu_device *adev) 5773 { 5774 amdgpu_vf_error_trans_all(adev); 5775 adev->mp1_state = PP_MP1_STATE_NONE; 5776 } 5777 5778 static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev) 5779 { 5780 struct pci_dev *p = NULL; 5781 5782 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus), 5783 adev->pdev->bus->number, 1); 5784 if (p) { 5785 pm_runtime_enable(&(p->dev)); 5786 pm_runtime_resume(&(p->dev)); 5787 } 5788 5789 pci_dev_put(p); 5790 } 5791 5792 static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev) 5793 { 5794 enum amd_reset_method reset_method; 5795 struct pci_dev *p = NULL; 5796 u64 expires; 5797 5798 /* 5799 * For now, only BACO and mode1 reset are confirmed 5800 * to suffer the audio issue without proper suspended. 5801 */ 5802 reset_method = amdgpu_asic_reset_method(adev); 5803 if ((reset_method != AMD_RESET_METHOD_BACO) && 5804 (reset_method != AMD_RESET_METHOD_MODE1)) 5805 return -EINVAL; 5806 5807 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus), 5808 adev->pdev->bus->number, 1); 5809 if (!p) 5810 return -ENODEV; 5811 5812 expires = pm_runtime_autosuspend_expiration(&(p->dev)); 5813 if (!expires) 5814 /* 5815 * If we cannot get the audio device autosuspend delay, 5816 * a fixed 4S interval will be used. Considering 3S is 5817 * the audio controller default autosuspend delay setting. 5818 * 4S used here is guaranteed to cover that. 5819 */ 5820 expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL; 5821 5822 while (!pm_runtime_status_suspended(&(p->dev))) { 5823 if (!pm_runtime_suspend(&(p->dev))) 5824 break; 5825 5826 if (expires < ktime_get_mono_fast_ns()) { 5827 dev_warn(adev->dev, "failed to suspend display audio\n"); 5828 pci_dev_put(p); 5829 /* TODO: abort the succeeding gpu reset? */ 5830 return -ETIMEDOUT; 5831 } 5832 } 5833 5834 pm_runtime_disable(&(p->dev)); 5835 5836 pci_dev_put(p); 5837 return 0; 5838 } 5839 5840 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev) 5841 { 5842 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5843 5844 #if defined(CONFIG_DEBUG_FS) 5845 if (!amdgpu_sriov_vf(adev)) 5846 cancel_work(&adev->reset_work); 5847 #endif 5848 5849 if (adev->kfd.dev) 5850 cancel_work(&adev->kfd.reset_work); 5851 5852 if (amdgpu_sriov_vf(adev)) 5853 cancel_work(&adev->virt.flr_work); 5854 5855 if (con && adev->ras_enabled) 5856 cancel_work(&con->recovery_work); 5857 5858 } 5859 5860 static int amdgpu_device_health_check(struct list_head *device_list_handle) 5861 { 5862 struct amdgpu_device *tmp_adev; 5863 int ret = 0; 5864 u32 status; 5865 5866 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 5867 pci_read_config_dword(tmp_adev->pdev, PCI_COMMAND, &status); 5868 if (PCI_POSSIBLE_ERROR(status)) { 5869 dev_err(tmp_adev->dev, "device lost from bus!"); 5870 ret = -ENODEV; 5871 } 5872 } 5873 5874 return ret; 5875 } 5876 5877 /** 5878 * amdgpu_device_gpu_recover - reset the asic and recover scheduler 5879 * 5880 * @adev: amdgpu_device pointer 5881 * @job: which job trigger hang 5882 * @reset_context: amdgpu reset context pointer 5883 * 5884 * Attempt to reset the GPU if it has hung (all asics). 5885 * Attempt to do soft-reset or full-reset and reinitialize Asic 5886 * Returns 0 for success or an error on failure. 5887 */ 5888 5889 int amdgpu_device_gpu_recover(struct amdgpu_device *adev, 5890 struct amdgpu_job *job, 5891 struct amdgpu_reset_context *reset_context) 5892 { 5893 struct list_head device_list, *device_list_handle = NULL; 5894 bool job_signaled = false; 5895 struct amdgpu_hive_info *hive = NULL; 5896 struct amdgpu_device *tmp_adev = NULL; 5897 int i, r = 0; 5898 bool need_emergency_restart = false; 5899 bool audio_suspended = false; 5900 int retry_limit = AMDGPU_MAX_RETRY_LIMIT; 5901 5902 /* 5903 * If it reaches here because of hang/timeout and a RAS error is 5904 * detected at the same time, let RAS recovery take care of it. 5905 */ 5906 if (amdgpu_ras_is_err_state(adev, AMDGPU_RAS_BLOCK__ANY) && 5907 !amdgpu_sriov_vf(adev) && 5908 reset_context->src != AMDGPU_RESET_SRC_RAS) { 5909 dev_dbg(adev->dev, 5910 "Gpu recovery from source: %d yielding to RAS error recovery handling", 5911 reset_context->src); 5912 return 0; 5913 } 5914 /* 5915 * Special case: RAS triggered and full reset isn't supported 5916 */ 5917 need_emergency_restart = amdgpu_ras_need_emergency_restart(adev); 5918 5919 /* 5920 * Flush RAM to disk so that after reboot 5921 * the user can read log and see why the system rebooted. 5922 */ 5923 if (need_emergency_restart && amdgpu_ras_get_context(adev) && 5924 amdgpu_ras_get_context(adev)->reboot) { 5925 DRM_WARN("Emergency reboot."); 5926 5927 ksys_sync_helper(); 5928 emergency_restart(); 5929 } 5930 5931 dev_info(adev->dev, "GPU %s begin!\n", 5932 need_emergency_restart ? "jobs stop":"reset"); 5933 5934 if (!amdgpu_sriov_vf(adev)) 5935 hive = amdgpu_get_xgmi_hive(adev); 5936 if (hive) 5937 mutex_lock(&hive->hive_lock); 5938 5939 reset_context->job = job; 5940 reset_context->hive = hive; 5941 /* 5942 * Build list of devices to reset. 5943 * In case we are in XGMI hive mode, resort the device list 5944 * to put adev in the 1st position. 5945 */ 5946 INIT_LIST_HEAD(&device_list); 5947 if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1) && hive) { 5948 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { 5949 list_add_tail(&tmp_adev->reset_list, &device_list); 5950 if (adev->shutdown) 5951 tmp_adev->shutdown = true; 5952 } 5953 if (!list_is_first(&adev->reset_list, &device_list)) 5954 list_rotate_to_front(&adev->reset_list, &device_list); 5955 device_list_handle = &device_list; 5956 } else { 5957 list_add_tail(&adev->reset_list, &device_list); 5958 device_list_handle = &device_list; 5959 } 5960 5961 if (!amdgpu_sriov_vf(adev)) { 5962 r = amdgpu_device_health_check(device_list_handle); 5963 if (r) 5964 goto end_reset; 5965 } 5966 5967 /* We need to lock reset domain only once both for XGMI and single device */ 5968 tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device, 5969 reset_list); 5970 amdgpu_device_lock_reset_domain(tmp_adev->reset_domain); 5971 5972 /* block all schedulers and reset given job's ring */ 5973 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 5974 5975 amdgpu_device_set_mp1_state(tmp_adev); 5976 5977 /* 5978 * Try to put the audio codec into suspend state 5979 * before gpu reset started. 5980 * 5981 * Due to the power domain of the graphics device 5982 * is shared with AZ power domain. Without this, 5983 * we may change the audio hardware from behind 5984 * the audio driver's back. That will trigger 5985 * some audio codec errors. 5986 */ 5987 if (!amdgpu_device_suspend_display_audio(tmp_adev)) 5988 audio_suspended = true; 5989 5990 amdgpu_ras_set_error_query_ready(tmp_adev, false); 5991 5992 cancel_delayed_work_sync(&tmp_adev->delayed_init_work); 5993 5994 amdgpu_amdkfd_pre_reset(tmp_adev, reset_context); 5995 5996 /* 5997 * Mark these ASICs to be reset as untracked first 5998 * And add them back after reset completed 5999 */ 6000 amdgpu_unregister_gpu_instance(tmp_adev); 6001 6002 drm_client_dev_suspend(adev_to_drm(tmp_adev), false); 6003 6004 /* disable ras on ALL IPs */ 6005 if (!need_emergency_restart && 6006 amdgpu_device_ip_need_full_reset(tmp_adev)) 6007 amdgpu_ras_suspend(tmp_adev); 6008 6009 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 6010 struct amdgpu_ring *ring = tmp_adev->rings[i]; 6011 6012 if (!amdgpu_ring_sched_ready(ring)) 6013 continue; 6014 6015 drm_sched_stop(&ring->sched, job ? &job->base : NULL); 6016 6017 if (need_emergency_restart) 6018 amdgpu_job_stop_all_jobs_on_sched(&ring->sched); 6019 } 6020 atomic_inc(&tmp_adev->gpu_reset_counter); 6021 } 6022 6023 if (need_emergency_restart) 6024 goto skip_sched_resume; 6025 6026 /* 6027 * Must check guilty signal here since after this point all old 6028 * HW fences are force signaled. 6029 * 6030 * job->base holds a reference to parent fence 6031 */ 6032 if (job && dma_fence_is_signaled(&job->hw_fence)) { 6033 job_signaled = true; 6034 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset"); 6035 goto skip_hw_reset; 6036 } 6037 6038 retry: /* Rest of adevs pre asic reset from XGMI hive. */ 6039 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 6040 r = amdgpu_device_pre_asic_reset(tmp_adev, reset_context); 6041 /*TODO Should we stop ?*/ 6042 if (r) { 6043 dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ", 6044 r, adev_to_drm(tmp_adev)->unique); 6045 tmp_adev->asic_reset_res = r; 6046 } 6047 } 6048 6049 /* Actual ASIC resets if needed.*/ 6050 /* Host driver will handle XGMI hive reset for SRIOV */ 6051 if (amdgpu_sriov_vf(adev)) { 6052 if (amdgpu_ras_get_fed_status(adev) || amdgpu_virt_rcvd_ras_interrupt(adev)) { 6053 dev_dbg(adev->dev, "Detected RAS error, wait for FLR completion\n"); 6054 amdgpu_ras_set_fed(adev, true); 6055 set_bit(AMDGPU_HOST_FLR, &reset_context->flags); 6056 } 6057 6058 r = amdgpu_device_reset_sriov(adev, reset_context); 6059 if (AMDGPU_RETRY_SRIOV_RESET(r) && (retry_limit--) > 0) { 6060 amdgpu_virt_release_full_gpu(adev, true); 6061 goto retry; 6062 } 6063 if (r) 6064 adev->asic_reset_res = r; 6065 } else { 6066 r = amdgpu_do_asic_reset(device_list_handle, reset_context); 6067 if (r && r == -EAGAIN) 6068 goto retry; 6069 } 6070 6071 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 6072 /* 6073 * Drop any pending non scheduler resets queued before reset is done. 6074 * Any reset scheduled after this point would be valid. Scheduler resets 6075 * were already dropped during drm_sched_stop and no new ones can come 6076 * in before drm_sched_start. 6077 */ 6078 amdgpu_device_stop_pending_resets(tmp_adev); 6079 } 6080 6081 skip_hw_reset: 6082 6083 /* Post ASIC reset for all devs .*/ 6084 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 6085 6086 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 6087 struct amdgpu_ring *ring = tmp_adev->rings[i]; 6088 6089 if (!amdgpu_ring_sched_ready(ring)) 6090 continue; 6091 6092 drm_sched_start(&ring->sched, 0); 6093 } 6094 6095 if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled) 6096 drm_helper_resume_force_mode(adev_to_drm(tmp_adev)); 6097 6098 if (tmp_adev->asic_reset_res) 6099 r = tmp_adev->asic_reset_res; 6100 6101 tmp_adev->asic_reset_res = 0; 6102 6103 if (r) { 6104 /* bad news, how to tell it to userspace ? 6105 * for ras error, we should report GPU bad status instead of 6106 * reset failure 6107 */ 6108 if (reset_context->src != AMDGPU_RESET_SRC_RAS || 6109 !amdgpu_ras_eeprom_check_err_threshold(tmp_adev)) 6110 dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", 6111 atomic_read(&tmp_adev->gpu_reset_counter)); 6112 amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r); 6113 } else { 6114 dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter)); 6115 if (amdgpu_acpi_smart_shift_update(adev_to_drm(tmp_adev), AMDGPU_SS_DEV_D0)) 6116 DRM_WARN("smart shift update failed\n"); 6117 } 6118 } 6119 6120 skip_sched_resume: 6121 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 6122 /* unlock kfd: SRIOV would do it separately */ 6123 if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev)) 6124 amdgpu_amdkfd_post_reset(tmp_adev); 6125 6126 /* kfd_post_reset will do nothing if kfd device is not initialized, 6127 * need to bring up kfd here if it's not be initialized before 6128 */ 6129 if (!adev->kfd.init_complete) 6130 amdgpu_amdkfd_device_init(adev); 6131 6132 if (audio_suspended) 6133 amdgpu_device_resume_display_audio(tmp_adev); 6134 6135 amdgpu_device_unset_mp1_state(tmp_adev); 6136 6137 amdgpu_ras_set_error_query_ready(tmp_adev, true); 6138 } 6139 6140 tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device, 6141 reset_list); 6142 amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain); 6143 6144 end_reset: 6145 if (hive) { 6146 mutex_unlock(&hive->hive_lock); 6147 amdgpu_put_xgmi_hive(hive); 6148 } 6149 6150 if (r) 6151 dev_info(adev->dev, "GPU reset end with ret = %d\n", r); 6152 6153 atomic_set(&adev->reset_domain->reset_res, r); 6154 return r; 6155 } 6156 6157 /** 6158 * amdgpu_device_partner_bandwidth - find the bandwidth of appropriate partner 6159 * 6160 * @adev: amdgpu_device pointer 6161 * @speed: pointer to the speed of the link 6162 * @width: pointer to the width of the link 6163 * 6164 * Evaluate the hierarchy to find the speed and bandwidth capabilities of the 6165 * first physical partner to an AMD dGPU. 6166 * This will exclude any virtual switches and links. 6167 */ 6168 static void amdgpu_device_partner_bandwidth(struct amdgpu_device *adev, 6169 enum pci_bus_speed *speed, 6170 enum pcie_link_width *width) 6171 { 6172 struct pci_dev *parent = adev->pdev; 6173 6174 if (!speed || !width) 6175 return; 6176 6177 *speed = PCI_SPEED_UNKNOWN; 6178 *width = PCIE_LNK_WIDTH_UNKNOWN; 6179 6180 if (amdgpu_device_pcie_dynamic_switching_supported(adev)) { 6181 while ((parent = pci_upstream_bridge(parent))) { 6182 /* skip upstream/downstream switches internal to dGPU*/ 6183 if (parent->vendor == PCI_VENDOR_ID_ATI) 6184 continue; 6185 *speed = pcie_get_speed_cap(parent); 6186 *width = pcie_get_width_cap(parent); 6187 break; 6188 } 6189 } else { 6190 /* use the current speeds rather than max if switching is not supported */ 6191 pcie_bandwidth_available(adev->pdev, NULL, speed, width); 6192 } 6193 } 6194 6195 /** 6196 * amdgpu_device_gpu_bandwidth - find the bandwidth of the GPU 6197 * 6198 * @adev: amdgpu_device pointer 6199 * @speed: pointer to the speed of the link 6200 * @width: pointer to the width of the link 6201 * 6202 * Evaluate the hierarchy to find the speed and bandwidth capabilities of the 6203 * AMD dGPU which may be a virtual upstream bridge. 6204 */ 6205 static void amdgpu_device_gpu_bandwidth(struct amdgpu_device *adev, 6206 enum pci_bus_speed *speed, 6207 enum pcie_link_width *width) 6208 { 6209 struct pci_dev *parent = adev->pdev; 6210 6211 if (!speed || !width) 6212 return; 6213 6214 parent = pci_upstream_bridge(parent); 6215 if (parent && parent->vendor == PCI_VENDOR_ID_ATI) { 6216 /* use the upstream/downstream switches internal to dGPU */ 6217 *speed = pcie_get_speed_cap(parent); 6218 *width = pcie_get_width_cap(parent); 6219 while ((parent = pci_upstream_bridge(parent))) { 6220 if (parent->vendor == PCI_VENDOR_ID_ATI) { 6221 /* use the upstream/downstream switches internal to dGPU */ 6222 *speed = pcie_get_speed_cap(parent); 6223 *width = pcie_get_width_cap(parent); 6224 } 6225 } 6226 } else { 6227 /* use the device itself */ 6228 *speed = pcie_get_speed_cap(adev->pdev); 6229 *width = pcie_get_width_cap(adev->pdev); 6230 } 6231 } 6232 6233 /** 6234 * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot 6235 * 6236 * @adev: amdgpu_device pointer 6237 * 6238 * Fetches and stores in the driver the PCIE capabilities (gen speed 6239 * and lanes) of the slot the device is in. Handles APUs and 6240 * virtualized environments where PCIE config space may not be available. 6241 */ 6242 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev) 6243 { 6244 enum pci_bus_speed speed_cap, platform_speed_cap; 6245 enum pcie_link_width platform_link_width, link_width; 6246 6247 if (amdgpu_pcie_gen_cap) 6248 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap; 6249 6250 if (amdgpu_pcie_lane_cap) 6251 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap; 6252 6253 /* covers APUs as well */ 6254 if (pci_is_root_bus(adev->pdev->bus) && !amdgpu_passthrough(adev)) { 6255 if (adev->pm.pcie_gen_mask == 0) 6256 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK; 6257 if (adev->pm.pcie_mlw_mask == 0) 6258 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK; 6259 return; 6260 } 6261 6262 if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask) 6263 return; 6264 6265 amdgpu_device_partner_bandwidth(adev, &platform_speed_cap, 6266 &platform_link_width); 6267 amdgpu_device_gpu_bandwidth(adev, &speed_cap, &link_width); 6268 6269 if (adev->pm.pcie_gen_mask == 0) { 6270 /* asic caps */ 6271 if (speed_cap == PCI_SPEED_UNKNOWN) { 6272 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6273 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 6274 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3); 6275 } else { 6276 if (speed_cap == PCIE_SPEED_32_0GT) 6277 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6278 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 6279 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 | 6280 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 | 6281 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5); 6282 else if (speed_cap == PCIE_SPEED_16_0GT) 6283 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6284 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 6285 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 | 6286 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4); 6287 else if (speed_cap == PCIE_SPEED_8_0GT) 6288 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6289 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 6290 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3); 6291 else if (speed_cap == PCIE_SPEED_5_0GT) 6292 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6293 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2); 6294 else 6295 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1; 6296 } 6297 /* platform caps */ 6298 if (platform_speed_cap == PCI_SPEED_UNKNOWN) { 6299 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6300 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2); 6301 } else { 6302 if (platform_speed_cap == PCIE_SPEED_32_0GT) 6303 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6304 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | 6305 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 | 6306 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 | 6307 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5); 6308 else if (platform_speed_cap == PCIE_SPEED_16_0GT) 6309 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6310 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | 6311 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 | 6312 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4); 6313 else if (platform_speed_cap == PCIE_SPEED_8_0GT) 6314 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6315 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | 6316 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3); 6317 else if (platform_speed_cap == PCIE_SPEED_5_0GT) 6318 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6319 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2); 6320 else 6321 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1; 6322 6323 } 6324 } 6325 if (adev->pm.pcie_mlw_mask == 0) { 6326 /* asic caps */ 6327 if (link_width == PCIE_LNK_WIDTH_UNKNOWN) { 6328 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_ASIC_PCIE_MLW_MASK; 6329 } else { 6330 switch (link_width) { 6331 case PCIE_LNK_X32: 6332 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X32 | 6333 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X16 | 6334 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X12 | 6335 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 | 6336 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 | 6337 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 | 6338 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1); 6339 break; 6340 case PCIE_LNK_X16: 6341 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X16 | 6342 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X12 | 6343 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 | 6344 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 | 6345 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 | 6346 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1); 6347 break; 6348 case PCIE_LNK_X12: 6349 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X12 | 6350 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 | 6351 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 | 6352 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 | 6353 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1); 6354 break; 6355 case PCIE_LNK_X8: 6356 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 | 6357 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 | 6358 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 | 6359 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1); 6360 break; 6361 case PCIE_LNK_X4: 6362 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 | 6363 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 | 6364 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1); 6365 break; 6366 case PCIE_LNK_X2: 6367 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 | 6368 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1); 6369 break; 6370 case PCIE_LNK_X1: 6371 adev->pm.pcie_mlw_mask |= CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1; 6372 break; 6373 default: 6374 break; 6375 } 6376 } 6377 /* platform caps */ 6378 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) { 6379 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK; 6380 } else { 6381 switch (platform_link_width) { 6382 case PCIE_LNK_X32: 6383 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 | 6384 CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 | 6385 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | 6386 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 6387 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 6388 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 6389 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 6390 break; 6391 case PCIE_LNK_X16: 6392 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 | 6393 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | 6394 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 6395 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 6396 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 6397 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 6398 break; 6399 case PCIE_LNK_X12: 6400 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | 6401 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 6402 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 6403 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 6404 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 6405 break; 6406 case PCIE_LNK_X8: 6407 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 6408 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 6409 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 6410 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 6411 break; 6412 case PCIE_LNK_X4: 6413 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 6414 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 6415 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 6416 break; 6417 case PCIE_LNK_X2: 6418 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 6419 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 6420 break; 6421 case PCIE_LNK_X1: 6422 adev->pm.pcie_mlw_mask |= CAIL_PCIE_LINK_WIDTH_SUPPORT_X1; 6423 break; 6424 default: 6425 break; 6426 } 6427 } 6428 } 6429 } 6430 6431 /** 6432 * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR 6433 * 6434 * @adev: amdgpu_device pointer 6435 * @peer_adev: amdgpu_device pointer for peer device trying to access @adev 6436 * 6437 * Return true if @peer_adev can access (DMA) @adev through the PCIe 6438 * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of 6439 * @peer_adev. 6440 */ 6441 bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev, 6442 struct amdgpu_device *peer_adev) 6443 { 6444 #ifdef CONFIG_HSA_AMD_P2P 6445 bool p2p_access = 6446 !adev->gmc.xgmi.connected_to_cpu && 6447 !(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0); 6448 if (!p2p_access) 6449 dev_info(adev->dev, "PCIe P2P access from peer device %s is not supported by the chipset\n", 6450 pci_name(peer_adev->pdev)); 6451 6452 bool is_large_bar = adev->gmc.visible_vram_size && 6453 adev->gmc.real_vram_size == adev->gmc.visible_vram_size; 6454 bool p2p_addressable = amdgpu_device_check_iommu_remap(peer_adev); 6455 6456 if (!p2p_addressable) { 6457 uint64_t address_mask = peer_adev->dev->dma_mask ? 6458 ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1); 6459 resource_size_t aper_limit = 6460 adev->gmc.aper_base + adev->gmc.aper_size - 1; 6461 6462 p2p_addressable = !(adev->gmc.aper_base & address_mask || 6463 aper_limit & address_mask); 6464 } 6465 return pcie_p2p && is_large_bar && p2p_access && p2p_addressable; 6466 #else 6467 return false; 6468 #endif 6469 } 6470 6471 int amdgpu_device_baco_enter(struct drm_device *dev) 6472 { 6473 struct amdgpu_device *adev = drm_to_adev(dev); 6474 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 6475 6476 if (!amdgpu_device_supports_baco(dev)) 6477 return -ENOTSUPP; 6478 6479 if (ras && adev->ras_enabled && 6480 adev->nbio.funcs->enable_doorbell_interrupt) 6481 adev->nbio.funcs->enable_doorbell_interrupt(adev, false); 6482 6483 return amdgpu_dpm_baco_enter(adev); 6484 } 6485 6486 int amdgpu_device_baco_exit(struct drm_device *dev) 6487 { 6488 struct amdgpu_device *adev = drm_to_adev(dev); 6489 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 6490 int ret = 0; 6491 6492 if (!amdgpu_device_supports_baco(dev)) 6493 return -ENOTSUPP; 6494 6495 ret = amdgpu_dpm_baco_exit(adev); 6496 if (ret) 6497 return ret; 6498 6499 if (ras && adev->ras_enabled && 6500 adev->nbio.funcs->enable_doorbell_interrupt) 6501 adev->nbio.funcs->enable_doorbell_interrupt(adev, true); 6502 6503 if (amdgpu_passthrough(adev) && adev->nbio.funcs && 6504 adev->nbio.funcs->clear_doorbell_interrupt) 6505 adev->nbio.funcs->clear_doorbell_interrupt(adev); 6506 6507 return 0; 6508 } 6509 6510 /** 6511 * amdgpu_pci_error_detected - Called when a PCI error is detected. 6512 * @pdev: PCI device struct 6513 * @state: PCI channel state 6514 * 6515 * Description: Called when a PCI error is detected. 6516 * 6517 * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT. 6518 */ 6519 pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state) 6520 { 6521 struct drm_device *dev = pci_get_drvdata(pdev); 6522 struct amdgpu_device *adev = drm_to_adev(dev); 6523 int i; 6524 6525 DRM_INFO("PCI error: detected callback, state(%d)!!\n", state); 6526 6527 if (adev->gmc.xgmi.num_physical_nodes > 1) { 6528 DRM_WARN("No support for XGMI hive yet..."); 6529 return PCI_ERS_RESULT_DISCONNECT; 6530 } 6531 6532 adev->pci_channel_state = state; 6533 6534 switch (state) { 6535 case pci_channel_io_normal: 6536 return PCI_ERS_RESULT_CAN_RECOVER; 6537 /* Fatal error, prepare for slot reset */ 6538 case pci_channel_io_frozen: 6539 /* 6540 * Locking adev->reset_domain->sem will prevent any external access 6541 * to GPU during PCI error recovery 6542 */ 6543 amdgpu_device_lock_reset_domain(adev->reset_domain); 6544 amdgpu_device_set_mp1_state(adev); 6545 6546 /* 6547 * Block any work scheduling as we do for regular GPU reset 6548 * for the duration of the recovery 6549 */ 6550 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 6551 struct amdgpu_ring *ring = adev->rings[i]; 6552 6553 if (!amdgpu_ring_sched_ready(ring)) 6554 continue; 6555 6556 drm_sched_stop(&ring->sched, NULL); 6557 } 6558 atomic_inc(&adev->gpu_reset_counter); 6559 return PCI_ERS_RESULT_NEED_RESET; 6560 case pci_channel_io_perm_failure: 6561 /* Permanent error, prepare for device removal */ 6562 return PCI_ERS_RESULT_DISCONNECT; 6563 } 6564 6565 return PCI_ERS_RESULT_NEED_RESET; 6566 } 6567 6568 /** 6569 * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers 6570 * @pdev: pointer to PCI device 6571 */ 6572 pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev) 6573 { 6574 6575 DRM_INFO("PCI error: mmio enabled callback!!\n"); 6576 6577 /* TODO - dump whatever for debugging purposes */ 6578 6579 /* This called only if amdgpu_pci_error_detected returns 6580 * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still 6581 * works, no need to reset slot. 6582 */ 6583 6584 return PCI_ERS_RESULT_RECOVERED; 6585 } 6586 6587 /** 6588 * amdgpu_pci_slot_reset - Called when PCI slot has been reset. 6589 * @pdev: PCI device struct 6590 * 6591 * Description: This routine is called by the pci error recovery 6592 * code after the PCI slot has been reset, just before we 6593 * should resume normal operations. 6594 */ 6595 pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev) 6596 { 6597 struct drm_device *dev = pci_get_drvdata(pdev); 6598 struct amdgpu_device *adev = drm_to_adev(dev); 6599 int r, i; 6600 struct amdgpu_reset_context reset_context; 6601 u32 memsize; 6602 struct list_head device_list; 6603 6604 /* PCI error slot reset should be skipped During RAS recovery */ 6605 if ((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) || 6606 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4)) && 6607 amdgpu_ras_in_recovery(adev)) 6608 return PCI_ERS_RESULT_RECOVERED; 6609 6610 DRM_INFO("PCI error: slot reset callback!!\n"); 6611 6612 memset(&reset_context, 0, sizeof(reset_context)); 6613 6614 INIT_LIST_HEAD(&device_list); 6615 list_add_tail(&adev->reset_list, &device_list); 6616 6617 /* wait for asic to come out of reset */ 6618 msleep(500); 6619 6620 /* Restore PCI confspace */ 6621 amdgpu_device_load_pci_state(pdev); 6622 6623 /* confirm ASIC came out of reset */ 6624 for (i = 0; i < adev->usec_timeout; i++) { 6625 memsize = amdgpu_asic_get_config_memsize(adev); 6626 6627 if (memsize != 0xffffffff) 6628 break; 6629 udelay(1); 6630 } 6631 if (memsize == 0xffffffff) { 6632 r = -ETIME; 6633 goto out; 6634 } 6635 6636 reset_context.method = AMD_RESET_METHOD_NONE; 6637 reset_context.reset_req_dev = adev; 6638 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 6639 set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags); 6640 6641 adev->no_hw_access = true; 6642 r = amdgpu_device_pre_asic_reset(adev, &reset_context); 6643 adev->no_hw_access = false; 6644 if (r) 6645 goto out; 6646 6647 r = amdgpu_do_asic_reset(&device_list, &reset_context); 6648 6649 out: 6650 if (!r) { 6651 if (amdgpu_device_cache_pci_state(adev->pdev)) 6652 pci_restore_state(adev->pdev); 6653 6654 DRM_INFO("PCIe error recovery succeeded\n"); 6655 } else { 6656 DRM_ERROR("PCIe error recovery failed, err:%d", r); 6657 amdgpu_device_unset_mp1_state(adev); 6658 amdgpu_device_unlock_reset_domain(adev->reset_domain); 6659 } 6660 6661 return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED; 6662 } 6663 6664 /** 6665 * amdgpu_pci_resume() - resume normal ops after PCI reset 6666 * @pdev: pointer to PCI device 6667 * 6668 * Called when the error recovery driver tells us that its 6669 * OK to resume normal operation. 6670 */ 6671 void amdgpu_pci_resume(struct pci_dev *pdev) 6672 { 6673 struct drm_device *dev = pci_get_drvdata(pdev); 6674 struct amdgpu_device *adev = drm_to_adev(dev); 6675 int i; 6676 6677 6678 DRM_INFO("PCI error: resume callback!!\n"); 6679 6680 /* Only continue execution for the case of pci_channel_io_frozen */ 6681 if (adev->pci_channel_state != pci_channel_io_frozen) 6682 return; 6683 6684 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 6685 struct amdgpu_ring *ring = adev->rings[i]; 6686 6687 if (!amdgpu_ring_sched_ready(ring)) 6688 continue; 6689 6690 drm_sched_start(&ring->sched, 0); 6691 } 6692 6693 amdgpu_device_unset_mp1_state(adev); 6694 amdgpu_device_unlock_reset_domain(adev->reset_domain); 6695 } 6696 6697 bool amdgpu_device_cache_pci_state(struct pci_dev *pdev) 6698 { 6699 struct drm_device *dev = pci_get_drvdata(pdev); 6700 struct amdgpu_device *adev = drm_to_adev(dev); 6701 int r; 6702 6703 if (amdgpu_sriov_vf(adev)) 6704 return false; 6705 6706 r = pci_save_state(pdev); 6707 if (!r) { 6708 kfree(adev->pci_state); 6709 6710 adev->pci_state = pci_store_saved_state(pdev); 6711 6712 if (!adev->pci_state) { 6713 DRM_ERROR("Failed to store PCI saved state"); 6714 return false; 6715 } 6716 } else { 6717 DRM_WARN("Failed to save PCI state, err:%d\n", r); 6718 return false; 6719 } 6720 6721 return true; 6722 } 6723 6724 bool amdgpu_device_load_pci_state(struct pci_dev *pdev) 6725 { 6726 struct drm_device *dev = pci_get_drvdata(pdev); 6727 struct amdgpu_device *adev = drm_to_adev(dev); 6728 int r; 6729 6730 if (!adev->pci_state) 6731 return false; 6732 6733 r = pci_load_saved_state(pdev, adev->pci_state); 6734 6735 if (!r) { 6736 pci_restore_state(pdev); 6737 } else { 6738 DRM_WARN("Failed to load PCI state, err:%d\n", r); 6739 return false; 6740 } 6741 6742 return true; 6743 } 6744 6745 void amdgpu_device_flush_hdp(struct amdgpu_device *adev, 6746 struct amdgpu_ring *ring) 6747 { 6748 #ifdef CONFIG_X86_64 6749 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) 6750 return; 6751 #endif 6752 if (adev->gmc.xgmi.connected_to_cpu) 6753 return; 6754 6755 if (ring && ring->funcs->emit_hdp_flush) 6756 amdgpu_ring_emit_hdp_flush(ring); 6757 else 6758 amdgpu_asic_flush_hdp(adev, ring); 6759 } 6760 6761 void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev, 6762 struct amdgpu_ring *ring) 6763 { 6764 #ifdef CONFIG_X86_64 6765 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) 6766 return; 6767 #endif 6768 if (adev->gmc.xgmi.connected_to_cpu) 6769 return; 6770 6771 amdgpu_asic_invalidate_hdp(adev, ring); 6772 } 6773 6774 int amdgpu_in_reset(struct amdgpu_device *adev) 6775 { 6776 return atomic_read(&adev->reset_domain->in_gpu_reset); 6777 } 6778 6779 /** 6780 * amdgpu_device_halt() - bring hardware to some kind of halt state 6781 * 6782 * @adev: amdgpu_device pointer 6783 * 6784 * Bring hardware to some kind of halt state so that no one can touch it 6785 * any more. It will help to maintain error context when error occurred. 6786 * Compare to a simple hang, the system will keep stable at least for SSH 6787 * access. Then it should be trivial to inspect the hardware state and 6788 * see what's going on. Implemented as following: 6789 * 6790 * 1. drm_dev_unplug() makes device inaccessible to user space(IOCTLs, etc), 6791 * clears all CPU mappings to device, disallows remappings through page faults 6792 * 2. amdgpu_irq_disable_all() disables all interrupts 6793 * 3. amdgpu_fence_driver_hw_fini() signals all HW fences 6794 * 4. set adev->no_hw_access to avoid potential crashes after setp 5 6795 * 5. amdgpu_device_unmap_mmio() clears all MMIO mappings 6796 * 6. pci_disable_device() and pci_wait_for_pending_transaction() 6797 * flush any in flight DMA operations 6798 */ 6799 void amdgpu_device_halt(struct amdgpu_device *adev) 6800 { 6801 struct pci_dev *pdev = adev->pdev; 6802 struct drm_device *ddev = adev_to_drm(adev); 6803 6804 amdgpu_xcp_dev_unplug(adev); 6805 drm_dev_unplug(ddev); 6806 6807 amdgpu_irq_disable_all(adev); 6808 6809 amdgpu_fence_driver_hw_fini(adev); 6810 6811 adev->no_hw_access = true; 6812 6813 amdgpu_device_unmap_mmio(adev); 6814 6815 pci_disable_device(pdev); 6816 pci_wait_for_pending_transaction(pdev); 6817 } 6818 6819 u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev, 6820 u32 reg) 6821 { 6822 unsigned long flags, address, data; 6823 u32 r; 6824 6825 address = adev->nbio.funcs->get_pcie_port_index_offset(adev); 6826 data = adev->nbio.funcs->get_pcie_port_data_offset(adev); 6827 6828 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 6829 WREG32(address, reg * 4); 6830 (void)RREG32(address); 6831 r = RREG32(data); 6832 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 6833 return r; 6834 } 6835 6836 void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev, 6837 u32 reg, u32 v) 6838 { 6839 unsigned long flags, address, data; 6840 6841 address = adev->nbio.funcs->get_pcie_port_index_offset(adev); 6842 data = adev->nbio.funcs->get_pcie_port_data_offset(adev); 6843 6844 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 6845 WREG32(address, reg * 4); 6846 (void)RREG32(address); 6847 WREG32(data, v); 6848 (void)RREG32(data); 6849 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 6850 } 6851 6852 /** 6853 * amdgpu_device_get_gang - return a reference to the current gang 6854 * @adev: amdgpu_device pointer 6855 * 6856 * Returns: A new reference to the current gang leader. 6857 */ 6858 struct dma_fence *amdgpu_device_get_gang(struct amdgpu_device *adev) 6859 { 6860 struct dma_fence *fence; 6861 6862 rcu_read_lock(); 6863 fence = dma_fence_get_rcu_safe(&adev->gang_submit); 6864 rcu_read_unlock(); 6865 return fence; 6866 } 6867 6868 /** 6869 * amdgpu_device_switch_gang - switch to a new gang 6870 * @adev: amdgpu_device pointer 6871 * @gang: the gang to switch to 6872 * 6873 * Try to switch to a new gang. 6874 * Returns: NULL if we switched to the new gang or a reference to the current 6875 * gang leader. 6876 */ 6877 struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev, 6878 struct dma_fence *gang) 6879 { 6880 struct dma_fence *old = NULL; 6881 6882 do { 6883 dma_fence_put(old); 6884 old = amdgpu_device_get_gang(adev); 6885 if (old == gang) 6886 break; 6887 6888 if (!dma_fence_is_signaled(old)) 6889 return old; 6890 6891 } while (cmpxchg((struct dma_fence __force **)&adev->gang_submit, 6892 old, gang) != old); 6893 6894 dma_fence_put(old); 6895 return NULL; 6896 } 6897 6898 bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev) 6899 { 6900 switch (adev->asic_type) { 6901 #ifdef CONFIG_DRM_AMDGPU_SI 6902 case CHIP_HAINAN: 6903 #endif 6904 case CHIP_TOPAZ: 6905 /* chips with no display hardware */ 6906 return false; 6907 #ifdef CONFIG_DRM_AMDGPU_SI 6908 case CHIP_TAHITI: 6909 case CHIP_PITCAIRN: 6910 case CHIP_VERDE: 6911 case CHIP_OLAND: 6912 #endif 6913 #ifdef CONFIG_DRM_AMDGPU_CIK 6914 case CHIP_BONAIRE: 6915 case CHIP_HAWAII: 6916 case CHIP_KAVERI: 6917 case CHIP_KABINI: 6918 case CHIP_MULLINS: 6919 #endif 6920 case CHIP_TONGA: 6921 case CHIP_FIJI: 6922 case CHIP_POLARIS10: 6923 case CHIP_POLARIS11: 6924 case CHIP_POLARIS12: 6925 case CHIP_VEGAM: 6926 case CHIP_CARRIZO: 6927 case CHIP_STONEY: 6928 /* chips with display hardware */ 6929 return true; 6930 default: 6931 /* IP discovery */ 6932 if (!amdgpu_ip_version(adev, DCE_HWIP, 0) || 6933 (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK)) 6934 return false; 6935 return true; 6936 } 6937 } 6938 6939 uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev, 6940 uint32_t inst, uint32_t reg_addr, char reg_name[], 6941 uint32_t expected_value, uint32_t mask) 6942 { 6943 uint32_t ret = 0; 6944 uint32_t old_ = 0; 6945 uint32_t tmp_ = RREG32(reg_addr); 6946 uint32_t loop = adev->usec_timeout; 6947 6948 while ((tmp_ & (mask)) != (expected_value)) { 6949 if (old_ != tmp_) { 6950 loop = adev->usec_timeout; 6951 old_ = tmp_; 6952 } else 6953 udelay(1); 6954 tmp_ = RREG32(reg_addr); 6955 loop--; 6956 if (!loop) { 6957 DRM_WARN("Register(%d) [%s] failed to reach value 0x%08x != 0x%08xn", 6958 inst, reg_name, (uint32_t)expected_value, 6959 (uint32_t)(tmp_ & (mask))); 6960 ret = -ETIMEDOUT; 6961 break; 6962 } 6963 } 6964 return ret; 6965 } 6966 6967 ssize_t amdgpu_get_soft_full_reset_mask(struct amdgpu_ring *ring) 6968 { 6969 ssize_t size = 0; 6970 6971 if (!ring || !ring->adev) 6972 return size; 6973 6974 if (amdgpu_device_should_recover_gpu(ring->adev)) 6975 size |= AMDGPU_RESET_TYPE_FULL; 6976 6977 if (unlikely(!ring->adev->debug_disable_soft_recovery) && 6978 !amdgpu_sriov_vf(ring->adev) && ring->funcs->soft_recovery) 6979 size |= AMDGPU_RESET_TYPE_SOFT_RESET; 6980 6981 return size; 6982 } 6983 6984 ssize_t amdgpu_show_reset_mask(char *buf, uint32_t supported_reset) 6985 { 6986 ssize_t size = 0; 6987 6988 if (supported_reset == 0) { 6989 size += sysfs_emit_at(buf, size, "unsupported"); 6990 size += sysfs_emit_at(buf, size, "\n"); 6991 return size; 6992 6993 } 6994 6995 if (supported_reset & AMDGPU_RESET_TYPE_SOFT_RESET) 6996 size += sysfs_emit_at(buf, size, "soft "); 6997 6998 if (supported_reset & AMDGPU_RESET_TYPE_PER_QUEUE) 6999 size += sysfs_emit_at(buf, size, "queue "); 7000 7001 if (supported_reset & AMDGPU_RESET_TYPE_PER_PIPE) 7002 size += sysfs_emit_at(buf, size, "pipe "); 7003 7004 if (supported_reset & AMDGPU_RESET_TYPE_FULL) 7005 size += sysfs_emit_at(buf, size, "full "); 7006 7007 size += sysfs_emit_at(buf, size, "\n"); 7008 return size; 7009 } 7010