1 /* 2 * Copyright 2008 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: Dave Airlie 25 * Alex Deucher 26 * Jerome Glisse 27 */ 28 29 #include <linux/aperture.h> 30 #include <linux/power_supply.h> 31 #include <linux/kthread.h> 32 #include <linux/module.h> 33 #include <linux/console.h> 34 #include <linux/slab.h> 35 #include <linux/iommu.h> 36 #include <linux/pci.h> 37 #include <linux/pci-p2pdma.h> 38 #include <linux/apple-gmux.h> 39 40 #include <drm/drm_atomic_helper.h> 41 #include <drm/drm_client_event.h> 42 #include <drm/drm_crtc_helper.h> 43 #include <drm/drm_probe_helper.h> 44 #include <drm/amdgpu_drm.h> 45 #include <linux/device.h> 46 #include <linux/vgaarb.h> 47 #include <linux/vga_switcheroo.h> 48 #include <linux/efi.h> 49 #include "amdgpu.h" 50 #include "amdgpu_trace.h" 51 #include "amdgpu_i2c.h" 52 #include "atom.h" 53 #include "amdgpu_atombios.h" 54 #include "amdgpu_atomfirmware.h" 55 #include "amd_pcie.h" 56 #ifdef CONFIG_DRM_AMDGPU_SI 57 #include "si.h" 58 #endif 59 #ifdef CONFIG_DRM_AMDGPU_CIK 60 #include "cik.h" 61 #endif 62 #include "vi.h" 63 #include "soc15.h" 64 #include "nv.h" 65 #include "bif/bif_4_1_d.h" 66 #include <linux/firmware.h> 67 #include "amdgpu_vf_error.h" 68 69 #include "amdgpu_amdkfd.h" 70 #include "amdgpu_pm.h" 71 72 #include "amdgpu_xgmi.h" 73 #include "amdgpu_ras.h" 74 #include "amdgpu_pmu.h" 75 #include "amdgpu_fru_eeprom.h" 76 #include "amdgpu_reset.h" 77 #include "amdgpu_virt.h" 78 #include "amdgpu_dev_coredump.h" 79 80 #include <linux/suspend.h> 81 #include <drm/task_barrier.h> 82 #include <linux/pm_runtime.h> 83 84 #include <drm/drm_drv.h> 85 86 #if IS_ENABLED(CONFIG_X86) 87 #include <asm/intel-family.h> 88 #endif 89 90 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin"); 91 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin"); 92 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin"); 93 MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin"); 94 MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin"); 95 MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin"); 96 MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin"); 97 98 #define AMDGPU_RESUME_MS 2000 99 #define AMDGPU_MAX_RETRY_LIMIT 2 100 #define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL) 101 #define AMDGPU_PCIE_INDEX_FALLBACK (0x38 >> 2) 102 #define AMDGPU_PCIE_INDEX_HI_FALLBACK (0x44 >> 2) 103 #define AMDGPU_PCIE_DATA_FALLBACK (0x3C >> 2) 104 105 #define AMDGPU_VBIOS_SKIP (1U << 0) 106 #define AMDGPU_VBIOS_OPTIONAL (1U << 1) 107 108 static const struct drm_driver amdgpu_kms_driver; 109 110 const char *amdgpu_asic_name[] = { 111 "TAHITI", 112 "PITCAIRN", 113 "VERDE", 114 "OLAND", 115 "HAINAN", 116 "BONAIRE", 117 "KAVERI", 118 "KABINI", 119 "HAWAII", 120 "MULLINS", 121 "TOPAZ", 122 "TONGA", 123 "FIJI", 124 "CARRIZO", 125 "STONEY", 126 "POLARIS10", 127 "POLARIS11", 128 "POLARIS12", 129 "VEGAM", 130 "VEGA10", 131 "VEGA12", 132 "VEGA20", 133 "RAVEN", 134 "ARCTURUS", 135 "RENOIR", 136 "ALDEBARAN", 137 "NAVI10", 138 "CYAN_SKILLFISH", 139 "NAVI14", 140 "NAVI12", 141 "SIENNA_CICHLID", 142 "NAVY_FLOUNDER", 143 "VANGOGH", 144 "DIMGREY_CAVEFISH", 145 "BEIGE_GOBY", 146 "YELLOW_CARP", 147 "IP DISCOVERY", 148 "LAST", 149 }; 150 151 #define AMDGPU_IP_BLK_MASK_ALL GENMASK(AMD_IP_BLOCK_TYPE_NUM - 1, 0) 152 /* 153 * Default init level where all blocks are expected to be initialized. This is 154 * the level of initialization expected by default and also after a full reset 155 * of the device. 156 */ 157 struct amdgpu_init_level amdgpu_init_default = { 158 .level = AMDGPU_INIT_LEVEL_DEFAULT, 159 .hwini_ip_block_mask = AMDGPU_IP_BLK_MASK_ALL, 160 }; 161 162 struct amdgpu_init_level amdgpu_init_recovery = { 163 .level = AMDGPU_INIT_LEVEL_RESET_RECOVERY, 164 .hwini_ip_block_mask = AMDGPU_IP_BLK_MASK_ALL, 165 }; 166 167 /* 168 * Minimal blocks needed to be initialized before a XGMI hive can be reset. This 169 * is used for cases like reset on initialization where the entire hive needs to 170 * be reset before first use. 171 */ 172 struct amdgpu_init_level amdgpu_init_minimal_xgmi = { 173 .level = AMDGPU_INIT_LEVEL_MINIMAL_XGMI, 174 .hwini_ip_block_mask = 175 BIT(AMD_IP_BLOCK_TYPE_GMC) | BIT(AMD_IP_BLOCK_TYPE_SMC) | 176 BIT(AMD_IP_BLOCK_TYPE_COMMON) | BIT(AMD_IP_BLOCK_TYPE_IH) | 177 BIT(AMD_IP_BLOCK_TYPE_PSP) 178 }; 179 180 static inline bool amdgpu_ip_member_of_hwini(struct amdgpu_device *adev, 181 enum amd_ip_block_type block) 182 { 183 return (adev->init_lvl->hwini_ip_block_mask & (1U << block)) != 0; 184 } 185 186 void amdgpu_set_init_level(struct amdgpu_device *adev, 187 enum amdgpu_init_lvl_id lvl) 188 { 189 switch (lvl) { 190 case AMDGPU_INIT_LEVEL_MINIMAL_XGMI: 191 adev->init_lvl = &amdgpu_init_minimal_xgmi; 192 break; 193 case AMDGPU_INIT_LEVEL_RESET_RECOVERY: 194 adev->init_lvl = &amdgpu_init_recovery; 195 break; 196 case AMDGPU_INIT_LEVEL_DEFAULT: 197 fallthrough; 198 default: 199 adev->init_lvl = &amdgpu_init_default; 200 break; 201 } 202 } 203 204 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev); 205 static int amdgpu_device_pm_notifier(struct notifier_block *nb, unsigned long mode, 206 void *data); 207 208 /** 209 * DOC: pcie_replay_count 210 * 211 * The amdgpu driver provides a sysfs API for reporting the total number 212 * of PCIe replays (NAKs). 213 * The file pcie_replay_count is used for this and returns the total 214 * number of replays as a sum of the NAKs generated and NAKs received. 215 */ 216 217 static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev, 218 struct device_attribute *attr, char *buf) 219 { 220 struct drm_device *ddev = dev_get_drvdata(dev); 221 struct amdgpu_device *adev = drm_to_adev(ddev); 222 uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev); 223 224 return sysfs_emit(buf, "%llu\n", cnt); 225 } 226 227 static DEVICE_ATTR(pcie_replay_count, 0444, 228 amdgpu_device_get_pcie_replay_count, NULL); 229 230 static ssize_t amdgpu_sysfs_reg_state_get(struct file *f, struct kobject *kobj, 231 struct bin_attribute *attr, char *buf, 232 loff_t ppos, size_t count) 233 { 234 struct device *dev = kobj_to_dev(kobj); 235 struct drm_device *ddev = dev_get_drvdata(dev); 236 struct amdgpu_device *adev = drm_to_adev(ddev); 237 ssize_t bytes_read; 238 239 switch (ppos) { 240 case AMDGPU_SYS_REG_STATE_XGMI: 241 bytes_read = amdgpu_asic_get_reg_state( 242 adev, AMDGPU_REG_STATE_TYPE_XGMI, buf, count); 243 break; 244 case AMDGPU_SYS_REG_STATE_WAFL: 245 bytes_read = amdgpu_asic_get_reg_state( 246 adev, AMDGPU_REG_STATE_TYPE_WAFL, buf, count); 247 break; 248 case AMDGPU_SYS_REG_STATE_PCIE: 249 bytes_read = amdgpu_asic_get_reg_state( 250 adev, AMDGPU_REG_STATE_TYPE_PCIE, buf, count); 251 break; 252 case AMDGPU_SYS_REG_STATE_USR: 253 bytes_read = amdgpu_asic_get_reg_state( 254 adev, AMDGPU_REG_STATE_TYPE_USR, buf, count); 255 break; 256 case AMDGPU_SYS_REG_STATE_USR_1: 257 bytes_read = amdgpu_asic_get_reg_state( 258 adev, AMDGPU_REG_STATE_TYPE_USR_1, buf, count); 259 break; 260 default: 261 return -EINVAL; 262 } 263 264 return bytes_read; 265 } 266 267 BIN_ATTR(reg_state, 0444, amdgpu_sysfs_reg_state_get, NULL, 268 AMDGPU_SYS_REG_STATE_END); 269 270 int amdgpu_reg_state_sysfs_init(struct amdgpu_device *adev) 271 { 272 int ret; 273 274 if (!amdgpu_asic_get_reg_state_supported(adev)) 275 return 0; 276 277 ret = sysfs_create_bin_file(&adev->dev->kobj, &bin_attr_reg_state); 278 279 return ret; 280 } 281 282 void amdgpu_reg_state_sysfs_fini(struct amdgpu_device *adev) 283 { 284 if (!amdgpu_asic_get_reg_state_supported(adev)) 285 return; 286 sysfs_remove_bin_file(&adev->dev->kobj, &bin_attr_reg_state); 287 } 288 289 int amdgpu_ip_block_suspend(struct amdgpu_ip_block *ip_block) 290 { 291 int r; 292 293 if (ip_block->version->funcs->suspend) { 294 r = ip_block->version->funcs->suspend(ip_block); 295 if (r) { 296 dev_err(ip_block->adev->dev, 297 "suspend of IP block <%s> failed %d\n", 298 ip_block->version->funcs->name, r); 299 return r; 300 } 301 } 302 303 ip_block->status.hw = false; 304 return 0; 305 } 306 307 int amdgpu_ip_block_resume(struct amdgpu_ip_block *ip_block) 308 { 309 int r; 310 311 if (ip_block->version->funcs->resume) { 312 r = ip_block->version->funcs->resume(ip_block); 313 if (r) { 314 dev_err(ip_block->adev->dev, 315 "resume of IP block <%s> failed %d\n", 316 ip_block->version->funcs->name, r); 317 return r; 318 } 319 } 320 321 ip_block->status.hw = true; 322 return 0; 323 } 324 325 /** 326 * DOC: board_info 327 * 328 * The amdgpu driver provides a sysfs API for giving board related information. 329 * It provides the form factor information in the format 330 * 331 * type : form factor 332 * 333 * Possible form factor values 334 * 335 * - "cem" - PCIE CEM card 336 * - "oam" - Open Compute Accelerator Module 337 * - "unknown" - Not known 338 * 339 */ 340 341 static ssize_t amdgpu_device_get_board_info(struct device *dev, 342 struct device_attribute *attr, 343 char *buf) 344 { 345 struct drm_device *ddev = dev_get_drvdata(dev); 346 struct amdgpu_device *adev = drm_to_adev(ddev); 347 enum amdgpu_pkg_type pkg_type = AMDGPU_PKG_TYPE_CEM; 348 const char *pkg; 349 350 if (adev->smuio.funcs && adev->smuio.funcs->get_pkg_type) 351 pkg_type = adev->smuio.funcs->get_pkg_type(adev); 352 353 switch (pkg_type) { 354 case AMDGPU_PKG_TYPE_CEM: 355 pkg = "cem"; 356 break; 357 case AMDGPU_PKG_TYPE_OAM: 358 pkg = "oam"; 359 break; 360 default: 361 pkg = "unknown"; 362 break; 363 } 364 365 return sysfs_emit(buf, "%s : %s\n", "type", pkg); 366 } 367 368 static DEVICE_ATTR(board_info, 0444, amdgpu_device_get_board_info, NULL); 369 370 static struct attribute *amdgpu_board_attrs[] = { 371 &dev_attr_board_info.attr, 372 NULL, 373 }; 374 375 static umode_t amdgpu_board_attrs_is_visible(struct kobject *kobj, 376 struct attribute *attr, int n) 377 { 378 struct device *dev = kobj_to_dev(kobj); 379 struct drm_device *ddev = dev_get_drvdata(dev); 380 struct amdgpu_device *adev = drm_to_adev(ddev); 381 382 if (adev->flags & AMD_IS_APU) 383 return 0; 384 385 return attr->mode; 386 } 387 388 static const struct attribute_group amdgpu_board_attrs_group = { 389 .attrs = amdgpu_board_attrs, 390 .is_visible = amdgpu_board_attrs_is_visible 391 }; 392 393 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev); 394 395 396 /** 397 * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control 398 * 399 * @dev: drm_device pointer 400 * 401 * Returns true if the device is a dGPU with ATPX power control, 402 * otherwise return false. 403 */ 404 bool amdgpu_device_supports_px(struct drm_device *dev) 405 { 406 struct amdgpu_device *adev = drm_to_adev(dev); 407 408 if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid()) 409 return true; 410 return false; 411 } 412 413 /** 414 * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources 415 * 416 * @dev: drm_device pointer 417 * 418 * Returns true if the device is a dGPU with ACPI power control, 419 * otherwise return false. 420 */ 421 bool amdgpu_device_supports_boco(struct drm_device *dev) 422 { 423 struct amdgpu_device *adev = drm_to_adev(dev); 424 425 if (!IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE)) 426 return false; 427 428 if (adev->has_pr3 || 429 ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid())) 430 return true; 431 return false; 432 } 433 434 /** 435 * amdgpu_device_supports_baco - Does the device support BACO 436 * 437 * @dev: drm_device pointer 438 * 439 * Return: 440 * 1 if the device supports BACO; 441 * 3 if the device supports MACO (only works if BACO is supported) 442 * otherwise return 0. 443 */ 444 int amdgpu_device_supports_baco(struct drm_device *dev) 445 { 446 struct amdgpu_device *adev = drm_to_adev(dev); 447 448 return amdgpu_asic_supports_baco(adev); 449 } 450 451 void amdgpu_device_detect_runtime_pm_mode(struct amdgpu_device *adev) 452 { 453 struct drm_device *dev; 454 int bamaco_support; 455 456 dev = adev_to_drm(adev); 457 458 adev->pm.rpm_mode = AMDGPU_RUNPM_NONE; 459 bamaco_support = amdgpu_device_supports_baco(dev); 460 461 switch (amdgpu_runtime_pm) { 462 case 2: 463 if (bamaco_support & MACO_SUPPORT) { 464 adev->pm.rpm_mode = AMDGPU_RUNPM_BAMACO; 465 dev_info(adev->dev, "Forcing BAMACO for runtime pm\n"); 466 } else if (bamaco_support == BACO_SUPPORT) { 467 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO; 468 dev_info(adev->dev, "Requested mode BAMACO not available,fallback to use BACO\n"); 469 } 470 break; 471 case 1: 472 if (bamaco_support & BACO_SUPPORT) { 473 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO; 474 dev_info(adev->dev, "Forcing BACO for runtime pm\n"); 475 } 476 break; 477 case -1: 478 case -2: 479 if (amdgpu_device_supports_px(dev)) { /* enable PX as runtime mode */ 480 adev->pm.rpm_mode = AMDGPU_RUNPM_PX; 481 dev_info(adev->dev, "Using ATPX for runtime pm\n"); 482 } else if (amdgpu_device_supports_boco(dev)) { /* enable boco as runtime mode */ 483 adev->pm.rpm_mode = AMDGPU_RUNPM_BOCO; 484 dev_info(adev->dev, "Using BOCO for runtime pm\n"); 485 } else { 486 if (!bamaco_support) 487 goto no_runtime_pm; 488 489 switch (adev->asic_type) { 490 case CHIP_VEGA20: 491 case CHIP_ARCTURUS: 492 /* BACO are not supported on vega20 and arctrus */ 493 break; 494 case CHIP_VEGA10: 495 /* enable BACO as runpm mode if noretry=0 */ 496 if (!adev->gmc.noretry) 497 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO; 498 break; 499 default: 500 /* enable BACO as runpm mode on CI+ */ 501 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO; 502 break; 503 } 504 505 if (adev->pm.rpm_mode == AMDGPU_RUNPM_BACO) { 506 if (bamaco_support & MACO_SUPPORT) { 507 adev->pm.rpm_mode = AMDGPU_RUNPM_BAMACO; 508 dev_info(adev->dev, "Using BAMACO for runtime pm\n"); 509 } else { 510 dev_info(adev->dev, "Using BACO for runtime pm\n"); 511 } 512 } 513 } 514 break; 515 case 0: 516 dev_info(adev->dev, "runtime pm is manually disabled\n"); 517 break; 518 default: 519 break; 520 } 521 522 no_runtime_pm: 523 if (adev->pm.rpm_mode == AMDGPU_RUNPM_NONE) 524 dev_info(adev->dev, "Runtime PM not available\n"); 525 } 526 /** 527 * amdgpu_device_supports_smart_shift - Is the device dGPU with 528 * smart shift support 529 * 530 * @dev: drm_device pointer 531 * 532 * Returns true if the device is a dGPU with Smart Shift support, 533 * otherwise returns false. 534 */ 535 bool amdgpu_device_supports_smart_shift(struct drm_device *dev) 536 { 537 return (amdgpu_device_supports_boco(dev) && 538 amdgpu_acpi_is_power_shift_control_supported()); 539 } 540 541 /* 542 * VRAM access helper functions 543 */ 544 545 /** 546 * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA 547 * 548 * @adev: amdgpu_device pointer 549 * @pos: offset of the buffer in vram 550 * @buf: virtual address of the buffer in system memory 551 * @size: read/write size, sizeof(@buf) must > @size 552 * @write: true - write to vram, otherwise - read from vram 553 */ 554 void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos, 555 void *buf, size_t size, bool write) 556 { 557 unsigned long flags; 558 uint32_t hi = ~0, tmp = 0; 559 uint32_t *data = buf; 560 uint64_t last; 561 int idx; 562 563 if (!drm_dev_enter(adev_to_drm(adev), &idx)) 564 return; 565 566 BUG_ON(!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4)); 567 568 spin_lock_irqsave(&adev->mmio_idx_lock, flags); 569 for (last = pos + size; pos < last; pos += 4) { 570 tmp = pos >> 31; 571 572 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000); 573 if (tmp != hi) { 574 WREG32_NO_KIQ(mmMM_INDEX_HI, tmp); 575 hi = tmp; 576 } 577 if (write) 578 WREG32_NO_KIQ(mmMM_DATA, *data++); 579 else 580 *data++ = RREG32_NO_KIQ(mmMM_DATA); 581 } 582 583 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); 584 drm_dev_exit(idx); 585 } 586 587 /** 588 * amdgpu_device_aper_access - access vram by vram aperture 589 * 590 * @adev: amdgpu_device pointer 591 * @pos: offset of the buffer in vram 592 * @buf: virtual address of the buffer in system memory 593 * @size: read/write size, sizeof(@buf) must > @size 594 * @write: true - write to vram, otherwise - read from vram 595 * 596 * The return value means how many bytes have been transferred. 597 */ 598 size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos, 599 void *buf, size_t size, bool write) 600 { 601 #ifdef CONFIG_64BIT 602 void __iomem *addr; 603 size_t count = 0; 604 uint64_t last; 605 606 if (!adev->mman.aper_base_kaddr) 607 return 0; 608 609 last = min(pos + size, adev->gmc.visible_vram_size); 610 if (last > pos) { 611 addr = adev->mman.aper_base_kaddr + pos; 612 count = last - pos; 613 614 if (write) { 615 memcpy_toio(addr, buf, count); 616 /* Make sure HDP write cache flush happens without any reordering 617 * after the system memory contents are sent over PCIe device 618 */ 619 mb(); 620 amdgpu_device_flush_hdp(adev, NULL); 621 } else { 622 amdgpu_device_invalidate_hdp(adev, NULL); 623 /* Make sure HDP read cache is invalidated before issuing a read 624 * to the PCIe device 625 */ 626 mb(); 627 memcpy_fromio(buf, addr, count); 628 } 629 630 } 631 632 return count; 633 #else 634 return 0; 635 #endif 636 } 637 638 /** 639 * amdgpu_device_vram_access - read/write a buffer in vram 640 * 641 * @adev: amdgpu_device pointer 642 * @pos: offset of the buffer in vram 643 * @buf: virtual address of the buffer in system memory 644 * @size: read/write size, sizeof(@buf) must > @size 645 * @write: true - write to vram, otherwise - read from vram 646 */ 647 void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos, 648 void *buf, size_t size, bool write) 649 { 650 size_t count; 651 652 /* try to using vram apreature to access vram first */ 653 count = amdgpu_device_aper_access(adev, pos, buf, size, write); 654 size -= count; 655 if (size) { 656 /* using MM to access rest vram */ 657 pos += count; 658 buf += count; 659 amdgpu_device_mm_access(adev, pos, buf, size, write); 660 } 661 } 662 663 /* 664 * register access helper functions. 665 */ 666 667 /* Check if hw access should be skipped because of hotplug or device error */ 668 bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev) 669 { 670 if (adev->no_hw_access) 671 return true; 672 673 #ifdef CONFIG_LOCKDEP 674 /* 675 * This is a bit complicated to understand, so worth a comment. What we assert 676 * here is that the GPU reset is not running on another thread in parallel. 677 * 678 * For this we trylock the read side of the reset semaphore, if that succeeds 679 * we know that the reset is not running in parallel. 680 * 681 * If the trylock fails we assert that we are either already holding the read 682 * side of the lock or are the reset thread itself and hold the write side of 683 * the lock. 684 */ 685 if (in_task()) { 686 if (down_read_trylock(&adev->reset_domain->sem)) 687 up_read(&adev->reset_domain->sem); 688 else 689 lockdep_assert_held(&adev->reset_domain->sem); 690 } 691 #endif 692 return false; 693 } 694 695 /** 696 * amdgpu_device_rreg - read a memory mapped IO or indirect register 697 * 698 * @adev: amdgpu_device pointer 699 * @reg: dword aligned register offset 700 * @acc_flags: access flags which require special behavior 701 * 702 * Returns the 32 bit value from the offset specified. 703 */ 704 uint32_t amdgpu_device_rreg(struct amdgpu_device *adev, 705 uint32_t reg, uint32_t acc_flags) 706 { 707 uint32_t ret; 708 709 if (amdgpu_device_skip_hw_access(adev)) 710 return 0; 711 712 if ((reg * 4) < adev->rmmio_size) { 713 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && 714 amdgpu_sriov_runtime(adev) && 715 down_read_trylock(&adev->reset_domain->sem)) { 716 ret = amdgpu_kiq_rreg(adev, reg, 0); 717 up_read(&adev->reset_domain->sem); 718 } else { 719 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); 720 } 721 } else { 722 ret = adev->pcie_rreg(adev, reg * 4); 723 } 724 725 trace_amdgpu_device_rreg(adev->pdev->device, reg, ret); 726 727 return ret; 728 } 729 730 /* 731 * MMIO register read with bytes helper functions 732 * @offset:bytes offset from MMIO start 733 */ 734 735 /** 736 * amdgpu_mm_rreg8 - read a memory mapped IO register 737 * 738 * @adev: amdgpu_device pointer 739 * @offset: byte aligned register offset 740 * 741 * Returns the 8 bit value from the offset specified. 742 */ 743 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset) 744 { 745 if (amdgpu_device_skip_hw_access(adev)) 746 return 0; 747 748 if (offset < adev->rmmio_size) 749 return (readb(adev->rmmio + offset)); 750 BUG(); 751 } 752 753 754 /** 755 * amdgpu_device_xcc_rreg - read a memory mapped IO or indirect register with specific XCC 756 * 757 * @adev: amdgpu_device pointer 758 * @reg: dword aligned register offset 759 * @acc_flags: access flags which require special behavior 760 * @xcc_id: xcc accelerated compute core id 761 * 762 * Returns the 32 bit value from the offset specified. 763 */ 764 uint32_t amdgpu_device_xcc_rreg(struct amdgpu_device *adev, 765 uint32_t reg, uint32_t acc_flags, 766 uint32_t xcc_id) 767 { 768 uint32_t ret, rlcg_flag; 769 770 if (amdgpu_device_skip_hw_access(adev)) 771 return 0; 772 773 if ((reg * 4) < adev->rmmio_size) { 774 if (amdgpu_sriov_vf(adev) && 775 !amdgpu_sriov_runtime(adev) && 776 adev->gfx.rlc.rlcg_reg_access_supported && 777 amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags, 778 GC_HWIP, false, 779 &rlcg_flag)) { 780 ret = amdgpu_virt_rlcg_reg_rw(adev, reg, 0, rlcg_flag, GET_INST(GC, xcc_id)); 781 } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && 782 amdgpu_sriov_runtime(adev) && 783 down_read_trylock(&adev->reset_domain->sem)) { 784 ret = amdgpu_kiq_rreg(adev, reg, xcc_id); 785 up_read(&adev->reset_domain->sem); 786 } else { 787 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); 788 } 789 } else { 790 ret = adev->pcie_rreg(adev, reg * 4); 791 } 792 793 return ret; 794 } 795 796 /* 797 * MMIO register write with bytes helper functions 798 * @offset:bytes offset from MMIO start 799 * @value: the value want to be written to the register 800 */ 801 802 /** 803 * amdgpu_mm_wreg8 - read a memory mapped IO register 804 * 805 * @adev: amdgpu_device pointer 806 * @offset: byte aligned register offset 807 * @value: 8 bit value to write 808 * 809 * Writes the value specified to the offset specified. 810 */ 811 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value) 812 { 813 if (amdgpu_device_skip_hw_access(adev)) 814 return; 815 816 if (offset < adev->rmmio_size) 817 writeb(value, adev->rmmio + offset); 818 else 819 BUG(); 820 } 821 822 /** 823 * amdgpu_device_wreg - write to a memory mapped IO or indirect register 824 * 825 * @adev: amdgpu_device pointer 826 * @reg: dword aligned register offset 827 * @v: 32 bit value to write to the register 828 * @acc_flags: access flags which require special behavior 829 * 830 * Writes the value specified to the offset specified. 831 */ 832 void amdgpu_device_wreg(struct amdgpu_device *adev, 833 uint32_t reg, uint32_t v, 834 uint32_t acc_flags) 835 { 836 if (amdgpu_device_skip_hw_access(adev)) 837 return; 838 839 if ((reg * 4) < adev->rmmio_size) { 840 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && 841 amdgpu_sriov_runtime(adev) && 842 down_read_trylock(&adev->reset_domain->sem)) { 843 amdgpu_kiq_wreg(adev, reg, v, 0); 844 up_read(&adev->reset_domain->sem); 845 } else { 846 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); 847 } 848 } else { 849 adev->pcie_wreg(adev, reg * 4, v); 850 } 851 852 trace_amdgpu_device_wreg(adev->pdev->device, reg, v); 853 } 854 855 /** 856 * amdgpu_mm_wreg_mmio_rlc - write register either with direct/indirect mmio or with RLC path if in range 857 * 858 * @adev: amdgpu_device pointer 859 * @reg: mmio/rlc register 860 * @v: value to write 861 * @xcc_id: xcc accelerated compute core id 862 * 863 * this function is invoked only for the debugfs register access 864 */ 865 void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev, 866 uint32_t reg, uint32_t v, 867 uint32_t xcc_id) 868 { 869 if (amdgpu_device_skip_hw_access(adev)) 870 return; 871 872 if (amdgpu_sriov_fullaccess(adev) && 873 adev->gfx.rlc.funcs && 874 adev->gfx.rlc.funcs->is_rlcg_access_range) { 875 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg)) 876 return amdgpu_sriov_wreg(adev, reg, v, 0, 0, xcc_id); 877 } else if ((reg * 4) >= adev->rmmio_size) { 878 adev->pcie_wreg(adev, reg * 4, v); 879 } else { 880 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); 881 } 882 } 883 884 /** 885 * amdgpu_device_xcc_wreg - write to a memory mapped IO or indirect register with specific XCC 886 * 887 * @adev: amdgpu_device pointer 888 * @reg: dword aligned register offset 889 * @v: 32 bit value to write to the register 890 * @acc_flags: access flags which require special behavior 891 * @xcc_id: xcc accelerated compute core id 892 * 893 * Writes the value specified to the offset specified. 894 */ 895 void amdgpu_device_xcc_wreg(struct amdgpu_device *adev, 896 uint32_t reg, uint32_t v, 897 uint32_t acc_flags, uint32_t xcc_id) 898 { 899 uint32_t rlcg_flag; 900 901 if (amdgpu_device_skip_hw_access(adev)) 902 return; 903 904 if ((reg * 4) < adev->rmmio_size) { 905 if (amdgpu_sriov_vf(adev) && 906 !amdgpu_sriov_runtime(adev) && 907 adev->gfx.rlc.rlcg_reg_access_supported && 908 amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags, 909 GC_HWIP, true, 910 &rlcg_flag)) { 911 amdgpu_virt_rlcg_reg_rw(adev, reg, v, rlcg_flag, GET_INST(GC, xcc_id)); 912 } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && 913 amdgpu_sriov_runtime(adev) && 914 down_read_trylock(&adev->reset_domain->sem)) { 915 amdgpu_kiq_wreg(adev, reg, v, xcc_id); 916 up_read(&adev->reset_domain->sem); 917 } else { 918 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); 919 } 920 } else { 921 adev->pcie_wreg(adev, reg * 4, v); 922 } 923 } 924 925 /** 926 * amdgpu_device_indirect_rreg - read an indirect register 927 * 928 * @adev: amdgpu_device pointer 929 * @reg_addr: indirect register address to read from 930 * 931 * Returns the value of indirect register @reg_addr 932 */ 933 u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev, 934 u32 reg_addr) 935 { 936 unsigned long flags, pcie_index, pcie_data; 937 void __iomem *pcie_index_offset; 938 void __iomem *pcie_data_offset; 939 u32 r; 940 941 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 942 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 943 944 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 945 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 946 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 947 948 writel(reg_addr, pcie_index_offset); 949 readl(pcie_index_offset); 950 r = readl(pcie_data_offset); 951 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 952 953 return r; 954 } 955 956 u32 amdgpu_device_indirect_rreg_ext(struct amdgpu_device *adev, 957 u64 reg_addr) 958 { 959 unsigned long flags, pcie_index, pcie_index_hi, pcie_data; 960 u32 r; 961 void __iomem *pcie_index_offset; 962 void __iomem *pcie_index_hi_offset; 963 void __iomem *pcie_data_offset; 964 965 if (unlikely(!adev->nbio.funcs)) { 966 pcie_index = AMDGPU_PCIE_INDEX_FALLBACK; 967 pcie_data = AMDGPU_PCIE_DATA_FALLBACK; 968 } else { 969 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 970 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 971 } 972 973 if (reg_addr >> 32) { 974 if (unlikely(!adev->nbio.funcs)) 975 pcie_index_hi = AMDGPU_PCIE_INDEX_HI_FALLBACK; 976 else 977 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev); 978 } else { 979 pcie_index_hi = 0; 980 } 981 982 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 983 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 984 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 985 if (pcie_index_hi != 0) 986 pcie_index_hi_offset = (void __iomem *)adev->rmmio + 987 pcie_index_hi * 4; 988 989 writel(reg_addr, pcie_index_offset); 990 readl(pcie_index_offset); 991 if (pcie_index_hi != 0) { 992 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); 993 readl(pcie_index_hi_offset); 994 } 995 r = readl(pcie_data_offset); 996 997 /* clear the high bits */ 998 if (pcie_index_hi != 0) { 999 writel(0, pcie_index_hi_offset); 1000 readl(pcie_index_hi_offset); 1001 } 1002 1003 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 1004 1005 return r; 1006 } 1007 1008 /** 1009 * amdgpu_device_indirect_rreg64 - read a 64bits indirect register 1010 * 1011 * @adev: amdgpu_device pointer 1012 * @reg_addr: indirect register address to read from 1013 * 1014 * Returns the value of indirect register @reg_addr 1015 */ 1016 u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev, 1017 u32 reg_addr) 1018 { 1019 unsigned long flags, pcie_index, pcie_data; 1020 void __iomem *pcie_index_offset; 1021 void __iomem *pcie_data_offset; 1022 u64 r; 1023 1024 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 1025 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 1026 1027 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 1028 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 1029 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 1030 1031 /* read low 32 bits */ 1032 writel(reg_addr, pcie_index_offset); 1033 readl(pcie_index_offset); 1034 r = readl(pcie_data_offset); 1035 /* read high 32 bits */ 1036 writel(reg_addr + 4, pcie_index_offset); 1037 readl(pcie_index_offset); 1038 r |= ((u64)readl(pcie_data_offset) << 32); 1039 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 1040 1041 return r; 1042 } 1043 1044 u64 amdgpu_device_indirect_rreg64_ext(struct amdgpu_device *adev, 1045 u64 reg_addr) 1046 { 1047 unsigned long flags, pcie_index, pcie_data; 1048 unsigned long pcie_index_hi = 0; 1049 void __iomem *pcie_index_offset; 1050 void __iomem *pcie_index_hi_offset; 1051 void __iomem *pcie_data_offset; 1052 u64 r; 1053 1054 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 1055 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 1056 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset)) 1057 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev); 1058 1059 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 1060 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 1061 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 1062 if (pcie_index_hi != 0) 1063 pcie_index_hi_offset = (void __iomem *)adev->rmmio + 1064 pcie_index_hi * 4; 1065 1066 /* read low 32 bits */ 1067 writel(reg_addr, pcie_index_offset); 1068 readl(pcie_index_offset); 1069 if (pcie_index_hi != 0) { 1070 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); 1071 readl(pcie_index_hi_offset); 1072 } 1073 r = readl(pcie_data_offset); 1074 /* read high 32 bits */ 1075 writel(reg_addr + 4, pcie_index_offset); 1076 readl(pcie_index_offset); 1077 if (pcie_index_hi != 0) { 1078 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); 1079 readl(pcie_index_hi_offset); 1080 } 1081 r |= ((u64)readl(pcie_data_offset) << 32); 1082 1083 /* clear the high bits */ 1084 if (pcie_index_hi != 0) { 1085 writel(0, pcie_index_hi_offset); 1086 readl(pcie_index_hi_offset); 1087 } 1088 1089 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 1090 1091 return r; 1092 } 1093 1094 /** 1095 * amdgpu_device_indirect_wreg - write an indirect register address 1096 * 1097 * @adev: amdgpu_device pointer 1098 * @reg_addr: indirect register offset 1099 * @reg_data: indirect register data 1100 * 1101 */ 1102 void amdgpu_device_indirect_wreg(struct amdgpu_device *adev, 1103 u32 reg_addr, u32 reg_data) 1104 { 1105 unsigned long flags, pcie_index, pcie_data; 1106 void __iomem *pcie_index_offset; 1107 void __iomem *pcie_data_offset; 1108 1109 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 1110 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 1111 1112 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 1113 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 1114 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 1115 1116 writel(reg_addr, pcie_index_offset); 1117 readl(pcie_index_offset); 1118 writel(reg_data, pcie_data_offset); 1119 readl(pcie_data_offset); 1120 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 1121 } 1122 1123 void amdgpu_device_indirect_wreg_ext(struct amdgpu_device *adev, 1124 u64 reg_addr, u32 reg_data) 1125 { 1126 unsigned long flags, pcie_index, pcie_index_hi, pcie_data; 1127 void __iomem *pcie_index_offset; 1128 void __iomem *pcie_index_hi_offset; 1129 void __iomem *pcie_data_offset; 1130 1131 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 1132 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 1133 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset)) 1134 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev); 1135 else 1136 pcie_index_hi = 0; 1137 1138 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 1139 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 1140 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 1141 if (pcie_index_hi != 0) 1142 pcie_index_hi_offset = (void __iomem *)adev->rmmio + 1143 pcie_index_hi * 4; 1144 1145 writel(reg_addr, pcie_index_offset); 1146 readl(pcie_index_offset); 1147 if (pcie_index_hi != 0) { 1148 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); 1149 readl(pcie_index_hi_offset); 1150 } 1151 writel(reg_data, pcie_data_offset); 1152 readl(pcie_data_offset); 1153 1154 /* clear the high bits */ 1155 if (pcie_index_hi != 0) { 1156 writel(0, pcie_index_hi_offset); 1157 readl(pcie_index_hi_offset); 1158 } 1159 1160 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 1161 } 1162 1163 /** 1164 * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address 1165 * 1166 * @adev: amdgpu_device pointer 1167 * @reg_addr: indirect register offset 1168 * @reg_data: indirect register data 1169 * 1170 */ 1171 void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev, 1172 u32 reg_addr, u64 reg_data) 1173 { 1174 unsigned long flags, pcie_index, pcie_data; 1175 void __iomem *pcie_index_offset; 1176 void __iomem *pcie_data_offset; 1177 1178 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 1179 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 1180 1181 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 1182 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 1183 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 1184 1185 /* write low 32 bits */ 1186 writel(reg_addr, pcie_index_offset); 1187 readl(pcie_index_offset); 1188 writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset); 1189 readl(pcie_data_offset); 1190 /* write high 32 bits */ 1191 writel(reg_addr + 4, pcie_index_offset); 1192 readl(pcie_index_offset); 1193 writel((u32)(reg_data >> 32), pcie_data_offset); 1194 readl(pcie_data_offset); 1195 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 1196 } 1197 1198 void amdgpu_device_indirect_wreg64_ext(struct amdgpu_device *adev, 1199 u64 reg_addr, u64 reg_data) 1200 { 1201 unsigned long flags, pcie_index, pcie_data; 1202 unsigned long pcie_index_hi = 0; 1203 void __iomem *pcie_index_offset; 1204 void __iomem *pcie_index_hi_offset; 1205 void __iomem *pcie_data_offset; 1206 1207 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev); 1208 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev); 1209 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset)) 1210 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev); 1211 1212 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 1213 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 1214 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 1215 if (pcie_index_hi != 0) 1216 pcie_index_hi_offset = (void __iomem *)adev->rmmio + 1217 pcie_index_hi * 4; 1218 1219 /* write low 32 bits */ 1220 writel(reg_addr, pcie_index_offset); 1221 readl(pcie_index_offset); 1222 if (pcie_index_hi != 0) { 1223 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); 1224 readl(pcie_index_hi_offset); 1225 } 1226 writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset); 1227 readl(pcie_data_offset); 1228 /* write high 32 bits */ 1229 writel(reg_addr + 4, pcie_index_offset); 1230 readl(pcie_index_offset); 1231 if (pcie_index_hi != 0) { 1232 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset); 1233 readl(pcie_index_hi_offset); 1234 } 1235 writel((u32)(reg_data >> 32), pcie_data_offset); 1236 readl(pcie_data_offset); 1237 1238 /* clear the high bits */ 1239 if (pcie_index_hi != 0) { 1240 writel(0, pcie_index_hi_offset); 1241 readl(pcie_index_hi_offset); 1242 } 1243 1244 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 1245 } 1246 1247 /** 1248 * amdgpu_device_get_rev_id - query device rev_id 1249 * 1250 * @adev: amdgpu_device pointer 1251 * 1252 * Return device rev_id 1253 */ 1254 u32 amdgpu_device_get_rev_id(struct amdgpu_device *adev) 1255 { 1256 return adev->nbio.funcs->get_rev_id(adev); 1257 } 1258 1259 /** 1260 * amdgpu_invalid_rreg - dummy reg read function 1261 * 1262 * @adev: amdgpu_device pointer 1263 * @reg: offset of register 1264 * 1265 * Dummy register read function. Used for register blocks 1266 * that certain asics don't have (all asics). 1267 * Returns the value in the register. 1268 */ 1269 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg) 1270 { 1271 DRM_ERROR("Invalid callback to read register 0x%04X\n", reg); 1272 BUG(); 1273 return 0; 1274 } 1275 1276 static uint32_t amdgpu_invalid_rreg_ext(struct amdgpu_device *adev, uint64_t reg) 1277 { 1278 DRM_ERROR("Invalid callback to read register 0x%llX\n", reg); 1279 BUG(); 1280 return 0; 1281 } 1282 1283 /** 1284 * amdgpu_invalid_wreg - dummy reg write function 1285 * 1286 * @adev: amdgpu_device pointer 1287 * @reg: offset of register 1288 * @v: value to write to the register 1289 * 1290 * Dummy register read function. Used for register blocks 1291 * that certain asics don't have (all asics). 1292 */ 1293 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) 1294 { 1295 DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n", 1296 reg, v); 1297 BUG(); 1298 } 1299 1300 static void amdgpu_invalid_wreg_ext(struct amdgpu_device *adev, uint64_t reg, uint32_t v) 1301 { 1302 DRM_ERROR("Invalid callback to write register 0x%llX with 0x%08X\n", 1303 reg, v); 1304 BUG(); 1305 } 1306 1307 /** 1308 * amdgpu_invalid_rreg64 - dummy 64 bit reg read function 1309 * 1310 * @adev: amdgpu_device pointer 1311 * @reg: offset of register 1312 * 1313 * Dummy register read function. Used for register blocks 1314 * that certain asics don't have (all asics). 1315 * Returns the value in the register. 1316 */ 1317 static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg) 1318 { 1319 DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg); 1320 BUG(); 1321 return 0; 1322 } 1323 1324 static uint64_t amdgpu_invalid_rreg64_ext(struct amdgpu_device *adev, uint64_t reg) 1325 { 1326 DRM_ERROR("Invalid callback to read register 0x%llX\n", reg); 1327 BUG(); 1328 return 0; 1329 } 1330 1331 /** 1332 * amdgpu_invalid_wreg64 - dummy reg write function 1333 * 1334 * @adev: amdgpu_device pointer 1335 * @reg: offset of register 1336 * @v: value to write to the register 1337 * 1338 * Dummy register read function. Used for register blocks 1339 * that certain asics don't have (all asics). 1340 */ 1341 static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v) 1342 { 1343 DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n", 1344 reg, v); 1345 BUG(); 1346 } 1347 1348 static void amdgpu_invalid_wreg64_ext(struct amdgpu_device *adev, uint64_t reg, uint64_t v) 1349 { 1350 DRM_ERROR("Invalid callback to write 64 bit register 0x%llX with 0x%08llX\n", 1351 reg, v); 1352 BUG(); 1353 } 1354 1355 /** 1356 * amdgpu_block_invalid_rreg - dummy reg read function 1357 * 1358 * @adev: amdgpu_device pointer 1359 * @block: offset of instance 1360 * @reg: offset of register 1361 * 1362 * Dummy register read function. Used for register blocks 1363 * that certain asics don't have (all asics). 1364 * Returns the value in the register. 1365 */ 1366 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev, 1367 uint32_t block, uint32_t reg) 1368 { 1369 DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n", 1370 reg, block); 1371 BUG(); 1372 return 0; 1373 } 1374 1375 /** 1376 * amdgpu_block_invalid_wreg - dummy reg write function 1377 * 1378 * @adev: amdgpu_device pointer 1379 * @block: offset of instance 1380 * @reg: offset of register 1381 * @v: value to write to the register 1382 * 1383 * Dummy register read function. Used for register blocks 1384 * that certain asics don't have (all asics). 1385 */ 1386 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev, 1387 uint32_t block, 1388 uint32_t reg, uint32_t v) 1389 { 1390 DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n", 1391 reg, block, v); 1392 BUG(); 1393 } 1394 1395 static uint32_t amdgpu_device_get_vbios_flags(struct amdgpu_device *adev) 1396 { 1397 if (hweight32(adev->aid_mask) && (adev->flags & AMD_IS_APU)) 1398 return AMDGPU_VBIOS_SKIP; 1399 1400 if (hweight32(adev->aid_mask) && amdgpu_passthrough(adev)) 1401 return AMDGPU_VBIOS_OPTIONAL; 1402 1403 return 0; 1404 } 1405 1406 /** 1407 * amdgpu_device_asic_init - Wrapper for atom asic_init 1408 * 1409 * @adev: amdgpu_device pointer 1410 * 1411 * Does any asic specific work and then calls atom asic init. 1412 */ 1413 static int amdgpu_device_asic_init(struct amdgpu_device *adev) 1414 { 1415 uint32_t flags; 1416 bool optional; 1417 int ret; 1418 1419 amdgpu_asic_pre_asic_init(adev); 1420 flags = amdgpu_device_get_vbios_flags(adev); 1421 optional = !!(flags & (AMDGPU_VBIOS_OPTIONAL | AMDGPU_VBIOS_SKIP)); 1422 1423 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) || 1424 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4) || 1425 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0) || 1426 amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0)) { 1427 amdgpu_psp_wait_for_bootloader(adev); 1428 if (optional && !adev->bios) 1429 return 0; 1430 1431 ret = amdgpu_atomfirmware_asic_init(adev, true); 1432 return ret; 1433 } else { 1434 if (optional && !adev->bios) 1435 return 0; 1436 1437 return amdgpu_atom_asic_init(adev->mode_info.atom_context); 1438 } 1439 1440 return 0; 1441 } 1442 1443 /** 1444 * amdgpu_device_mem_scratch_init - allocate the VRAM scratch page 1445 * 1446 * @adev: amdgpu_device pointer 1447 * 1448 * Allocates a scratch page of VRAM for use by various things in the 1449 * driver. 1450 */ 1451 static int amdgpu_device_mem_scratch_init(struct amdgpu_device *adev) 1452 { 1453 return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, PAGE_SIZE, 1454 AMDGPU_GEM_DOMAIN_VRAM | 1455 AMDGPU_GEM_DOMAIN_GTT, 1456 &adev->mem_scratch.robj, 1457 &adev->mem_scratch.gpu_addr, 1458 (void **)&adev->mem_scratch.ptr); 1459 } 1460 1461 /** 1462 * amdgpu_device_mem_scratch_fini - Free the VRAM scratch page 1463 * 1464 * @adev: amdgpu_device pointer 1465 * 1466 * Frees the VRAM scratch page. 1467 */ 1468 static void amdgpu_device_mem_scratch_fini(struct amdgpu_device *adev) 1469 { 1470 amdgpu_bo_free_kernel(&adev->mem_scratch.robj, NULL, NULL); 1471 } 1472 1473 /** 1474 * amdgpu_device_program_register_sequence - program an array of registers. 1475 * 1476 * @adev: amdgpu_device pointer 1477 * @registers: pointer to the register array 1478 * @array_size: size of the register array 1479 * 1480 * Programs an array or registers with and or masks. 1481 * This is a helper for setting golden registers. 1482 */ 1483 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev, 1484 const u32 *registers, 1485 const u32 array_size) 1486 { 1487 u32 tmp, reg, and_mask, or_mask; 1488 int i; 1489 1490 if (array_size % 3) 1491 return; 1492 1493 for (i = 0; i < array_size; i += 3) { 1494 reg = registers[i + 0]; 1495 and_mask = registers[i + 1]; 1496 or_mask = registers[i + 2]; 1497 1498 if (and_mask == 0xffffffff) { 1499 tmp = or_mask; 1500 } else { 1501 tmp = RREG32(reg); 1502 tmp &= ~and_mask; 1503 if (adev->family >= AMDGPU_FAMILY_AI) 1504 tmp |= (or_mask & and_mask); 1505 else 1506 tmp |= or_mask; 1507 } 1508 WREG32(reg, tmp); 1509 } 1510 } 1511 1512 /** 1513 * amdgpu_device_pci_config_reset - reset the GPU 1514 * 1515 * @adev: amdgpu_device pointer 1516 * 1517 * Resets the GPU using the pci config reset sequence. 1518 * Only applicable to asics prior to vega10. 1519 */ 1520 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev) 1521 { 1522 pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA); 1523 } 1524 1525 /** 1526 * amdgpu_device_pci_reset - reset the GPU using generic PCI means 1527 * 1528 * @adev: amdgpu_device pointer 1529 * 1530 * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.). 1531 */ 1532 int amdgpu_device_pci_reset(struct amdgpu_device *adev) 1533 { 1534 return pci_reset_function(adev->pdev); 1535 } 1536 1537 /* 1538 * amdgpu_device_wb_*() 1539 * Writeback is the method by which the GPU updates special pages in memory 1540 * with the status of certain GPU events (fences, ring pointers,etc.). 1541 */ 1542 1543 /** 1544 * amdgpu_device_wb_fini - Disable Writeback and free memory 1545 * 1546 * @adev: amdgpu_device pointer 1547 * 1548 * Disables Writeback and frees the Writeback memory (all asics). 1549 * Used at driver shutdown. 1550 */ 1551 static void amdgpu_device_wb_fini(struct amdgpu_device *adev) 1552 { 1553 if (adev->wb.wb_obj) { 1554 amdgpu_bo_free_kernel(&adev->wb.wb_obj, 1555 &adev->wb.gpu_addr, 1556 (void **)&adev->wb.wb); 1557 adev->wb.wb_obj = NULL; 1558 } 1559 } 1560 1561 /** 1562 * amdgpu_device_wb_init - Init Writeback driver info and allocate memory 1563 * 1564 * @adev: amdgpu_device pointer 1565 * 1566 * Initializes writeback and allocates writeback memory (all asics). 1567 * Used at driver startup. 1568 * Returns 0 on success or an -error on failure. 1569 */ 1570 static int amdgpu_device_wb_init(struct amdgpu_device *adev) 1571 { 1572 int r; 1573 1574 if (adev->wb.wb_obj == NULL) { 1575 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */ 1576 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8, 1577 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 1578 &adev->wb.wb_obj, &adev->wb.gpu_addr, 1579 (void **)&adev->wb.wb); 1580 if (r) { 1581 dev_warn(adev->dev, "(%d) create WB bo failed\n", r); 1582 return r; 1583 } 1584 1585 adev->wb.num_wb = AMDGPU_MAX_WB; 1586 memset(&adev->wb.used, 0, sizeof(adev->wb.used)); 1587 1588 /* clear wb memory */ 1589 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8); 1590 } 1591 1592 return 0; 1593 } 1594 1595 /** 1596 * amdgpu_device_wb_get - Allocate a wb entry 1597 * 1598 * @adev: amdgpu_device pointer 1599 * @wb: wb index 1600 * 1601 * Allocate a wb slot for use by the driver (all asics). 1602 * Returns 0 on success or -EINVAL on failure. 1603 */ 1604 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb) 1605 { 1606 unsigned long flags, offset; 1607 1608 spin_lock_irqsave(&adev->wb.lock, flags); 1609 offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb); 1610 if (offset < adev->wb.num_wb) { 1611 __set_bit(offset, adev->wb.used); 1612 spin_unlock_irqrestore(&adev->wb.lock, flags); 1613 *wb = offset << 3; /* convert to dw offset */ 1614 return 0; 1615 } else { 1616 spin_unlock_irqrestore(&adev->wb.lock, flags); 1617 return -EINVAL; 1618 } 1619 } 1620 1621 /** 1622 * amdgpu_device_wb_free - Free a wb entry 1623 * 1624 * @adev: amdgpu_device pointer 1625 * @wb: wb index 1626 * 1627 * Free a wb slot allocated for use by the driver (all asics) 1628 */ 1629 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb) 1630 { 1631 unsigned long flags; 1632 1633 wb >>= 3; 1634 spin_lock_irqsave(&adev->wb.lock, flags); 1635 if (wb < adev->wb.num_wb) 1636 __clear_bit(wb, adev->wb.used); 1637 spin_unlock_irqrestore(&adev->wb.lock, flags); 1638 } 1639 1640 /** 1641 * amdgpu_device_resize_fb_bar - try to resize FB BAR 1642 * 1643 * @adev: amdgpu_device pointer 1644 * 1645 * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not 1646 * to fail, but if any of the BARs is not accessible after the size we abort 1647 * driver loading by returning -ENODEV. 1648 */ 1649 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev) 1650 { 1651 int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size); 1652 struct pci_bus *root; 1653 struct resource *res; 1654 unsigned int i; 1655 u16 cmd; 1656 int r; 1657 1658 if (!IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT)) 1659 return 0; 1660 1661 /* Bypass for VF */ 1662 if (amdgpu_sriov_vf(adev)) 1663 return 0; 1664 1665 /* PCI_EXT_CAP_ID_VNDR extended capability is located at 0x100 */ 1666 if (!pci_find_ext_capability(adev->pdev, PCI_EXT_CAP_ID_VNDR)) 1667 DRM_WARN("System can't access extended configuration space, please check!!\n"); 1668 1669 /* skip if the bios has already enabled large BAR */ 1670 if (adev->gmc.real_vram_size && 1671 (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size)) 1672 return 0; 1673 1674 /* Check if the root BUS has 64bit memory resources */ 1675 root = adev->pdev->bus; 1676 while (root->parent) 1677 root = root->parent; 1678 1679 pci_bus_for_each_resource(root, res, i) { 1680 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) && 1681 res->start > 0x100000000ull) 1682 break; 1683 } 1684 1685 /* Trying to resize is pointless without a root hub window above 4GB */ 1686 if (!res) 1687 return 0; 1688 1689 /* Limit the BAR size to what is available */ 1690 rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1, 1691 rbar_size); 1692 1693 /* Disable memory decoding while we change the BAR addresses and size */ 1694 pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd); 1695 pci_write_config_word(adev->pdev, PCI_COMMAND, 1696 cmd & ~PCI_COMMAND_MEMORY); 1697 1698 /* Free the VRAM and doorbell BAR, we most likely need to move both. */ 1699 amdgpu_doorbell_fini(adev); 1700 if (adev->asic_type >= CHIP_BONAIRE) 1701 pci_release_resource(adev->pdev, 2); 1702 1703 pci_release_resource(adev->pdev, 0); 1704 1705 r = pci_resize_resource(adev->pdev, 0, rbar_size); 1706 if (r == -ENOSPC) 1707 DRM_INFO("Not enough PCI address space for a large BAR."); 1708 else if (r && r != -ENOTSUPP) 1709 DRM_ERROR("Problem resizing BAR0 (%d).", r); 1710 1711 pci_assign_unassigned_bus_resources(adev->pdev->bus); 1712 1713 /* When the doorbell or fb BAR isn't available we have no chance of 1714 * using the device. 1715 */ 1716 r = amdgpu_doorbell_init(adev); 1717 if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET)) 1718 return -ENODEV; 1719 1720 pci_write_config_word(adev->pdev, PCI_COMMAND, cmd); 1721 1722 return 0; 1723 } 1724 1725 /* 1726 * GPU helpers function. 1727 */ 1728 /** 1729 * amdgpu_device_need_post - check if the hw need post or not 1730 * 1731 * @adev: amdgpu_device pointer 1732 * 1733 * Check if the asic has been initialized (all asics) at driver startup 1734 * or post is needed if hw reset is performed. 1735 * Returns true if need or false if not. 1736 */ 1737 bool amdgpu_device_need_post(struct amdgpu_device *adev) 1738 { 1739 uint32_t reg, flags; 1740 1741 if (amdgpu_sriov_vf(adev)) 1742 return false; 1743 1744 flags = amdgpu_device_get_vbios_flags(adev); 1745 if (flags & AMDGPU_VBIOS_SKIP) 1746 return false; 1747 if ((flags & AMDGPU_VBIOS_OPTIONAL) && !adev->bios) 1748 return false; 1749 1750 if (amdgpu_passthrough(adev)) { 1751 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot 1752 * some old smc fw still need driver do vPost otherwise gpu hang, while 1753 * those smc fw version above 22.15 doesn't have this flaw, so we force 1754 * vpost executed for smc version below 22.15 1755 */ 1756 if (adev->asic_type == CHIP_FIJI) { 1757 int err; 1758 uint32_t fw_ver; 1759 1760 err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev); 1761 /* force vPost if error occurred */ 1762 if (err) 1763 return true; 1764 1765 fw_ver = *((uint32_t *)adev->pm.fw->data + 69); 1766 release_firmware(adev->pm.fw); 1767 if (fw_ver < 0x00160e00) 1768 return true; 1769 } 1770 } 1771 1772 /* Don't post if we need to reset whole hive on init */ 1773 if (adev->init_lvl->level == AMDGPU_INIT_LEVEL_MINIMAL_XGMI) 1774 return false; 1775 1776 if (adev->has_hw_reset) { 1777 adev->has_hw_reset = false; 1778 return true; 1779 } 1780 1781 /* bios scratch used on CIK+ */ 1782 if (adev->asic_type >= CHIP_BONAIRE) 1783 return amdgpu_atombios_scratch_need_asic_init(adev); 1784 1785 /* check MEM_SIZE for older asics */ 1786 reg = amdgpu_asic_get_config_memsize(adev); 1787 1788 if ((reg != 0) && (reg != 0xffffffff)) 1789 return false; 1790 1791 return true; 1792 } 1793 1794 /* 1795 * Check whether seamless boot is supported. 1796 * 1797 * So far we only support seamless boot on DCE 3.0 or later. 1798 * If users report that it works on older ASICS as well, we may 1799 * loosen this. 1800 */ 1801 bool amdgpu_device_seamless_boot_supported(struct amdgpu_device *adev) 1802 { 1803 switch (amdgpu_seamless) { 1804 case -1: 1805 break; 1806 case 1: 1807 return true; 1808 case 0: 1809 return false; 1810 default: 1811 DRM_ERROR("Invalid value for amdgpu.seamless: %d\n", 1812 amdgpu_seamless); 1813 return false; 1814 } 1815 1816 if (!(adev->flags & AMD_IS_APU)) 1817 return false; 1818 1819 if (adev->mman.keep_stolen_vga_memory) 1820 return false; 1821 1822 return amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(3, 0, 0); 1823 } 1824 1825 /* 1826 * Intel hosts such as Rocket Lake, Alder Lake, Raptor Lake and Sapphire Rapids 1827 * don't support dynamic speed switching. Until we have confirmation from Intel 1828 * that a specific host supports it, it's safer that we keep it disabled for all. 1829 * 1830 * https://edc.intel.com/content/www/us/en/design/products/platforms/details/raptor-lake-s/13th-generation-core-processors-datasheet-volume-1-of-2/005/pci-express-support/ 1831 * https://gitlab.freedesktop.org/drm/amd/-/issues/2663 1832 */ 1833 static bool amdgpu_device_pcie_dynamic_switching_supported(struct amdgpu_device *adev) 1834 { 1835 #if IS_ENABLED(CONFIG_X86) 1836 struct cpuinfo_x86 *c = &cpu_data(0); 1837 1838 /* eGPU change speeds based on USB4 fabric conditions */ 1839 if (dev_is_removable(adev->dev)) 1840 return true; 1841 1842 if (c->x86_vendor == X86_VENDOR_INTEL) 1843 return false; 1844 #endif 1845 return true; 1846 } 1847 1848 /** 1849 * amdgpu_device_should_use_aspm - check if the device should program ASPM 1850 * 1851 * @adev: amdgpu_device pointer 1852 * 1853 * Confirm whether the module parameter and pcie bridge agree that ASPM should 1854 * be set for this device. 1855 * 1856 * Returns true if it should be used or false if not. 1857 */ 1858 bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev) 1859 { 1860 switch (amdgpu_aspm) { 1861 case -1: 1862 break; 1863 case 0: 1864 return false; 1865 case 1: 1866 return true; 1867 default: 1868 return false; 1869 } 1870 if (adev->flags & AMD_IS_APU) 1871 return false; 1872 if (!(adev->pm.pp_feature & PP_PCIE_DPM_MASK)) 1873 return false; 1874 return pcie_aspm_enabled(adev->pdev); 1875 } 1876 1877 /* if we get transitioned to only one device, take VGA back */ 1878 /** 1879 * amdgpu_device_vga_set_decode - enable/disable vga decode 1880 * 1881 * @pdev: PCI device pointer 1882 * @state: enable/disable vga decode 1883 * 1884 * Enable/disable vga decode (all asics). 1885 * Returns VGA resource flags. 1886 */ 1887 static unsigned int amdgpu_device_vga_set_decode(struct pci_dev *pdev, 1888 bool state) 1889 { 1890 struct amdgpu_device *adev = drm_to_adev(pci_get_drvdata(pdev)); 1891 1892 amdgpu_asic_set_vga_state(adev, state); 1893 if (state) 1894 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM | 1895 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 1896 else 1897 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 1898 } 1899 1900 /** 1901 * amdgpu_device_check_block_size - validate the vm block size 1902 * 1903 * @adev: amdgpu_device pointer 1904 * 1905 * Validates the vm block size specified via module parameter. 1906 * The vm block size defines number of bits in page table versus page directory, 1907 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the 1908 * page table and the remaining bits are in the page directory. 1909 */ 1910 static void amdgpu_device_check_block_size(struct amdgpu_device *adev) 1911 { 1912 /* defines number of bits in page table versus page directory, 1913 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the 1914 * page table and the remaining bits are in the page directory 1915 */ 1916 if (amdgpu_vm_block_size == -1) 1917 return; 1918 1919 if (amdgpu_vm_block_size < 9) { 1920 dev_warn(adev->dev, "VM page table size (%d) too small\n", 1921 amdgpu_vm_block_size); 1922 amdgpu_vm_block_size = -1; 1923 } 1924 } 1925 1926 /** 1927 * amdgpu_device_check_vm_size - validate the vm size 1928 * 1929 * @adev: amdgpu_device pointer 1930 * 1931 * Validates the vm size in GB specified via module parameter. 1932 * The VM size is the size of the GPU virtual memory space in GB. 1933 */ 1934 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev) 1935 { 1936 /* no need to check the default value */ 1937 if (amdgpu_vm_size == -1) 1938 return; 1939 1940 if (amdgpu_vm_size < 1) { 1941 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n", 1942 amdgpu_vm_size); 1943 amdgpu_vm_size = -1; 1944 } 1945 } 1946 1947 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev) 1948 { 1949 struct sysinfo si; 1950 bool is_os_64 = (sizeof(void *) == 8); 1951 uint64_t total_memory; 1952 uint64_t dram_size_seven_GB = 0x1B8000000; 1953 uint64_t dram_size_three_GB = 0xB8000000; 1954 1955 if (amdgpu_smu_memory_pool_size == 0) 1956 return; 1957 1958 if (!is_os_64) { 1959 DRM_WARN("Not 64-bit OS, feature not supported\n"); 1960 goto def_value; 1961 } 1962 si_meminfo(&si); 1963 total_memory = (uint64_t)si.totalram * si.mem_unit; 1964 1965 if ((amdgpu_smu_memory_pool_size == 1) || 1966 (amdgpu_smu_memory_pool_size == 2)) { 1967 if (total_memory < dram_size_three_GB) 1968 goto def_value1; 1969 } else if ((amdgpu_smu_memory_pool_size == 4) || 1970 (amdgpu_smu_memory_pool_size == 8)) { 1971 if (total_memory < dram_size_seven_GB) 1972 goto def_value1; 1973 } else { 1974 DRM_WARN("Smu memory pool size not supported\n"); 1975 goto def_value; 1976 } 1977 adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28; 1978 1979 return; 1980 1981 def_value1: 1982 DRM_WARN("No enough system memory\n"); 1983 def_value: 1984 adev->pm.smu_prv_buffer_size = 0; 1985 } 1986 1987 static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev) 1988 { 1989 if (!(adev->flags & AMD_IS_APU) || 1990 adev->asic_type < CHIP_RAVEN) 1991 return 0; 1992 1993 switch (adev->asic_type) { 1994 case CHIP_RAVEN: 1995 if (adev->pdev->device == 0x15dd) 1996 adev->apu_flags |= AMD_APU_IS_RAVEN; 1997 if (adev->pdev->device == 0x15d8) 1998 adev->apu_flags |= AMD_APU_IS_PICASSO; 1999 break; 2000 case CHIP_RENOIR: 2001 if ((adev->pdev->device == 0x1636) || 2002 (adev->pdev->device == 0x164c)) 2003 adev->apu_flags |= AMD_APU_IS_RENOIR; 2004 else 2005 adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE; 2006 break; 2007 case CHIP_VANGOGH: 2008 adev->apu_flags |= AMD_APU_IS_VANGOGH; 2009 break; 2010 case CHIP_YELLOW_CARP: 2011 break; 2012 case CHIP_CYAN_SKILLFISH: 2013 if ((adev->pdev->device == 0x13FE) || 2014 (adev->pdev->device == 0x143F)) 2015 adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2; 2016 break; 2017 default: 2018 break; 2019 } 2020 2021 return 0; 2022 } 2023 2024 /** 2025 * amdgpu_device_check_arguments - validate module params 2026 * 2027 * @adev: amdgpu_device pointer 2028 * 2029 * Validates certain module parameters and updates 2030 * the associated values used by the driver (all asics). 2031 */ 2032 static int amdgpu_device_check_arguments(struct amdgpu_device *adev) 2033 { 2034 int i; 2035 2036 if (amdgpu_sched_jobs < 4) { 2037 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n", 2038 amdgpu_sched_jobs); 2039 amdgpu_sched_jobs = 4; 2040 } else if (!is_power_of_2(amdgpu_sched_jobs)) { 2041 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n", 2042 amdgpu_sched_jobs); 2043 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs); 2044 } 2045 2046 if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) { 2047 /* gart size must be greater or equal to 32M */ 2048 dev_warn(adev->dev, "gart size (%d) too small\n", 2049 amdgpu_gart_size); 2050 amdgpu_gart_size = -1; 2051 } 2052 2053 if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) { 2054 /* gtt size must be greater or equal to 32M */ 2055 dev_warn(adev->dev, "gtt size (%d) too small\n", 2056 amdgpu_gtt_size); 2057 amdgpu_gtt_size = -1; 2058 } 2059 2060 /* valid range is between 4 and 9 inclusive */ 2061 if (amdgpu_vm_fragment_size != -1 && 2062 (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) { 2063 dev_warn(adev->dev, "valid range is between 4 and 9\n"); 2064 amdgpu_vm_fragment_size = -1; 2065 } 2066 2067 if (amdgpu_sched_hw_submission < 2) { 2068 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n", 2069 amdgpu_sched_hw_submission); 2070 amdgpu_sched_hw_submission = 2; 2071 } else if (!is_power_of_2(amdgpu_sched_hw_submission)) { 2072 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n", 2073 amdgpu_sched_hw_submission); 2074 amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission); 2075 } 2076 2077 if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) { 2078 dev_warn(adev->dev, "invalid option for reset method, reverting to default\n"); 2079 amdgpu_reset_method = -1; 2080 } 2081 2082 amdgpu_device_check_smu_prv_buffer_size(adev); 2083 2084 amdgpu_device_check_vm_size(adev); 2085 2086 amdgpu_device_check_block_size(adev); 2087 2088 adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type); 2089 2090 for (i = 0; i < MAX_XCP; i++) 2091 adev->enforce_isolation[i] = !!enforce_isolation; 2092 2093 return 0; 2094 } 2095 2096 /** 2097 * amdgpu_switcheroo_set_state - set switcheroo state 2098 * 2099 * @pdev: pci dev pointer 2100 * @state: vga_switcheroo state 2101 * 2102 * Callback for the switcheroo driver. Suspends or resumes 2103 * the asics before or after it is powered up using ACPI methods. 2104 */ 2105 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, 2106 enum vga_switcheroo_state state) 2107 { 2108 struct drm_device *dev = pci_get_drvdata(pdev); 2109 int r; 2110 2111 if (amdgpu_device_supports_px(dev) && state == VGA_SWITCHEROO_OFF) 2112 return; 2113 2114 if (state == VGA_SWITCHEROO_ON) { 2115 pr_info("switched on\n"); 2116 /* don't suspend or resume card normally */ 2117 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; 2118 2119 pci_set_power_state(pdev, PCI_D0); 2120 amdgpu_device_load_pci_state(pdev); 2121 r = pci_enable_device(pdev); 2122 if (r) 2123 DRM_WARN("pci_enable_device failed (%d)\n", r); 2124 amdgpu_device_resume(dev, true); 2125 2126 dev->switch_power_state = DRM_SWITCH_POWER_ON; 2127 } else { 2128 pr_info("switched off\n"); 2129 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; 2130 amdgpu_device_prepare(dev); 2131 amdgpu_device_suspend(dev, true); 2132 amdgpu_device_cache_pci_state(pdev); 2133 /* Shut down the device */ 2134 pci_disable_device(pdev); 2135 pci_set_power_state(pdev, PCI_D3cold); 2136 dev->switch_power_state = DRM_SWITCH_POWER_OFF; 2137 } 2138 } 2139 2140 /** 2141 * amdgpu_switcheroo_can_switch - see if switcheroo state can change 2142 * 2143 * @pdev: pci dev pointer 2144 * 2145 * Callback for the switcheroo driver. Check of the switcheroo 2146 * state can be changed. 2147 * Returns true if the state can be changed, false if not. 2148 */ 2149 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev) 2150 { 2151 struct drm_device *dev = pci_get_drvdata(pdev); 2152 2153 /* 2154 * FIXME: open_count is protected by drm_global_mutex but that would lead to 2155 * locking inversion with the driver load path. And the access here is 2156 * completely racy anyway. So don't bother with locking for now. 2157 */ 2158 return atomic_read(&dev->open_count) == 0; 2159 } 2160 2161 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = { 2162 .set_gpu_state = amdgpu_switcheroo_set_state, 2163 .reprobe = NULL, 2164 .can_switch = amdgpu_switcheroo_can_switch, 2165 }; 2166 2167 /** 2168 * amdgpu_device_ip_set_clockgating_state - set the CG state 2169 * 2170 * @dev: amdgpu_device pointer 2171 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 2172 * @state: clockgating state (gate or ungate) 2173 * 2174 * Sets the requested clockgating state for all instances of 2175 * the hardware IP specified. 2176 * Returns the error code from the last instance. 2177 */ 2178 int amdgpu_device_ip_set_clockgating_state(void *dev, 2179 enum amd_ip_block_type block_type, 2180 enum amd_clockgating_state state) 2181 { 2182 struct amdgpu_device *adev = dev; 2183 int i, r = 0; 2184 2185 for (i = 0; i < adev->num_ip_blocks; i++) { 2186 if (!adev->ip_blocks[i].status.valid) 2187 continue; 2188 if (adev->ip_blocks[i].version->type != block_type) 2189 continue; 2190 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state) 2191 continue; 2192 r = adev->ip_blocks[i].version->funcs->set_clockgating_state( 2193 &adev->ip_blocks[i], state); 2194 if (r) 2195 DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n", 2196 adev->ip_blocks[i].version->funcs->name, r); 2197 } 2198 return r; 2199 } 2200 2201 /** 2202 * amdgpu_device_ip_set_powergating_state - set the PG state 2203 * 2204 * @dev: amdgpu_device pointer 2205 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 2206 * @state: powergating state (gate or ungate) 2207 * 2208 * Sets the requested powergating state for all instances of 2209 * the hardware IP specified. 2210 * Returns the error code from the last instance. 2211 */ 2212 int amdgpu_device_ip_set_powergating_state(void *dev, 2213 enum amd_ip_block_type block_type, 2214 enum amd_powergating_state state) 2215 { 2216 struct amdgpu_device *adev = dev; 2217 int i, r = 0; 2218 2219 for (i = 0; i < adev->num_ip_blocks; i++) { 2220 if (!adev->ip_blocks[i].status.valid) 2221 continue; 2222 if (adev->ip_blocks[i].version->type != block_type) 2223 continue; 2224 if (!adev->ip_blocks[i].version->funcs->set_powergating_state) 2225 continue; 2226 r = adev->ip_blocks[i].version->funcs->set_powergating_state( 2227 &adev->ip_blocks[i], state); 2228 if (r) 2229 DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n", 2230 adev->ip_blocks[i].version->funcs->name, r); 2231 } 2232 return r; 2233 } 2234 2235 /** 2236 * amdgpu_device_ip_get_clockgating_state - get the CG state 2237 * 2238 * @adev: amdgpu_device pointer 2239 * @flags: clockgating feature flags 2240 * 2241 * Walks the list of IPs on the device and updates the clockgating 2242 * flags for each IP. 2243 * Updates @flags with the feature flags for each hardware IP where 2244 * clockgating is enabled. 2245 */ 2246 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev, 2247 u64 *flags) 2248 { 2249 int i; 2250 2251 for (i = 0; i < adev->num_ip_blocks; i++) { 2252 if (!adev->ip_blocks[i].status.valid) 2253 continue; 2254 if (adev->ip_blocks[i].version->funcs->get_clockgating_state) 2255 adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags); 2256 } 2257 } 2258 2259 /** 2260 * amdgpu_device_ip_wait_for_idle - wait for idle 2261 * 2262 * @adev: amdgpu_device pointer 2263 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 2264 * 2265 * Waits for the request hardware IP to be idle. 2266 * Returns 0 for success or a negative error code on failure. 2267 */ 2268 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev, 2269 enum amd_ip_block_type block_type) 2270 { 2271 int i, r; 2272 2273 for (i = 0; i < adev->num_ip_blocks; i++) { 2274 if (!adev->ip_blocks[i].status.valid) 2275 continue; 2276 if (adev->ip_blocks[i].version->type == block_type) { 2277 if (adev->ip_blocks[i].version->funcs->wait_for_idle) { 2278 r = adev->ip_blocks[i].version->funcs->wait_for_idle( 2279 &adev->ip_blocks[i]); 2280 if (r) 2281 return r; 2282 } 2283 break; 2284 } 2285 } 2286 return 0; 2287 2288 } 2289 2290 /** 2291 * amdgpu_device_ip_is_valid - is the hardware IP enabled 2292 * 2293 * @adev: amdgpu_device pointer 2294 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 2295 * 2296 * Check if the hardware IP is enable or not. 2297 * Returns true if it the IP is enable, false if not. 2298 */ 2299 bool amdgpu_device_ip_is_valid(struct amdgpu_device *adev, 2300 enum amd_ip_block_type block_type) 2301 { 2302 int i; 2303 2304 for (i = 0; i < adev->num_ip_blocks; i++) { 2305 if (adev->ip_blocks[i].version->type == block_type) 2306 return adev->ip_blocks[i].status.valid; 2307 } 2308 return false; 2309 2310 } 2311 2312 /** 2313 * amdgpu_device_ip_get_ip_block - get a hw IP pointer 2314 * 2315 * @adev: amdgpu_device pointer 2316 * @type: Type of hardware IP (SMU, GFX, UVD, etc.) 2317 * 2318 * Returns a pointer to the hardware IP block structure 2319 * if it exists for the asic, otherwise NULL. 2320 */ 2321 struct amdgpu_ip_block * 2322 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev, 2323 enum amd_ip_block_type type) 2324 { 2325 int i; 2326 2327 for (i = 0; i < adev->num_ip_blocks; i++) 2328 if (adev->ip_blocks[i].version->type == type) 2329 return &adev->ip_blocks[i]; 2330 2331 return NULL; 2332 } 2333 2334 /** 2335 * amdgpu_device_ip_block_version_cmp 2336 * 2337 * @adev: amdgpu_device pointer 2338 * @type: enum amd_ip_block_type 2339 * @major: major version 2340 * @minor: minor version 2341 * 2342 * return 0 if equal or greater 2343 * return 1 if smaller or the ip_block doesn't exist 2344 */ 2345 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev, 2346 enum amd_ip_block_type type, 2347 u32 major, u32 minor) 2348 { 2349 struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type); 2350 2351 if (ip_block && ((ip_block->version->major > major) || 2352 ((ip_block->version->major == major) && 2353 (ip_block->version->minor >= minor)))) 2354 return 0; 2355 2356 return 1; 2357 } 2358 2359 /** 2360 * amdgpu_device_ip_block_add 2361 * 2362 * @adev: amdgpu_device pointer 2363 * @ip_block_version: pointer to the IP to add 2364 * 2365 * Adds the IP block driver information to the collection of IPs 2366 * on the asic. 2367 */ 2368 int amdgpu_device_ip_block_add(struct amdgpu_device *adev, 2369 const struct amdgpu_ip_block_version *ip_block_version) 2370 { 2371 if (!ip_block_version) 2372 return -EINVAL; 2373 2374 switch (ip_block_version->type) { 2375 case AMD_IP_BLOCK_TYPE_VCN: 2376 if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK) 2377 return 0; 2378 break; 2379 case AMD_IP_BLOCK_TYPE_JPEG: 2380 if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK) 2381 return 0; 2382 break; 2383 default: 2384 break; 2385 } 2386 2387 dev_info(adev->dev, "detected ip block number %d <%s>\n", 2388 adev->num_ip_blocks, ip_block_version->funcs->name); 2389 2390 adev->ip_blocks[adev->num_ip_blocks].adev = adev; 2391 2392 adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version; 2393 2394 return 0; 2395 } 2396 2397 /** 2398 * amdgpu_device_enable_virtual_display - enable virtual display feature 2399 * 2400 * @adev: amdgpu_device pointer 2401 * 2402 * Enabled the virtual display feature if the user has enabled it via 2403 * the module parameter virtual_display. This feature provides a virtual 2404 * display hardware on headless boards or in virtualized environments. 2405 * This function parses and validates the configuration string specified by 2406 * the user and configures the virtual display configuration (number of 2407 * virtual connectors, crtcs, etc.) specified. 2408 */ 2409 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev) 2410 { 2411 adev->enable_virtual_display = false; 2412 2413 if (amdgpu_virtual_display) { 2414 const char *pci_address_name = pci_name(adev->pdev); 2415 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname; 2416 2417 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL); 2418 pciaddstr_tmp = pciaddstr; 2419 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) { 2420 pciaddname = strsep(&pciaddname_tmp, ","); 2421 if (!strcmp("all", pciaddname) 2422 || !strcmp(pci_address_name, pciaddname)) { 2423 long num_crtc; 2424 int res = -1; 2425 2426 adev->enable_virtual_display = true; 2427 2428 if (pciaddname_tmp) 2429 res = kstrtol(pciaddname_tmp, 10, 2430 &num_crtc); 2431 2432 if (!res) { 2433 if (num_crtc < 1) 2434 num_crtc = 1; 2435 if (num_crtc > 6) 2436 num_crtc = 6; 2437 adev->mode_info.num_crtc = num_crtc; 2438 } else { 2439 adev->mode_info.num_crtc = 1; 2440 } 2441 break; 2442 } 2443 } 2444 2445 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n", 2446 amdgpu_virtual_display, pci_address_name, 2447 adev->enable_virtual_display, adev->mode_info.num_crtc); 2448 2449 kfree(pciaddstr); 2450 } 2451 } 2452 2453 void amdgpu_device_set_sriov_virtual_display(struct amdgpu_device *adev) 2454 { 2455 if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) { 2456 adev->mode_info.num_crtc = 1; 2457 adev->enable_virtual_display = true; 2458 DRM_INFO("virtual_display:%d, num_crtc:%d\n", 2459 adev->enable_virtual_display, adev->mode_info.num_crtc); 2460 } 2461 } 2462 2463 /** 2464 * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware 2465 * 2466 * @adev: amdgpu_device pointer 2467 * 2468 * Parses the asic configuration parameters specified in the gpu info 2469 * firmware and makes them available to the driver for use in configuring 2470 * the asic. 2471 * Returns 0 on success, -EINVAL on failure. 2472 */ 2473 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev) 2474 { 2475 const char *chip_name; 2476 int err; 2477 const struct gpu_info_firmware_header_v1_0 *hdr; 2478 2479 adev->firmware.gpu_info_fw = NULL; 2480 2481 if (adev->mman.discovery_bin) 2482 return 0; 2483 2484 switch (adev->asic_type) { 2485 default: 2486 return 0; 2487 case CHIP_VEGA10: 2488 chip_name = "vega10"; 2489 break; 2490 case CHIP_VEGA12: 2491 chip_name = "vega12"; 2492 break; 2493 case CHIP_RAVEN: 2494 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 2495 chip_name = "raven2"; 2496 else if (adev->apu_flags & AMD_APU_IS_PICASSO) 2497 chip_name = "picasso"; 2498 else 2499 chip_name = "raven"; 2500 break; 2501 case CHIP_ARCTURUS: 2502 chip_name = "arcturus"; 2503 break; 2504 case CHIP_NAVI12: 2505 chip_name = "navi12"; 2506 break; 2507 } 2508 2509 err = amdgpu_ucode_request(adev, &adev->firmware.gpu_info_fw, 2510 AMDGPU_UCODE_OPTIONAL, 2511 "amdgpu/%s_gpu_info.bin", chip_name); 2512 if (err) { 2513 dev_err(adev->dev, 2514 "Failed to get gpu_info firmware \"%s_gpu_info.bin\"\n", 2515 chip_name); 2516 goto out; 2517 } 2518 2519 hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data; 2520 amdgpu_ucode_print_gpu_info_hdr(&hdr->header); 2521 2522 switch (hdr->version_major) { 2523 case 1: 2524 { 2525 const struct gpu_info_firmware_v1_0 *gpu_info_fw = 2526 (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data + 2527 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 2528 2529 /* 2530 * Should be dropped when DAL no longer needs it. 2531 */ 2532 if (adev->asic_type == CHIP_NAVI12) 2533 goto parse_soc_bounding_box; 2534 2535 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se); 2536 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh); 2537 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se); 2538 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se); 2539 adev->gfx.config.max_texture_channel_caches = 2540 le32_to_cpu(gpu_info_fw->gc_num_tccs); 2541 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs); 2542 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds); 2543 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth); 2544 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth); 2545 adev->gfx.config.double_offchip_lds_buf = 2546 le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer); 2547 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size); 2548 adev->gfx.cu_info.max_waves_per_simd = 2549 le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd); 2550 adev->gfx.cu_info.max_scratch_slots_per_cu = 2551 le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu); 2552 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size); 2553 if (hdr->version_minor >= 1) { 2554 const struct gpu_info_firmware_v1_1 *gpu_info_fw = 2555 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data + 2556 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 2557 adev->gfx.config.num_sc_per_sh = 2558 le32_to_cpu(gpu_info_fw->num_sc_per_sh); 2559 adev->gfx.config.num_packer_per_sc = 2560 le32_to_cpu(gpu_info_fw->num_packer_per_sc); 2561 } 2562 2563 parse_soc_bounding_box: 2564 /* 2565 * soc bounding box info is not integrated in disocovery table, 2566 * we always need to parse it from gpu info firmware if needed. 2567 */ 2568 if (hdr->version_minor == 2) { 2569 const struct gpu_info_firmware_v1_2 *gpu_info_fw = 2570 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data + 2571 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 2572 adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box; 2573 } 2574 break; 2575 } 2576 default: 2577 dev_err(adev->dev, 2578 "Unsupported gpu_info table %d\n", hdr->header.ucode_version); 2579 err = -EINVAL; 2580 goto out; 2581 } 2582 out: 2583 return err; 2584 } 2585 2586 /** 2587 * amdgpu_device_ip_early_init - run early init for hardware IPs 2588 * 2589 * @adev: amdgpu_device pointer 2590 * 2591 * Early initialization pass for hardware IPs. The hardware IPs that make 2592 * up each asic are discovered each IP's early_init callback is run. This 2593 * is the first stage in initializing the asic. 2594 * Returns 0 on success, negative error code on failure. 2595 */ 2596 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev) 2597 { 2598 struct amdgpu_ip_block *ip_block; 2599 struct pci_dev *parent; 2600 bool total, skip_bios; 2601 uint32_t bios_flags; 2602 int i, r; 2603 2604 amdgpu_device_enable_virtual_display(adev); 2605 2606 if (amdgpu_sriov_vf(adev)) { 2607 r = amdgpu_virt_request_full_gpu(adev, true); 2608 if (r) 2609 return r; 2610 } 2611 2612 switch (adev->asic_type) { 2613 #ifdef CONFIG_DRM_AMDGPU_SI 2614 case CHIP_VERDE: 2615 case CHIP_TAHITI: 2616 case CHIP_PITCAIRN: 2617 case CHIP_OLAND: 2618 case CHIP_HAINAN: 2619 adev->family = AMDGPU_FAMILY_SI; 2620 r = si_set_ip_blocks(adev); 2621 if (r) 2622 return r; 2623 break; 2624 #endif 2625 #ifdef CONFIG_DRM_AMDGPU_CIK 2626 case CHIP_BONAIRE: 2627 case CHIP_HAWAII: 2628 case CHIP_KAVERI: 2629 case CHIP_KABINI: 2630 case CHIP_MULLINS: 2631 if (adev->flags & AMD_IS_APU) 2632 adev->family = AMDGPU_FAMILY_KV; 2633 else 2634 adev->family = AMDGPU_FAMILY_CI; 2635 2636 r = cik_set_ip_blocks(adev); 2637 if (r) 2638 return r; 2639 break; 2640 #endif 2641 case CHIP_TOPAZ: 2642 case CHIP_TONGA: 2643 case CHIP_FIJI: 2644 case CHIP_POLARIS10: 2645 case CHIP_POLARIS11: 2646 case CHIP_POLARIS12: 2647 case CHIP_VEGAM: 2648 case CHIP_CARRIZO: 2649 case CHIP_STONEY: 2650 if (adev->flags & AMD_IS_APU) 2651 adev->family = AMDGPU_FAMILY_CZ; 2652 else 2653 adev->family = AMDGPU_FAMILY_VI; 2654 2655 r = vi_set_ip_blocks(adev); 2656 if (r) 2657 return r; 2658 break; 2659 default: 2660 r = amdgpu_discovery_set_ip_blocks(adev); 2661 if (r) 2662 return r; 2663 break; 2664 } 2665 2666 if (amdgpu_has_atpx() && 2667 (amdgpu_is_atpx_hybrid() || 2668 amdgpu_has_atpx_dgpu_power_cntl()) && 2669 ((adev->flags & AMD_IS_APU) == 0) && 2670 !dev_is_removable(&adev->pdev->dev)) 2671 adev->flags |= AMD_IS_PX; 2672 2673 if (!(adev->flags & AMD_IS_APU)) { 2674 parent = pcie_find_root_port(adev->pdev); 2675 adev->has_pr3 = parent ? pci_pr3_present(parent) : false; 2676 } 2677 2678 2679 adev->pm.pp_feature = amdgpu_pp_feature_mask; 2680 if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS) 2681 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 2682 if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID) 2683 adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK; 2684 if (!amdgpu_device_pcie_dynamic_switching_supported(adev)) 2685 adev->pm.pp_feature &= ~PP_PCIE_DPM_MASK; 2686 2687 total = true; 2688 for (i = 0; i < adev->num_ip_blocks; i++) { 2689 ip_block = &adev->ip_blocks[i]; 2690 2691 if ((amdgpu_ip_block_mask & (1 << i)) == 0) { 2692 DRM_WARN("disabled ip block: %d <%s>\n", 2693 i, adev->ip_blocks[i].version->funcs->name); 2694 adev->ip_blocks[i].status.valid = false; 2695 } else if (ip_block->version->funcs->early_init) { 2696 r = ip_block->version->funcs->early_init(ip_block); 2697 if (r == -ENOENT) { 2698 adev->ip_blocks[i].status.valid = false; 2699 } else if (r) { 2700 DRM_ERROR("early_init of IP block <%s> failed %d\n", 2701 adev->ip_blocks[i].version->funcs->name, r); 2702 total = false; 2703 } else { 2704 adev->ip_blocks[i].status.valid = true; 2705 } 2706 } else { 2707 adev->ip_blocks[i].status.valid = true; 2708 } 2709 /* get the vbios after the asic_funcs are set up */ 2710 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) { 2711 r = amdgpu_device_parse_gpu_info_fw(adev); 2712 if (r) 2713 return r; 2714 2715 bios_flags = amdgpu_device_get_vbios_flags(adev); 2716 skip_bios = !!(bios_flags & AMDGPU_VBIOS_SKIP); 2717 /* Read BIOS */ 2718 if (!skip_bios) { 2719 bool optional = 2720 !!(bios_flags & AMDGPU_VBIOS_OPTIONAL); 2721 if (!amdgpu_get_bios(adev) && !optional) 2722 return -EINVAL; 2723 2724 if (optional && !adev->bios) 2725 dev_info( 2726 adev->dev, 2727 "VBIOS image optional, proceeding without VBIOS image"); 2728 2729 if (adev->bios) { 2730 r = amdgpu_atombios_init(adev); 2731 if (r) { 2732 dev_err(adev->dev, 2733 "amdgpu_atombios_init failed\n"); 2734 amdgpu_vf_error_put( 2735 adev, 2736 AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 2737 0, 0); 2738 return r; 2739 } 2740 } 2741 } 2742 2743 /*get pf2vf msg info at it's earliest time*/ 2744 if (amdgpu_sriov_vf(adev)) 2745 amdgpu_virt_init_data_exchange(adev); 2746 2747 } 2748 } 2749 if (!total) 2750 return -ENODEV; 2751 2752 ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX); 2753 if (ip_block->status.valid != false) 2754 amdgpu_amdkfd_device_probe(adev); 2755 2756 adev->cg_flags &= amdgpu_cg_mask; 2757 adev->pg_flags &= amdgpu_pg_mask; 2758 2759 return 0; 2760 } 2761 2762 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev) 2763 { 2764 int i, r; 2765 2766 for (i = 0; i < adev->num_ip_blocks; i++) { 2767 if (!adev->ip_blocks[i].status.sw) 2768 continue; 2769 if (adev->ip_blocks[i].status.hw) 2770 continue; 2771 if (!amdgpu_ip_member_of_hwini( 2772 adev, adev->ip_blocks[i].version->type)) 2773 continue; 2774 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || 2775 (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) || 2776 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) { 2777 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]); 2778 if (r) { 2779 DRM_ERROR("hw_init of IP block <%s> failed %d\n", 2780 adev->ip_blocks[i].version->funcs->name, r); 2781 return r; 2782 } 2783 adev->ip_blocks[i].status.hw = true; 2784 } 2785 } 2786 2787 return 0; 2788 } 2789 2790 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev) 2791 { 2792 int i, r; 2793 2794 for (i = 0; i < adev->num_ip_blocks; i++) { 2795 if (!adev->ip_blocks[i].status.sw) 2796 continue; 2797 if (adev->ip_blocks[i].status.hw) 2798 continue; 2799 if (!amdgpu_ip_member_of_hwini( 2800 adev, adev->ip_blocks[i].version->type)) 2801 continue; 2802 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]); 2803 if (r) { 2804 DRM_ERROR("hw_init of IP block <%s> failed %d\n", 2805 adev->ip_blocks[i].version->funcs->name, r); 2806 return r; 2807 } 2808 adev->ip_blocks[i].status.hw = true; 2809 } 2810 2811 return 0; 2812 } 2813 2814 static int amdgpu_device_fw_loading(struct amdgpu_device *adev) 2815 { 2816 int r = 0; 2817 int i; 2818 uint32_t smu_version; 2819 2820 if (adev->asic_type >= CHIP_VEGA10) { 2821 for (i = 0; i < adev->num_ip_blocks; i++) { 2822 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP) 2823 continue; 2824 2825 if (!amdgpu_ip_member_of_hwini(adev, 2826 AMD_IP_BLOCK_TYPE_PSP)) 2827 break; 2828 2829 if (!adev->ip_blocks[i].status.sw) 2830 continue; 2831 2832 /* no need to do the fw loading again if already done*/ 2833 if (adev->ip_blocks[i].status.hw == true) 2834 break; 2835 2836 if (amdgpu_in_reset(adev) || adev->in_suspend) { 2837 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]); 2838 if (r) 2839 return r; 2840 } else { 2841 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]); 2842 if (r) { 2843 DRM_ERROR("hw_init of IP block <%s> failed %d\n", 2844 adev->ip_blocks[i].version->funcs->name, r); 2845 return r; 2846 } 2847 adev->ip_blocks[i].status.hw = true; 2848 } 2849 break; 2850 } 2851 } 2852 2853 if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA) 2854 r = amdgpu_pm_load_smu_firmware(adev, &smu_version); 2855 2856 return r; 2857 } 2858 2859 static int amdgpu_device_init_schedulers(struct amdgpu_device *adev) 2860 { 2861 long timeout; 2862 int r, i; 2863 2864 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 2865 struct amdgpu_ring *ring = adev->rings[i]; 2866 2867 /* No need to setup the GPU scheduler for rings that don't need it */ 2868 if (!ring || ring->no_scheduler) 2869 continue; 2870 2871 switch (ring->funcs->type) { 2872 case AMDGPU_RING_TYPE_GFX: 2873 timeout = adev->gfx_timeout; 2874 break; 2875 case AMDGPU_RING_TYPE_COMPUTE: 2876 timeout = adev->compute_timeout; 2877 break; 2878 case AMDGPU_RING_TYPE_SDMA: 2879 timeout = adev->sdma_timeout; 2880 break; 2881 default: 2882 timeout = adev->video_timeout; 2883 break; 2884 } 2885 2886 r = drm_sched_init(&ring->sched, &amdgpu_sched_ops, NULL, 2887 DRM_SCHED_PRIORITY_COUNT, 2888 ring->num_hw_submission, 0, 2889 timeout, adev->reset_domain->wq, 2890 ring->sched_score, ring->name, 2891 adev->dev); 2892 if (r) { 2893 DRM_ERROR("Failed to create scheduler on ring %s.\n", 2894 ring->name); 2895 return r; 2896 } 2897 r = amdgpu_uvd_entity_init(adev, ring); 2898 if (r) { 2899 DRM_ERROR("Failed to create UVD scheduling entity on ring %s.\n", 2900 ring->name); 2901 return r; 2902 } 2903 r = amdgpu_vce_entity_init(adev, ring); 2904 if (r) { 2905 DRM_ERROR("Failed to create VCE scheduling entity on ring %s.\n", 2906 ring->name); 2907 return r; 2908 } 2909 } 2910 2911 amdgpu_xcp_update_partition_sched_list(adev); 2912 2913 return 0; 2914 } 2915 2916 2917 /** 2918 * amdgpu_device_ip_init - run init for hardware IPs 2919 * 2920 * @adev: amdgpu_device pointer 2921 * 2922 * Main initialization pass for hardware IPs. The list of all the hardware 2923 * IPs that make up the asic is walked and the sw_init and hw_init callbacks 2924 * are run. sw_init initializes the software state associated with each IP 2925 * and hw_init initializes the hardware associated with each IP. 2926 * Returns 0 on success, negative error code on failure. 2927 */ 2928 static int amdgpu_device_ip_init(struct amdgpu_device *adev) 2929 { 2930 bool init_badpage; 2931 int i, r; 2932 2933 r = amdgpu_ras_init(adev); 2934 if (r) 2935 return r; 2936 2937 for (i = 0; i < adev->num_ip_blocks; i++) { 2938 if (!adev->ip_blocks[i].status.valid) 2939 continue; 2940 if (adev->ip_blocks[i].version->funcs->sw_init) { 2941 r = adev->ip_blocks[i].version->funcs->sw_init(&adev->ip_blocks[i]); 2942 if (r) { 2943 DRM_ERROR("sw_init of IP block <%s> failed %d\n", 2944 adev->ip_blocks[i].version->funcs->name, r); 2945 goto init_failed; 2946 } 2947 } 2948 adev->ip_blocks[i].status.sw = true; 2949 2950 if (!amdgpu_ip_member_of_hwini( 2951 adev, adev->ip_blocks[i].version->type)) 2952 continue; 2953 2954 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) { 2955 /* need to do common hw init early so everything is set up for gmc */ 2956 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]); 2957 if (r) { 2958 DRM_ERROR("hw_init %d failed %d\n", i, r); 2959 goto init_failed; 2960 } 2961 adev->ip_blocks[i].status.hw = true; 2962 } else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { 2963 /* need to do gmc hw init early so we can allocate gpu mem */ 2964 /* Try to reserve bad pages early */ 2965 if (amdgpu_sriov_vf(adev)) 2966 amdgpu_virt_exchange_data(adev); 2967 2968 r = amdgpu_device_mem_scratch_init(adev); 2969 if (r) { 2970 DRM_ERROR("amdgpu_mem_scratch_init failed %d\n", r); 2971 goto init_failed; 2972 } 2973 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]); 2974 if (r) { 2975 DRM_ERROR("hw_init %d failed %d\n", i, r); 2976 goto init_failed; 2977 } 2978 r = amdgpu_device_wb_init(adev); 2979 if (r) { 2980 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r); 2981 goto init_failed; 2982 } 2983 adev->ip_blocks[i].status.hw = true; 2984 2985 /* right after GMC hw init, we create CSA */ 2986 if (adev->gfx.mcbp) { 2987 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj, 2988 AMDGPU_GEM_DOMAIN_VRAM | 2989 AMDGPU_GEM_DOMAIN_GTT, 2990 AMDGPU_CSA_SIZE); 2991 if (r) { 2992 DRM_ERROR("allocate CSA failed %d\n", r); 2993 goto init_failed; 2994 } 2995 } 2996 2997 r = amdgpu_seq64_init(adev); 2998 if (r) { 2999 DRM_ERROR("allocate seq64 failed %d\n", r); 3000 goto init_failed; 3001 } 3002 } 3003 } 3004 3005 if (amdgpu_sriov_vf(adev)) 3006 amdgpu_virt_init_data_exchange(adev); 3007 3008 r = amdgpu_ib_pool_init(adev); 3009 if (r) { 3010 dev_err(adev->dev, "IB initialization failed (%d).\n", r); 3011 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r); 3012 goto init_failed; 3013 } 3014 3015 r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/ 3016 if (r) 3017 goto init_failed; 3018 3019 r = amdgpu_device_ip_hw_init_phase1(adev); 3020 if (r) 3021 goto init_failed; 3022 3023 r = amdgpu_device_fw_loading(adev); 3024 if (r) 3025 goto init_failed; 3026 3027 r = amdgpu_device_ip_hw_init_phase2(adev); 3028 if (r) 3029 goto init_failed; 3030 3031 /* 3032 * retired pages will be loaded from eeprom and reserved here, 3033 * it should be called after amdgpu_device_ip_hw_init_phase2 since 3034 * for some ASICs the RAS EEPROM code relies on SMU fully functioning 3035 * for I2C communication which only true at this point. 3036 * 3037 * amdgpu_ras_recovery_init may fail, but the upper only cares the 3038 * failure from bad gpu situation and stop amdgpu init process 3039 * accordingly. For other failed cases, it will still release all 3040 * the resource and print error message, rather than returning one 3041 * negative value to upper level. 3042 * 3043 * Note: theoretically, this should be called before all vram allocations 3044 * to protect retired page from abusing 3045 */ 3046 init_badpage = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI); 3047 r = amdgpu_ras_recovery_init(adev, init_badpage); 3048 if (r) 3049 goto init_failed; 3050 3051 /** 3052 * In case of XGMI grab extra reference for reset domain for this device 3053 */ 3054 if (adev->gmc.xgmi.num_physical_nodes > 1) { 3055 if (amdgpu_xgmi_add_device(adev) == 0) { 3056 if (!amdgpu_sriov_vf(adev)) { 3057 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 3058 3059 if (WARN_ON(!hive)) { 3060 r = -ENOENT; 3061 goto init_failed; 3062 } 3063 3064 if (!hive->reset_domain || 3065 !amdgpu_reset_get_reset_domain(hive->reset_domain)) { 3066 r = -ENOENT; 3067 amdgpu_put_xgmi_hive(hive); 3068 goto init_failed; 3069 } 3070 3071 /* Drop the early temporary reset domain we created for device */ 3072 amdgpu_reset_put_reset_domain(adev->reset_domain); 3073 adev->reset_domain = hive->reset_domain; 3074 amdgpu_put_xgmi_hive(hive); 3075 } 3076 } 3077 } 3078 3079 r = amdgpu_device_init_schedulers(adev); 3080 if (r) 3081 goto init_failed; 3082 3083 if (adev->mman.buffer_funcs_ring->sched.ready) 3084 amdgpu_ttm_set_buffer_funcs_status(adev, true); 3085 3086 /* Don't init kfd if whole hive need to be reset during init */ 3087 if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) { 3088 kgd2kfd_init_zone_device(adev); 3089 amdgpu_amdkfd_device_init(adev); 3090 } 3091 3092 amdgpu_fru_get_product_info(adev); 3093 3094 if (amdgpu_aca_is_enabled(adev)) 3095 r = amdgpu_cper_init(adev); 3096 3097 init_failed: 3098 3099 return r; 3100 } 3101 3102 /** 3103 * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer 3104 * 3105 * @adev: amdgpu_device pointer 3106 * 3107 * Writes a reset magic value to the gart pointer in VRAM. The driver calls 3108 * this function before a GPU reset. If the value is retained after a 3109 * GPU reset, VRAM has not been lost. Some GPU resets may destroy VRAM contents. 3110 */ 3111 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev) 3112 { 3113 memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM); 3114 } 3115 3116 /** 3117 * amdgpu_device_check_vram_lost - check if vram is valid 3118 * 3119 * @adev: amdgpu_device pointer 3120 * 3121 * Checks the reset magic value written to the gart pointer in VRAM. 3122 * The driver calls this after a GPU reset to see if the contents of 3123 * VRAM is lost or now. 3124 * returns true if vram is lost, false if not. 3125 */ 3126 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev) 3127 { 3128 if (memcmp(adev->gart.ptr, adev->reset_magic, 3129 AMDGPU_RESET_MAGIC_NUM)) 3130 return true; 3131 3132 if (!amdgpu_in_reset(adev)) 3133 return false; 3134 3135 /* 3136 * For all ASICs with baco/mode1 reset, the VRAM is 3137 * always assumed to be lost. 3138 */ 3139 switch (amdgpu_asic_reset_method(adev)) { 3140 case AMD_RESET_METHOD_BACO: 3141 case AMD_RESET_METHOD_MODE1: 3142 return true; 3143 default: 3144 return false; 3145 } 3146 } 3147 3148 /** 3149 * amdgpu_device_set_cg_state - set clockgating for amdgpu device 3150 * 3151 * @adev: amdgpu_device pointer 3152 * @state: clockgating state (gate or ungate) 3153 * 3154 * The list of all the hardware IPs that make up the asic is walked and the 3155 * set_clockgating_state callbacks are run. 3156 * Late initialization pass enabling clockgating for hardware IPs. 3157 * Fini or suspend, pass disabling clockgating for hardware IPs. 3158 * Returns 0 on success, negative error code on failure. 3159 */ 3160 3161 int amdgpu_device_set_cg_state(struct amdgpu_device *adev, 3162 enum amd_clockgating_state state) 3163 { 3164 int i, j, r; 3165 3166 if (amdgpu_emu_mode == 1) 3167 return 0; 3168 3169 for (j = 0; j < adev->num_ip_blocks; j++) { 3170 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; 3171 if (!adev->ip_blocks[i].status.late_initialized) 3172 continue; 3173 /* skip CG for GFX, SDMA on S0ix */ 3174 if (adev->in_s0ix && 3175 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX || 3176 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA)) 3177 continue; 3178 /* skip CG for VCE/UVD, it's handled specially */ 3179 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && 3180 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && 3181 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && 3182 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG && 3183 adev->ip_blocks[i].version->funcs->set_clockgating_state) { 3184 /* enable clockgating to save power */ 3185 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(&adev->ip_blocks[i], 3186 state); 3187 if (r) { 3188 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n", 3189 adev->ip_blocks[i].version->funcs->name, r); 3190 return r; 3191 } 3192 } 3193 } 3194 3195 return 0; 3196 } 3197 3198 int amdgpu_device_set_pg_state(struct amdgpu_device *adev, 3199 enum amd_powergating_state state) 3200 { 3201 int i, j, r; 3202 3203 if (amdgpu_emu_mode == 1) 3204 return 0; 3205 3206 for (j = 0; j < adev->num_ip_blocks; j++) { 3207 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; 3208 if (!adev->ip_blocks[i].status.late_initialized) 3209 continue; 3210 /* skip PG for GFX, SDMA on S0ix */ 3211 if (adev->in_s0ix && 3212 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX || 3213 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA)) 3214 continue; 3215 /* skip CG for VCE/UVD, it's handled specially */ 3216 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && 3217 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && 3218 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && 3219 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG && 3220 adev->ip_blocks[i].version->funcs->set_powergating_state) { 3221 /* enable powergating to save power */ 3222 r = adev->ip_blocks[i].version->funcs->set_powergating_state(&adev->ip_blocks[i], 3223 state); 3224 if (r) { 3225 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n", 3226 adev->ip_blocks[i].version->funcs->name, r); 3227 return r; 3228 } 3229 } 3230 } 3231 return 0; 3232 } 3233 3234 static int amdgpu_device_enable_mgpu_fan_boost(void) 3235 { 3236 struct amdgpu_gpu_instance *gpu_ins; 3237 struct amdgpu_device *adev; 3238 int i, ret = 0; 3239 3240 mutex_lock(&mgpu_info.mutex); 3241 3242 /* 3243 * MGPU fan boost feature should be enabled 3244 * only when there are two or more dGPUs in 3245 * the system 3246 */ 3247 if (mgpu_info.num_dgpu < 2) 3248 goto out; 3249 3250 for (i = 0; i < mgpu_info.num_dgpu; i++) { 3251 gpu_ins = &(mgpu_info.gpu_ins[i]); 3252 adev = gpu_ins->adev; 3253 if (!(adev->flags & AMD_IS_APU) && 3254 !gpu_ins->mgpu_fan_enabled) { 3255 ret = amdgpu_dpm_enable_mgpu_fan_boost(adev); 3256 if (ret) 3257 break; 3258 3259 gpu_ins->mgpu_fan_enabled = 1; 3260 } 3261 } 3262 3263 out: 3264 mutex_unlock(&mgpu_info.mutex); 3265 3266 return ret; 3267 } 3268 3269 /** 3270 * amdgpu_device_ip_late_init - run late init for hardware IPs 3271 * 3272 * @adev: amdgpu_device pointer 3273 * 3274 * Late initialization pass for hardware IPs. The list of all the hardware 3275 * IPs that make up the asic is walked and the late_init callbacks are run. 3276 * late_init covers any special initialization that an IP requires 3277 * after all of the have been initialized or something that needs to happen 3278 * late in the init process. 3279 * Returns 0 on success, negative error code on failure. 3280 */ 3281 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev) 3282 { 3283 struct amdgpu_gpu_instance *gpu_instance; 3284 int i = 0, r; 3285 3286 for (i = 0; i < adev->num_ip_blocks; i++) { 3287 if (!adev->ip_blocks[i].status.hw) 3288 continue; 3289 if (adev->ip_blocks[i].version->funcs->late_init) { 3290 r = adev->ip_blocks[i].version->funcs->late_init(&adev->ip_blocks[i]); 3291 if (r) { 3292 DRM_ERROR("late_init of IP block <%s> failed %d\n", 3293 adev->ip_blocks[i].version->funcs->name, r); 3294 return r; 3295 } 3296 } 3297 adev->ip_blocks[i].status.late_initialized = true; 3298 } 3299 3300 r = amdgpu_ras_late_init(adev); 3301 if (r) { 3302 DRM_ERROR("amdgpu_ras_late_init failed %d", r); 3303 return r; 3304 } 3305 3306 if (!amdgpu_reset_in_recovery(adev)) 3307 amdgpu_ras_set_error_query_ready(adev, true); 3308 3309 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE); 3310 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE); 3311 3312 amdgpu_device_fill_reset_magic(adev); 3313 3314 r = amdgpu_device_enable_mgpu_fan_boost(); 3315 if (r) 3316 DRM_ERROR("enable mgpu fan boost failed (%d).\n", r); 3317 3318 /* For passthrough configuration on arcturus and aldebaran, enable special handling SBR */ 3319 if (amdgpu_passthrough(adev) && 3320 ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1) || 3321 adev->asic_type == CHIP_ALDEBARAN)) 3322 amdgpu_dpm_handle_passthrough_sbr(adev, true); 3323 3324 if (adev->gmc.xgmi.num_physical_nodes > 1) { 3325 mutex_lock(&mgpu_info.mutex); 3326 3327 /* 3328 * Reset device p-state to low as this was booted with high. 3329 * 3330 * This should be performed only after all devices from the same 3331 * hive get initialized. 3332 * 3333 * However, it's unknown how many device in the hive in advance. 3334 * As this is counted one by one during devices initializations. 3335 * 3336 * So, we wait for all XGMI interlinked devices initialized. 3337 * This may bring some delays as those devices may come from 3338 * different hives. But that should be OK. 3339 */ 3340 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) { 3341 for (i = 0; i < mgpu_info.num_gpu; i++) { 3342 gpu_instance = &(mgpu_info.gpu_ins[i]); 3343 if (gpu_instance->adev->flags & AMD_IS_APU) 3344 continue; 3345 3346 r = amdgpu_xgmi_set_pstate(gpu_instance->adev, 3347 AMDGPU_XGMI_PSTATE_MIN); 3348 if (r) { 3349 DRM_ERROR("pstate setting failed (%d).\n", r); 3350 break; 3351 } 3352 } 3353 } 3354 3355 mutex_unlock(&mgpu_info.mutex); 3356 } 3357 3358 return 0; 3359 } 3360 3361 static void amdgpu_ip_block_hw_fini(struct amdgpu_ip_block *ip_block) 3362 { 3363 int r; 3364 3365 if (!ip_block->version->funcs->hw_fini) { 3366 DRM_ERROR("hw_fini of IP block <%s> not defined\n", 3367 ip_block->version->funcs->name); 3368 } else { 3369 r = ip_block->version->funcs->hw_fini(ip_block); 3370 /* XXX handle errors */ 3371 if (r) { 3372 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n", 3373 ip_block->version->funcs->name, r); 3374 } 3375 } 3376 3377 ip_block->status.hw = false; 3378 } 3379 3380 /** 3381 * amdgpu_device_smu_fini_early - smu hw_fini wrapper 3382 * 3383 * @adev: amdgpu_device pointer 3384 * 3385 * For ASICs need to disable SMC first 3386 */ 3387 static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev) 3388 { 3389 int i; 3390 3391 if (amdgpu_ip_version(adev, GC_HWIP, 0) > IP_VERSION(9, 0, 0)) 3392 return; 3393 3394 for (i = 0; i < adev->num_ip_blocks; i++) { 3395 if (!adev->ip_blocks[i].status.hw) 3396 continue; 3397 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) { 3398 amdgpu_ip_block_hw_fini(&adev->ip_blocks[i]); 3399 break; 3400 } 3401 } 3402 } 3403 3404 static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev) 3405 { 3406 int i, r; 3407 3408 for (i = 0; i < adev->num_ip_blocks; i++) { 3409 if (!adev->ip_blocks[i].version->funcs->early_fini) 3410 continue; 3411 3412 r = adev->ip_blocks[i].version->funcs->early_fini(&adev->ip_blocks[i]); 3413 if (r) { 3414 DRM_DEBUG("early_fini of IP block <%s> failed %d\n", 3415 adev->ip_blocks[i].version->funcs->name, r); 3416 } 3417 } 3418 3419 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); 3420 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); 3421 3422 amdgpu_amdkfd_suspend(adev, false); 3423 3424 /* Workaround for ASICs need to disable SMC first */ 3425 amdgpu_device_smu_fini_early(adev); 3426 3427 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 3428 if (!adev->ip_blocks[i].status.hw) 3429 continue; 3430 3431 amdgpu_ip_block_hw_fini(&adev->ip_blocks[i]); 3432 } 3433 3434 if (amdgpu_sriov_vf(adev)) { 3435 if (amdgpu_virt_release_full_gpu(adev, false)) 3436 DRM_ERROR("failed to release exclusive mode on fini\n"); 3437 } 3438 3439 return 0; 3440 } 3441 3442 /** 3443 * amdgpu_device_ip_fini - run fini for hardware IPs 3444 * 3445 * @adev: amdgpu_device pointer 3446 * 3447 * Main teardown pass for hardware IPs. The list of all the hardware 3448 * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks 3449 * are run. hw_fini tears down the hardware associated with each IP 3450 * and sw_fini tears down any software state associated with each IP. 3451 * Returns 0 on success, negative error code on failure. 3452 */ 3453 static int amdgpu_device_ip_fini(struct amdgpu_device *adev) 3454 { 3455 int i, r; 3456 3457 if (amdgpu_aca_is_enabled(adev)) 3458 amdgpu_cper_fini(adev); 3459 3460 if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done) 3461 amdgpu_virt_release_ras_err_handler_data(adev); 3462 3463 if (adev->gmc.xgmi.num_physical_nodes > 1) 3464 amdgpu_xgmi_remove_device(adev); 3465 3466 amdgpu_amdkfd_device_fini_sw(adev); 3467 3468 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 3469 if (!adev->ip_blocks[i].status.sw) 3470 continue; 3471 3472 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { 3473 amdgpu_ucode_free_bo(adev); 3474 amdgpu_free_static_csa(&adev->virt.csa_obj); 3475 amdgpu_device_wb_fini(adev); 3476 amdgpu_device_mem_scratch_fini(adev); 3477 amdgpu_ib_pool_fini(adev); 3478 amdgpu_seq64_fini(adev); 3479 } 3480 if (adev->ip_blocks[i].version->funcs->sw_fini) { 3481 r = adev->ip_blocks[i].version->funcs->sw_fini(&adev->ip_blocks[i]); 3482 /* XXX handle errors */ 3483 if (r) { 3484 DRM_DEBUG("sw_fini of IP block <%s> failed %d\n", 3485 adev->ip_blocks[i].version->funcs->name, r); 3486 } 3487 } 3488 adev->ip_blocks[i].status.sw = false; 3489 adev->ip_blocks[i].status.valid = false; 3490 } 3491 3492 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 3493 if (!adev->ip_blocks[i].status.late_initialized) 3494 continue; 3495 if (adev->ip_blocks[i].version->funcs->late_fini) 3496 adev->ip_blocks[i].version->funcs->late_fini(&adev->ip_blocks[i]); 3497 adev->ip_blocks[i].status.late_initialized = false; 3498 } 3499 3500 amdgpu_ras_fini(adev); 3501 3502 return 0; 3503 } 3504 3505 /** 3506 * amdgpu_device_delayed_init_work_handler - work handler for IB tests 3507 * 3508 * @work: work_struct. 3509 */ 3510 static void amdgpu_device_delayed_init_work_handler(struct work_struct *work) 3511 { 3512 struct amdgpu_device *adev = 3513 container_of(work, struct amdgpu_device, delayed_init_work.work); 3514 int r; 3515 3516 r = amdgpu_ib_ring_tests(adev); 3517 if (r) 3518 DRM_ERROR("ib ring test failed (%d).\n", r); 3519 } 3520 3521 static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work) 3522 { 3523 struct amdgpu_device *adev = 3524 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work); 3525 3526 WARN_ON_ONCE(adev->gfx.gfx_off_state); 3527 WARN_ON_ONCE(adev->gfx.gfx_off_req_count); 3528 3529 if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true, 0)) 3530 adev->gfx.gfx_off_state = true; 3531 } 3532 3533 /** 3534 * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1) 3535 * 3536 * @adev: amdgpu_device pointer 3537 * 3538 * Main suspend function for hardware IPs. The list of all the hardware 3539 * IPs that make up the asic is walked, clockgating is disabled and the 3540 * suspend callbacks are run. suspend puts the hardware and software state 3541 * in each IP into a state suitable for suspend. 3542 * Returns 0 on success, negative error code on failure. 3543 */ 3544 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev) 3545 { 3546 int i, r; 3547 3548 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); 3549 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); 3550 3551 /* 3552 * Per PMFW team's suggestion, driver needs to handle gfxoff 3553 * and df cstate features disablement for gpu reset(e.g. Mode1Reset) 3554 * scenario. Add the missing df cstate disablement here. 3555 */ 3556 if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW)) 3557 dev_warn(adev->dev, "Failed to disallow df cstate"); 3558 3559 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 3560 if (!adev->ip_blocks[i].status.valid) 3561 continue; 3562 3563 /* displays are handled separately */ 3564 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE) 3565 continue; 3566 3567 /* XXX handle errors */ 3568 r = amdgpu_ip_block_suspend(&adev->ip_blocks[i]); 3569 if (r) 3570 return r; 3571 } 3572 3573 return 0; 3574 } 3575 3576 /** 3577 * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2) 3578 * 3579 * @adev: amdgpu_device pointer 3580 * 3581 * Main suspend function for hardware IPs. The list of all the hardware 3582 * IPs that make up the asic is walked, clockgating is disabled and the 3583 * suspend callbacks are run. suspend puts the hardware and software state 3584 * in each IP into a state suitable for suspend. 3585 * Returns 0 on success, negative error code on failure. 3586 */ 3587 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev) 3588 { 3589 int i, r; 3590 3591 if (adev->in_s0ix) 3592 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D3Entry); 3593 3594 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 3595 if (!adev->ip_blocks[i].status.valid) 3596 continue; 3597 /* displays are handled in phase1 */ 3598 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) 3599 continue; 3600 /* PSP lost connection when err_event_athub occurs */ 3601 if (amdgpu_ras_intr_triggered() && 3602 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) { 3603 adev->ip_blocks[i].status.hw = false; 3604 continue; 3605 } 3606 3607 /* skip unnecessary suspend if we do not initialize them yet */ 3608 if (!amdgpu_ip_member_of_hwini( 3609 adev, adev->ip_blocks[i].version->type)) 3610 continue; 3611 3612 /* skip suspend of gfx/mes and psp for S0ix 3613 * gfx is in gfxoff state, so on resume it will exit gfxoff just 3614 * like at runtime. PSP is also part of the always on hardware 3615 * so no need to suspend it. 3616 */ 3617 if (adev->in_s0ix && 3618 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP || 3619 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX || 3620 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_MES)) 3621 continue; 3622 3623 /* SDMA 5.x+ is part of GFX power domain so it's covered by GFXOFF */ 3624 if (adev->in_s0ix && 3625 (amdgpu_ip_version(adev, SDMA0_HWIP, 0) >= 3626 IP_VERSION(5, 0, 0)) && 3627 (adev->ip_blocks[i].version->type == 3628 AMD_IP_BLOCK_TYPE_SDMA)) 3629 continue; 3630 3631 /* Once swPSP provides the IMU, RLC FW binaries to TOS during cold-boot. 3632 * These are in TMR, hence are expected to be reused by PSP-TOS to reload 3633 * from this location and RLC Autoload automatically also gets loaded 3634 * from here based on PMFW -> PSP message during re-init sequence. 3635 * Therefore, the psp suspend & resume should be skipped to avoid destroy 3636 * the TMR and reload FWs again for IMU enabled APU ASICs. 3637 */ 3638 if (amdgpu_in_reset(adev) && 3639 (adev->flags & AMD_IS_APU) && adev->gfx.imu.funcs && 3640 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) 3641 continue; 3642 3643 /* XXX handle errors */ 3644 r = amdgpu_ip_block_suspend(&adev->ip_blocks[i]); 3645 adev->ip_blocks[i].status.hw = false; 3646 3647 /* handle putting the SMC in the appropriate state */ 3648 if (!amdgpu_sriov_vf(adev)) { 3649 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) { 3650 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state); 3651 if (r) { 3652 DRM_ERROR("SMC failed to set mp1 state %d, %d\n", 3653 adev->mp1_state, r); 3654 return r; 3655 } 3656 } 3657 } 3658 } 3659 3660 return 0; 3661 } 3662 3663 /** 3664 * amdgpu_device_ip_suspend - run suspend for hardware IPs 3665 * 3666 * @adev: amdgpu_device pointer 3667 * 3668 * Main suspend function for hardware IPs. The list of all the hardware 3669 * IPs that make up the asic is walked, clockgating is disabled and the 3670 * suspend callbacks are run. suspend puts the hardware and software state 3671 * in each IP into a state suitable for suspend. 3672 * Returns 0 on success, negative error code on failure. 3673 */ 3674 int amdgpu_device_ip_suspend(struct amdgpu_device *adev) 3675 { 3676 int r; 3677 3678 if (amdgpu_sriov_vf(adev)) { 3679 amdgpu_virt_fini_data_exchange(adev); 3680 amdgpu_virt_request_full_gpu(adev, false); 3681 } 3682 3683 amdgpu_ttm_set_buffer_funcs_status(adev, false); 3684 3685 r = amdgpu_device_ip_suspend_phase1(adev); 3686 if (r) 3687 return r; 3688 r = amdgpu_device_ip_suspend_phase2(adev); 3689 3690 if (amdgpu_sriov_vf(adev)) 3691 amdgpu_virt_release_full_gpu(adev, false); 3692 3693 return r; 3694 } 3695 3696 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev) 3697 { 3698 int i, r; 3699 3700 static enum amd_ip_block_type ip_order[] = { 3701 AMD_IP_BLOCK_TYPE_COMMON, 3702 AMD_IP_BLOCK_TYPE_GMC, 3703 AMD_IP_BLOCK_TYPE_PSP, 3704 AMD_IP_BLOCK_TYPE_IH, 3705 }; 3706 3707 for (i = 0; i < adev->num_ip_blocks; i++) { 3708 int j; 3709 struct amdgpu_ip_block *block; 3710 3711 block = &adev->ip_blocks[i]; 3712 block->status.hw = false; 3713 3714 for (j = 0; j < ARRAY_SIZE(ip_order); j++) { 3715 3716 if (block->version->type != ip_order[j] || 3717 !block->status.valid) 3718 continue; 3719 3720 r = block->version->funcs->hw_init(&adev->ip_blocks[i]); 3721 if (r) { 3722 dev_err(adev->dev, "RE-INIT-early: %s failed\n", 3723 block->version->funcs->name); 3724 return r; 3725 } 3726 block->status.hw = true; 3727 } 3728 } 3729 3730 return 0; 3731 } 3732 3733 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev) 3734 { 3735 struct amdgpu_ip_block *block; 3736 int i, r = 0; 3737 3738 static enum amd_ip_block_type ip_order[] = { 3739 AMD_IP_BLOCK_TYPE_SMC, 3740 AMD_IP_BLOCK_TYPE_DCE, 3741 AMD_IP_BLOCK_TYPE_GFX, 3742 AMD_IP_BLOCK_TYPE_SDMA, 3743 AMD_IP_BLOCK_TYPE_MES, 3744 AMD_IP_BLOCK_TYPE_UVD, 3745 AMD_IP_BLOCK_TYPE_VCE, 3746 AMD_IP_BLOCK_TYPE_VCN, 3747 AMD_IP_BLOCK_TYPE_JPEG 3748 }; 3749 3750 for (i = 0; i < ARRAY_SIZE(ip_order); i++) { 3751 block = amdgpu_device_ip_get_ip_block(adev, ip_order[i]); 3752 3753 if (!block) 3754 continue; 3755 3756 if (block->status.valid && !block->status.hw) { 3757 if (block->version->type == AMD_IP_BLOCK_TYPE_SMC) { 3758 r = amdgpu_ip_block_resume(block); 3759 } else { 3760 r = block->version->funcs->hw_init(block); 3761 } 3762 3763 if (r) { 3764 dev_err(adev->dev, "RE-INIT-late: %s failed\n", 3765 block->version->funcs->name); 3766 break; 3767 } 3768 block->status.hw = true; 3769 } 3770 } 3771 3772 return r; 3773 } 3774 3775 /** 3776 * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs 3777 * 3778 * @adev: amdgpu_device pointer 3779 * 3780 * First resume function for hardware IPs. The list of all the hardware 3781 * IPs that make up the asic is walked and the resume callbacks are run for 3782 * COMMON, GMC, and IH. resume puts the hardware into a functional state 3783 * after a suspend and updates the software state as necessary. This 3784 * function is also used for restoring the GPU after a GPU reset. 3785 * Returns 0 on success, negative error code on failure. 3786 */ 3787 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev) 3788 { 3789 int i, r; 3790 3791 for (i = 0; i < adev->num_ip_blocks; i++) { 3792 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw) 3793 continue; 3794 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || 3795 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || 3796 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH || 3797 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) { 3798 3799 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]); 3800 if (r) 3801 return r; 3802 } 3803 } 3804 3805 return 0; 3806 } 3807 3808 /** 3809 * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs 3810 * 3811 * @adev: amdgpu_device pointer 3812 * 3813 * Second resume function for hardware IPs. The list of all the hardware 3814 * IPs that make up the asic is walked and the resume callbacks are run for 3815 * all blocks except COMMON, GMC, and IH. resume puts the hardware into a 3816 * functional state after a suspend and updates the software state as 3817 * necessary. This function is also used for restoring the GPU after a GPU 3818 * reset. 3819 * Returns 0 on success, negative error code on failure. 3820 */ 3821 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev) 3822 { 3823 int i, r; 3824 3825 for (i = 0; i < adev->num_ip_blocks; i++) { 3826 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw) 3827 continue; 3828 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || 3829 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || 3830 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH || 3831 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE || 3832 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) 3833 continue; 3834 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]); 3835 if (r) 3836 return r; 3837 } 3838 3839 return 0; 3840 } 3841 3842 /** 3843 * amdgpu_device_ip_resume_phase3 - run resume for hardware IPs 3844 * 3845 * @adev: amdgpu_device pointer 3846 * 3847 * Third resume function for hardware IPs. The list of all the hardware 3848 * IPs that make up the asic is walked and the resume callbacks are run for 3849 * all DCE. resume puts the hardware into a functional state after a suspend 3850 * and updates the software state as necessary. This function is also used 3851 * for restoring the GPU after a GPU reset. 3852 * 3853 * Returns 0 on success, negative error code on failure. 3854 */ 3855 static int amdgpu_device_ip_resume_phase3(struct amdgpu_device *adev) 3856 { 3857 int i, r; 3858 3859 for (i = 0; i < adev->num_ip_blocks; i++) { 3860 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw) 3861 continue; 3862 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) { 3863 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]); 3864 if (r) 3865 return r; 3866 } 3867 } 3868 3869 return 0; 3870 } 3871 3872 /** 3873 * amdgpu_device_ip_resume - run resume for hardware IPs 3874 * 3875 * @adev: amdgpu_device pointer 3876 * 3877 * Main resume function for hardware IPs. The hardware IPs 3878 * are split into two resume functions because they are 3879 * also used in recovering from a GPU reset and some additional 3880 * steps need to be take between them. In this case (S3/S4) they are 3881 * run sequentially. 3882 * Returns 0 on success, negative error code on failure. 3883 */ 3884 static int amdgpu_device_ip_resume(struct amdgpu_device *adev) 3885 { 3886 int r; 3887 3888 r = amdgpu_device_ip_resume_phase1(adev); 3889 if (r) 3890 return r; 3891 3892 r = amdgpu_device_fw_loading(adev); 3893 if (r) 3894 return r; 3895 3896 r = amdgpu_device_ip_resume_phase2(adev); 3897 3898 if (adev->mman.buffer_funcs_ring->sched.ready) 3899 amdgpu_ttm_set_buffer_funcs_status(adev, true); 3900 3901 if (r) 3902 return r; 3903 3904 amdgpu_fence_driver_hw_init(adev); 3905 3906 r = amdgpu_device_ip_resume_phase3(adev); 3907 3908 return r; 3909 } 3910 3911 /** 3912 * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV 3913 * 3914 * @adev: amdgpu_device pointer 3915 * 3916 * Query the VBIOS data tables to determine if the board supports SR-IOV. 3917 */ 3918 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev) 3919 { 3920 if (amdgpu_sriov_vf(adev)) { 3921 if (adev->is_atom_fw) { 3922 if (amdgpu_atomfirmware_gpu_virtualization_supported(adev)) 3923 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; 3924 } else { 3925 if (amdgpu_atombios_has_gpu_virtualization_table(adev)) 3926 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; 3927 } 3928 3929 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS)) 3930 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0); 3931 } 3932 } 3933 3934 /** 3935 * amdgpu_device_asic_has_dc_support - determine if DC supports the asic 3936 * 3937 * @asic_type: AMD asic type 3938 * 3939 * Check if there is DC (new modesetting infrastructre) support for an asic. 3940 * returns true if DC has support, false if not. 3941 */ 3942 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type) 3943 { 3944 switch (asic_type) { 3945 #ifdef CONFIG_DRM_AMDGPU_SI 3946 case CHIP_HAINAN: 3947 #endif 3948 case CHIP_TOPAZ: 3949 /* chips with no display hardware */ 3950 return false; 3951 #if defined(CONFIG_DRM_AMD_DC) 3952 case CHIP_TAHITI: 3953 case CHIP_PITCAIRN: 3954 case CHIP_VERDE: 3955 case CHIP_OLAND: 3956 /* 3957 * We have systems in the wild with these ASICs that require 3958 * LVDS and VGA support which is not supported with DC. 3959 * 3960 * Fallback to the non-DC driver here by default so as not to 3961 * cause regressions. 3962 */ 3963 #if defined(CONFIG_DRM_AMD_DC_SI) 3964 return amdgpu_dc > 0; 3965 #else 3966 return false; 3967 #endif 3968 case CHIP_BONAIRE: 3969 case CHIP_KAVERI: 3970 case CHIP_KABINI: 3971 case CHIP_MULLINS: 3972 /* 3973 * We have systems in the wild with these ASICs that require 3974 * VGA support which is not supported with DC. 3975 * 3976 * Fallback to the non-DC driver here by default so as not to 3977 * cause regressions. 3978 */ 3979 return amdgpu_dc > 0; 3980 default: 3981 return amdgpu_dc != 0; 3982 #else 3983 default: 3984 if (amdgpu_dc > 0) 3985 DRM_INFO_ONCE("Display Core has been requested via kernel parameter but isn't supported by ASIC, ignoring\n"); 3986 return false; 3987 #endif 3988 } 3989 } 3990 3991 /** 3992 * amdgpu_device_has_dc_support - check if dc is supported 3993 * 3994 * @adev: amdgpu_device pointer 3995 * 3996 * Returns true for supported, false for not supported 3997 */ 3998 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev) 3999 { 4000 if (adev->enable_virtual_display || 4001 (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK)) 4002 return false; 4003 4004 return amdgpu_device_asic_has_dc_support(adev->asic_type); 4005 } 4006 4007 static void amdgpu_device_xgmi_reset_func(struct work_struct *__work) 4008 { 4009 struct amdgpu_device *adev = 4010 container_of(__work, struct amdgpu_device, xgmi_reset_work); 4011 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 4012 4013 /* It's a bug to not have a hive within this function */ 4014 if (WARN_ON(!hive)) 4015 return; 4016 4017 /* 4018 * Use task barrier to synchronize all xgmi reset works across the 4019 * hive. task_barrier_enter and task_barrier_exit will block 4020 * until all the threads running the xgmi reset works reach 4021 * those points. task_barrier_full will do both blocks. 4022 */ 4023 if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) { 4024 4025 task_barrier_enter(&hive->tb); 4026 adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev)); 4027 4028 if (adev->asic_reset_res) 4029 goto fail; 4030 4031 task_barrier_exit(&hive->tb); 4032 adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev)); 4033 4034 if (adev->asic_reset_res) 4035 goto fail; 4036 4037 amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__MMHUB); 4038 } else { 4039 4040 task_barrier_full(&hive->tb); 4041 adev->asic_reset_res = amdgpu_asic_reset(adev); 4042 } 4043 4044 fail: 4045 if (adev->asic_reset_res) 4046 DRM_WARN("ASIC reset failed with error, %d for drm dev, %s", 4047 adev->asic_reset_res, adev_to_drm(adev)->unique); 4048 amdgpu_put_xgmi_hive(hive); 4049 } 4050 4051 static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev) 4052 { 4053 char *input = amdgpu_lockup_timeout; 4054 char *timeout_setting = NULL; 4055 int index = 0; 4056 long timeout; 4057 int ret = 0; 4058 4059 /* 4060 * By default timeout for non compute jobs is 10000 4061 * and 60000 for compute jobs. 4062 * In SR-IOV or passthrough mode, timeout for compute 4063 * jobs are 60000 by default. 4064 */ 4065 adev->gfx_timeout = msecs_to_jiffies(10000); 4066 adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout; 4067 if (amdgpu_sriov_vf(adev)) 4068 adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ? 4069 msecs_to_jiffies(60000) : msecs_to_jiffies(10000); 4070 else 4071 adev->compute_timeout = msecs_to_jiffies(60000); 4072 4073 if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) { 4074 while ((timeout_setting = strsep(&input, ",")) && 4075 strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) { 4076 ret = kstrtol(timeout_setting, 0, &timeout); 4077 if (ret) 4078 return ret; 4079 4080 if (timeout == 0) { 4081 index++; 4082 continue; 4083 } else if (timeout < 0) { 4084 timeout = MAX_SCHEDULE_TIMEOUT; 4085 dev_warn(adev->dev, "lockup timeout disabled"); 4086 add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK); 4087 } else { 4088 timeout = msecs_to_jiffies(timeout); 4089 } 4090 4091 switch (index++) { 4092 case 0: 4093 adev->gfx_timeout = timeout; 4094 break; 4095 case 1: 4096 adev->compute_timeout = timeout; 4097 break; 4098 case 2: 4099 adev->sdma_timeout = timeout; 4100 break; 4101 case 3: 4102 adev->video_timeout = timeout; 4103 break; 4104 default: 4105 break; 4106 } 4107 } 4108 /* 4109 * There is only one value specified and 4110 * it should apply to all non-compute jobs. 4111 */ 4112 if (index == 1) { 4113 adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout; 4114 if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev)) 4115 adev->compute_timeout = adev->gfx_timeout; 4116 } 4117 } 4118 4119 return ret; 4120 } 4121 4122 /** 4123 * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU 4124 * 4125 * @adev: amdgpu_device pointer 4126 * 4127 * RAM direct mapped to GPU if IOMMU is not enabled or is pass through mode 4128 */ 4129 static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev) 4130 { 4131 struct iommu_domain *domain; 4132 4133 domain = iommu_get_domain_for_dev(adev->dev); 4134 if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY) 4135 adev->ram_is_direct_mapped = true; 4136 } 4137 4138 #if defined(CONFIG_HSA_AMD_P2P) 4139 /** 4140 * amdgpu_device_check_iommu_remap - Check if DMA remapping is enabled. 4141 * 4142 * @adev: amdgpu_device pointer 4143 * 4144 * return if IOMMU remapping bar address 4145 */ 4146 static bool amdgpu_device_check_iommu_remap(struct amdgpu_device *adev) 4147 { 4148 struct iommu_domain *domain; 4149 4150 domain = iommu_get_domain_for_dev(adev->dev); 4151 if (domain && (domain->type == IOMMU_DOMAIN_DMA || 4152 domain->type == IOMMU_DOMAIN_DMA_FQ)) 4153 return true; 4154 4155 return false; 4156 } 4157 #endif 4158 4159 static const struct attribute *amdgpu_dev_attributes[] = { 4160 &dev_attr_pcie_replay_count.attr, 4161 NULL 4162 }; 4163 4164 static void amdgpu_device_set_mcbp(struct amdgpu_device *adev) 4165 { 4166 if (amdgpu_mcbp == 1) 4167 adev->gfx.mcbp = true; 4168 else if (amdgpu_mcbp == 0) 4169 adev->gfx.mcbp = false; 4170 4171 if (amdgpu_sriov_vf(adev)) 4172 adev->gfx.mcbp = true; 4173 4174 if (adev->gfx.mcbp) 4175 DRM_INFO("MCBP is enabled\n"); 4176 } 4177 4178 /** 4179 * amdgpu_device_init - initialize the driver 4180 * 4181 * @adev: amdgpu_device pointer 4182 * @flags: driver flags 4183 * 4184 * Initializes the driver info and hw (all asics). 4185 * Returns 0 for success or an error on failure. 4186 * Called at driver startup. 4187 */ 4188 int amdgpu_device_init(struct amdgpu_device *adev, 4189 uint32_t flags) 4190 { 4191 struct drm_device *ddev = adev_to_drm(adev); 4192 struct pci_dev *pdev = adev->pdev; 4193 int r, i; 4194 bool px = false; 4195 u32 max_MBps; 4196 int tmp; 4197 4198 adev->shutdown = false; 4199 adev->flags = flags; 4200 4201 if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST) 4202 adev->asic_type = amdgpu_force_asic_type; 4203 else 4204 adev->asic_type = flags & AMD_ASIC_MASK; 4205 4206 adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT; 4207 if (amdgpu_emu_mode == 1) 4208 adev->usec_timeout *= 10; 4209 adev->gmc.gart_size = 512 * 1024 * 1024; 4210 adev->accel_working = false; 4211 adev->num_rings = 0; 4212 RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub()); 4213 adev->mman.buffer_funcs = NULL; 4214 adev->mman.buffer_funcs_ring = NULL; 4215 adev->vm_manager.vm_pte_funcs = NULL; 4216 adev->vm_manager.vm_pte_num_scheds = 0; 4217 adev->gmc.gmc_funcs = NULL; 4218 adev->harvest_ip_mask = 0x0; 4219 adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS); 4220 bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 4221 4222 adev->smc_rreg = &amdgpu_invalid_rreg; 4223 adev->smc_wreg = &amdgpu_invalid_wreg; 4224 adev->pcie_rreg = &amdgpu_invalid_rreg; 4225 adev->pcie_wreg = &amdgpu_invalid_wreg; 4226 adev->pcie_rreg_ext = &amdgpu_invalid_rreg_ext; 4227 adev->pcie_wreg_ext = &amdgpu_invalid_wreg_ext; 4228 adev->pciep_rreg = &amdgpu_invalid_rreg; 4229 adev->pciep_wreg = &amdgpu_invalid_wreg; 4230 adev->pcie_rreg64 = &amdgpu_invalid_rreg64; 4231 adev->pcie_wreg64 = &amdgpu_invalid_wreg64; 4232 adev->pcie_rreg64_ext = &amdgpu_invalid_rreg64_ext; 4233 adev->pcie_wreg64_ext = &amdgpu_invalid_wreg64_ext; 4234 adev->uvd_ctx_rreg = &amdgpu_invalid_rreg; 4235 adev->uvd_ctx_wreg = &amdgpu_invalid_wreg; 4236 adev->didt_rreg = &amdgpu_invalid_rreg; 4237 adev->didt_wreg = &amdgpu_invalid_wreg; 4238 adev->gc_cac_rreg = &amdgpu_invalid_rreg; 4239 adev->gc_cac_wreg = &amdgpu_invalid_wreg; 4240 adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg; 4241 adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg; 4242 4243 DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n", 4244 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device, 4245 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision); 4246 4247 /* mutex initialization are all done here so we 4248 * can recall function without having locking issues 4249 */ 4250 mutex_init(&adev->firmware.mutex); 4251 mutex_init(&adev->pm.mutex); 4252 mutex_init(&adev->gfx.gpu_clock_mutex); 4253 mutex_init(&adev->srbm_mutex); 4254 mutex_init(&adev->gfx.pipe_reserve_mutex); 4255 mutex_init(&adev->gfx.gfx_off_mutex); 4256 mutex_init(&adev->gfx.partition_mutex); 4257 mutex_init(&adev->grbm_idx_mutex); 4258 mutex_init(&adev->mn_lock); 4259 mutex_init(&adev->virt.vf_errors.lock); 4260 mutex_init(&adev->virt.rlcg_reg_lock); 4261 hash_init(adev->mn_hash); 4262 mutex_init(&adev->psp.mutex); 4263 mutex_init(&adev->notifier_lock); 4264 mutex_init(&adev->pm.stable_pstate_ctx_lock); 4265 mutex_init(&adev->benchmark_mutex); 4266 mutex_init(&adev->gfx.reset_sem_mutex); 4267 /* Initialize the mutex for cleaner shader isolation between GFX and compute processes */ 4268 mutex_init(&adev->enforce_isolation_mutex); 4269 mutex_init(&adev->gfx.kfd_sch_mutex); 4270 4271 amdgpu_device_init_apu_flags(adev); 4272 4273 r = amdgpu_device_check_arguments(adev); 4274 if (r) 4275 return r; 4276 4277 spin_lock_init(&adev->mmio_idx_lock); 4278 spin_lock_init(&adev->smc_idx_lock); 4279 spin_lock_init(&adev->pcie_idx_lock); 4280 spin_lock_init(&adev->uvd_ctx_idx_lock); 4281 spin_lock_init(&adev->didt_idx_lock); 4282 spin_lock_init(&adev->gc_cac_idx_lock); 4283 spin_lock_init(&adev->se_cac_idx_lock); 4284 spin_lock_init(&adev->audio_endpt_idx_lock); 4285 spin_lock_init(&adev->mm_stats.lock); 4286 spin_lock_init(&adev->wb.lock); 4287 4288 INIT_LIST_HEAD(&adev->reset_list); 4289 4290 INIT_LIST_HEAD(&adev->ras_list); 4291 4292 INIT_LIST_HEAD(&adev->pm.od_kobj_list); 4293 4294 INIT_DELAYED_WORK(&adev->delayed_init_work, 4295 amdgpu_device_delayed_init_work_handler); 4296 INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work, 4297 amdgpu_device_delay_enable_gfx_off); 4298 /* 4299 * Initialize the enforce_isolation work structures for each XCP 4300 * partition. This work handler is responsible for enforcing shader 4301 * isolation on AMD GPUs. It counts the number of emitted fences for 4302 * each GFX and compute ring. If there are any fences, it schedules 4303 * the `enforce_isolation_work` to be run after a delay. If there are 4304 * no fences, it signals the Kernel Fusion Driver (KFD) to resume the 4305 * runqueue. 4306 */ 4307 for (i = 0; i < MAX_XCP; i++) { 4308 INIT_DELAYED_WORK(&adev->gfx.enforce_isolation[i].work, 4309 amdgpu_gfx_enforce_isolation_handler); 4310 adev->gfx.enforce_isolation[i].adev = adev; 4311 adev->gfx.enforce_isolation[i].xcp_id = i; 4312 } 4313 4314 INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func); 4315 4316 adev->gfx.gfx_off_req_count = 1; 4317 adev->gfx.gfx_off_residency = 0; 4318 adev->gfx.gfx_off_entrycount = 0; 4319 adev->pm.ac_power = power_supply_is_system_supplied() > 0; 4320 4321 atomic_set(&adev->throttling_logging_enabled, 1); 4322 /* 4323 * If throttling continues, logging will be performed every minute 4324 * to avoid log flooding. "-1" is subtracted since the thermal 4325 * throttling interrupt comes every second. Thus, the total logging 4326 * interval is 59 seconds(retelimited printk interval) + 1(waiting 4327 * for throttling interrupt) = 60 seconds. 4328 */ 4329 ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1); 4330 ratelimit_state_init(&adev->virt.ras_telemetry_rs, 5 * HZ, 1); 4331 4332 ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE); 4333 ratelimit_set_flags(&adev->virt.ras_telemetry_rs, RATELIMIT_MSG_ON_RELEASE); 4334 4335 /* Registers mapping */ 4336 /* TODO: block userspace mapping of io register */ 4337 if (adev->asic_type >= CHIP_BONAIRE) { 4338 adev->rmmio_base = pci_resource_start(adev->pdev, 5); 4339 adev->rmmio_size = pci_resource_len(adev->pdev, 5); 4340 } else { 4341 adev->rmmio_base = pci_resource_start(adev->pdev, 2); 4342 adev->rmmio_size = pci_resource_len(adev->pdev, 2); 4343 } 4344 4345 for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++) 4346 atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN); 4347 4348 adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size); 4349 if (!adev->rmmio) 4350 return -ENOMEM; 4351 4352 DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base); 4353 DRM_INFO("register mmio size: %u\n", (unsigned int)adev->rmmio_size); 4354 4355 /* 4356 * Reset domain needs to be present early, before XGMI hive discovered 4357 * (if any) and initialized to use reset sem and in_gpu reset flag 4358 * early on during init and before calling to RREG32. 4359 */ 4360 adev->reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "amdgpu-reset-dev"); 4361 if (!adev->reset_domain) 4362 return -ENOMEM; 4363 4364 /* detect hw virtualization here */ 4365 amdgpu_detect_virtualization(adev); 4366 4367 amdgpu_device_get_pcie_info(adev); 4368 4369 r = amdgpu_device_get_job_timeout_settings(adev); 4370 if (r) { 4371 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n"); 4372 return r; 4373 } 4374 4375 amdgpu_device_set_mcbp(adev); 4376 4377 /* 4378 * By default, use default mode where all blocks are expected to be 4379 * initialized. At present a 'swinit' of blocks is required to be 4380 * completed before the need for a different level is detected. 4381 */ 4382 amdgpu_set_init_level(adev, AMDGPU_INIT_LEVEL_DEFAULT); 4383 /* early init functions */ 4384 r = amdgpu_device_ip_early_init(adev); 4385 if (r) 4386 return r; 4387 4388 /* Get rid of things like offb */ 4389 r = aperture_remove_conflicting_pci_devices(adev->pdev, amdgpu_kms_driver.name); 4390 if (r) 4391 return r; 4392 4393 /* Enable TMZ based on IP_VERSION */ 4394 amdgpu_gmc_tmz_set(adev); 4395 4396 if (amdgpu_sriov_vf(adev) && 4397 amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 3, 0)) 4398 /* VF MMIO access (except mailbox range) from CPU 4399 * will be blocked during sriov runtime 4400 */ 4401 adev->virt.caps |= AMDGPU_VF_MMIO_ACCESS_PROTECT; 4402 4403 amdgpu_gmc_noretry_set(adev); 4404 /* Need to get xgmi info early to decide the reset behavior*/ 4405 if (adev->gmc.xgmi.supported) { 4406 r = adev->gfxhub.funcs->get_xgmi_info(adev); 4407 if (r) 4408 return r; 4409 } 4410 4411 /* enable PCIE atomic ops */ 4412 if (amdgpu_sriov_vf(adev)) { 4413 if (adev->virt.fw_reserve.p_pf2vf) 4414 adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *) 4415 adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags == 4416 (PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64); 4417 /* APUs w/ gfx9 onwards doesn't reply on PCIe atomics, rather it is a 4418 * internal path natively support atomics, set have_atomics_support to true. 4419 */ 4420 } else if ((adev->flags & AMD_IS_APU) && 4421 (amdgpu_ip_version(adev, GC_HWIP, 0) > 4422 IP_VERSION(9, 0, 0))) { 4423 adev->have_atomics_support = true; 4424 } else { 4425 adev->have_atomics_support = 4426 !pci_enable_atomic_ops_to_root(adev->pdev, 4427 PCI_EXP_DEVCAP2_ATOMIC_COMP32 | 4428 PCI_EXP_DEVCAP2_ATOMIC_COMP64); 4429 } 4430 4431 if (!adev->have_atomics_support) 4432 dev_info(adev->dev, "PCIE atomic ops is not supported\n"); 4433 4434 /* doorbell bar mapping and doorbell index init*/ 4435 amdgpu_doorbell_init(adev); 4436 4437 if (amdgpu_emu_mode == 1) { 4438 /* post the asic on emulation mode */ 4439 emu_soc_asic_init(adev); 4440 goto fence_driver_init; 4441 } 4442 4443 amdgpu_reset_init(adev); 4444 4445 /* detect if we are with an SRIOV vbios */ 4446 if (adev->bios) 4447 amdgpu_device_detect_sriov_bios(adev); 4448 4449 /* check if we need to reset the asic 4450 * E.g., driver was not cleanly unloaded previously, etc. 4451 */ 4452 if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) { 4453 if (adev->gmc.xgmi.num_physical_nodes) { 4454 dev_info(adev->dev, "Pending hive reset.\n"); 4455 amdgpu_set_init_level(adev, 4456 AMDGPU_INIT_LEVEL_MINIMAL_XGMI); 4457 } else if (amdgpu_ip_version(adev, MP1_HWIP, 0) == IP_VERSION(13, 0, 10) && 4458 !amdgpu_device_has_display_hardware(adev)) { 4459 r = psp_gpu_reset(adev); 4460 } else { 4461 tmp = amdgpu_reset_method; 4462 /* It should do a default reset when loading or reloading the driver, 4463 * regardless of the module parameter reset_method. 4464 */ 4465 amdgpu_reset_method = AMD_RESET_METHOD_NONE; 4466 r = amdgpu_asic_reset(adev); 4467 amdgpu_reset_method = tmp; 4468 } 4469 4470 if (r) { 4471 dev_err(adev->dev, "asic reset on init failed\n"); 4472 goto failed; 4473 } 4474 } 4475 4476 /* Post card if necessary */ 4477 if (amdgpu_device_need_post(adev)) { 4478 if (!adev->bios) { 4479 dev_err(adev->dev, "no vBIOS found\n"); 4480 r = -EINVAL; 4481 goto failed; 4482 } 4483 DRM_INFO("GPU posting now...\n"); 4484 r = amdgpu_device_asic_init(adev); 4485 if (r) { 4486 dev_err(adev->dev, "gpu post error!\n"); 4487 goto failed; 4488 } 4489 } 4490 4491 if (adev->bios) { 4492 if (adev->is_atom_fw) { 4493 /* Initialize clocks */ 4494 r = amdgpu_atomfirmware_get_clock_info(adev); 4495 if (r) { 4496 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n"); 4497 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0); 4498 goto failed; 4499 } 4500 } else { 4501 /* Initialize clocks */ 4502 r = amdgpu_atombios_get_clock_info(adev); 4503 if (r) { 4504 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n"); 4505 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0); 4506 goto failed; 4507 } 4508 /* init i2c buses */ 4509 amdgpu_i2c_init(adev); 4510 } 4511 } 4512 4513 fence_driver_init: 4514 /* Fence driver */ 4515 r = amdgpu_fence_driver_sw_init(adev); 4516 if (r) { 4517 dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n"); 4518 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0); 4519 goto failed; 4520 } 4521 4522 /* init the mode config */ 4523 drm_mode_config_init(adev_to_drm(adev)); 4524 4525 r = amdgpu_device_ip_init(adev); 4526 if (r) { 4527 dev_err(adev->dev, "amdgpu_device_ip_init failed\n"); 4528 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0); 4529 goto release_ras_con; 4530 } 4531 4532 amdgpu_fence_driver_hw_init(adev); 4533 4534 dev_info(adev->dev, 4535 "SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n", 4536 adev->gfx.config.max_shader_engines, 4537 adev->gfx.config.max_sh_per_se, 4538 adev->gfx.config.max_cu_per_sh, 4539 adev->gfx.cu_info.number); 4540 4541 adev->accel_working = true; 4542 4543 amdgpu_vm_check_compute_bug(adev); 4544 4545 /* Initialize the buffer migration limit. */ 4546 if (amdgpu_moverate >= 0) 4547 max_MBps = amdgpu_moverate; 4548 else 4549 max_MBps = 8; /* Allow 8 MB/s. */ 4550 /* Get a log2 for easy divisions. */ 4551 adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps)); 4552 4553 /* 4554 * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost. 4555 * Otherwise the mgpu fan boost feature will be skipped due to the 4556 * gpu instance is counted less. 4557 */ 4558 amdgpu_register_gpu_instance(adev); 4559 4560 /* enable clockgating, etc. after ib tests, etc. since some blocks require 4561 * explicit gating rather than handling it automatically. 4562 */ 4563 if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) { 4564 r = amdgpu_device_ip_late_init(adev); 4565 if (r) { 4566 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n"); 4567 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r); 4568 goto release_ras_con; 4569 } 4570 /* must succeed. */ 4571 amdgpu_ras_resume(adev); 4572 queue_delayed_work(system_wq, &adev->delayed_init_work, 4573 msecs_to_jiffies(AMDGPU_RESUME_MS)); 4574 } 4575 4576 if (amdgpu_sriov_vf(adev)) { 4577 amdgpu_virt_release_full_gpu(adev, true); 4578 flush_delayed_work(&adev->delayed_init_work); 4579 } 4580 4581 /* 4582 * Place those sysfs registering after `late_init`. As some of those 4583 * operations performed in `late_init` might affect the sysfs 4584 * interfaces creating. 4585 */ 4586 r = amdgpu_atombios_sysfs_init(adev); 4587 if (r) 4588 drm_err(&adev->ddev, 4589 "registering atombios sysfs failed (%d).\n", r); 4590 4591 r = amdgpu_pm_sysfs_init(adev); 4592 if (r) 4593 DRM_ERROR("registering pm sysfs failed (%d).\n", r); 4594 4595 r = amdgpu_ucode_sysfs_init(adev); 4596 if (r) { 4597 adev->ucode_sysfs_en = false; 4598 DRM_ERROR("Creating firmware sysfs failed (%d).\n", r); 4599 } else 4600 adev->ucode_sysfs_en = true; 4601 4602 r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes); 4603 if (r) 4604 dev_err(adev->dev, "Could not create amdgpu device attr\n"); 4605 4606 r = devm_device_add_group(adev->dev, &amdgpu_board_attrs_group); 4607 if (r) 4608 dev_err(adev->dev, 4609 "Could not create amdgpu board attributes\n"); 4610 4611 amdgpu_fru_sysfs_init(adev); 4612 amdgpu_reg_state_sysfs_init(adev); 4613 amdgpu_xcp_cfg_sysfs_init(adev); 4614 4615 if (IS_ENABLED(CONFIG_PERF_EVENTS)) 4616 r = amdgpu_pmu_init(adev); 4617 if (r) 4618 dev_err(adev->dev, "amdgpu_pmu_init failed\n"); 4619 4620 /* Have stored pci confspace at hand for restore in sudden PCI error */ 4621 if (amdgpu_device_cache_pci_state(adev->pdev)) 4622 pci_restore_state(pdev); 4623 4624 /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */ 4625 /* this will fail for cards that aren't VGA class devices, just 4626 * ignore it 4627 */ 4628 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) 4629 vga_client_register(adev->pdev, amdgpu_device_vga_set_decode); 4630 4631 px = amdgpu_device_supports_px(ddev); 4632 4633 if (px || (!dev_is_removable(&adev->pdev->dev) && 4634 apple_gmux_detect(NULL, NULL))) 4635 vga_switcheroo_register_client(adev->pdev, 4636 &amdgpu_switcheroo_ops, px); 4637 4638 if (px) 4639 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain); 4640 4641 if (adev->init_lvl->level == AMDGPU_INIT_LEVEL_MINIMAL_XGMI) 4642 amdgpu_xgmi_reset_on_init(adev); 4643 4644 amdgpu_device_check_iommu_direct_map(adev); 4645 4646 adev->pm_nb.notifier_call = amdgpu_device_pm_notifier; 4647 r = register_pm_notifier(&adev->pm_nb); 4648 if (r) 4649 goto failed; 4650 4651 return 0; 4652 4653 release_ras_con: 4654 if (amdgpu_sriov_vf(adev)) 4655 amdgpu_virt_release_full_gpu(adev, true); 4656 4657 /* failed in exclusive mode due to timeout */ 4658 if (amdgpu_sriov_vf(adev) && 4659 !amdgpu_sriov_runtime(adev) && 4660 amdgpu_virt_mmio_blocked(adev) && 4661 !amdgpu_virt_wait_reset(adev)) { 4662 dev_err(adev->dev, "VF exclusive mode timeout\n"); 4663 /* Don't send request since VF is inactive. */ 4664 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; 4665 adev->virt.ops = NULL; 4666 r = -EAGAIN; 4667 } 4668 amdgpu_release_ras_context(adev); 4669 4670 failed: 4671 amdgpu_vf_error_trans_all(adev); 4672 4673 return r; 4674 } 4675 4676 static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev) 4677 { 4678 4679 /* Clear all CPU mappings pointing to this device */ 4680 unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1); 4681 4682 /* Unmap all mapped bars - Doorbell, registers and VRAM */ 4683 amdgpu_doorbell_fini(adev); 4684 4685 iounmap(adev->rmmio); 4686 adev->rmmio = NULL; 4687 if (adev->mman.aper_base_kaddr) 4688 iounmap(adev->mman.aper_base_kaddr); 4689 adev->mman.aper_base_kaddr = NULL; 4690 4691 /* Memory manager related */ 4692 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { 4693 arch_phys_wc_del(adev->gmc.vram_mtrr); 4694 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); 4695 } 4696 } 4697 4698 /** 4699 * amdgpu_device_fini_hw - tear down the driver 4700 * 4701 * @adev: amdgpu_device pointer 4702 * 4703 * Tear down the driver info (all asics). 4704 * Called at driver shutdown. 4705 */ 4706 void amdgpu_device_fini_hw(struct amdgpu_device *adev) 4707 { 4708 dev_info(adev->dev, "amdgpu: finishing device.\n"); 4709 flush_delayed_work(&adev->delayed_init_work); 4710 4711 if (adev->mman.initialized) 4712 drain_workqueue(adev->mman.bdev.wq); 4713 adev->shutdown = true; 4714 4715 unregister_pm_notifier(&adev->pm_nb); 4716 4717 /* make sure IB test finished before entering exclusive mode 4718 * to avoid preemption on IB test 4719 */ 4720 if (amdgpu_sriov_vf(adev)) { 4721 amdgpu_virt_request_full_gpu(adev, false); 4722 amdgpu_virt_fini_data_exchange(adev); 4723 } 4724 4725 /* disable all interrupts */ 4726 amdgpu_irq_disable_all(adev); 4727 if (adev->mode_info.mode_config_initialized) { 4728 if (!drm_drv_uses_atomic_modeset(adev_to_drm(adev))) 4729 drm_helper_force_disable_all(adev_to_drm(adev)); 4730 else 4731 drm_atomic_helper_shutdown(adev_to_drm(adev)); 4732 } 4733 amdgpu_fence_driver_hw_fini(adev); 4734 4735 if (adev->pm.sysfs_initialized) 4736 amdgpu_pm_sysfs_fini(adev); 4737 if (adev->ucode_sysfs_en) 4738 amdgpu_ucode_sysfs_fini(adev); 4739 sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes); 4740 amdgpu_fru_sysfs_fini(adev); 4741 4742 amdgpu_reg_state_sysfs_fini(adev); 4743 amdgpu_xcp_cfg_sysfs_fini(adev); 4744 4745 /* disable ras feature must before hw fini */ 4746 amdgpu_ras_pre_fini(adev); 4747 4748 amdgpu_ttm_set_buffer_funcs_status(adev, false); 4749 4750 amdgpu_device_ip_fini_early(adev); 4751 4752 amdgpu_irq_fini_hw(adev); 4753 4754 if (adev->mman.initialized) 4755 ttm_device_clear_dma_mappings(&adev->mman.bdev); 4756 4757 amdgpu_gart_dummy_page_fini(adev); 4758 4759 if (drm_dev_is_unplugged(adev_to_drm(adev))) 4760 amdgpu_device_unmap_mmio(adev); 4761 4762 } 4763 4764 void amdgpu_device_fini_sw(struct amdgpu_device *adev) 4765 { 4766 int idx; 4767 bool px; 4768 4769 amdgpu_device_ip_fini(adev); 4770 amdgpu_fence_driver_sw_fini(adev); 4771 amdgpu_ucode_release(&adev->firmware.gpu_info_fw); 4772 adev->accel_working = false; 4773 dma_fence_put(rcu_dereference_protected(adev->gang_submit, true)); 4774 4775 amdgpu_reset_fini(adev); 4776 4777 /* free i2c buses */ 4778 amdgpu_i2c_fini(adev); 4779 4780 if (adev->bios) { 4781 if (amdgpu_emu_mode != 1) 4782 amdgpu_atombios_fini(adev); 4783 amdgpu_bios_release(adev); 4784 } 4785 4786 kfree(adev->fru_info); 4787 adev->fru_info = NULL; 4788 4789 px = amdgpu_device_supports_px(adev_to_drm(adev)); 4790 4791 if (px || (!dev_is_removable(&adev->pdev->dev) && 4792 apple_gmux_detect(NULL, NULL))) 4793 vga_switcheroo_unregister_client(adev->pdev); 4794 4795 if (px) 4796 vga_switcheroo_fini_domain_pm_ops(adev->dev); 4797 4798 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) 4799 vga_client_unregister(adev->pdev); 4800 4801 if (drm_dev_enter(adev_to_drm(adev), &idx)) { 4802 4803 iounmap(adev->rmmio); 4804 adev->rmmio = NULL; 4805 amdgpu_doorbell_fini(adev); 4806 drm_dev_exit(idx); 4807 } 4808 4809 if (IS_ENABLED(CONFIG_PERF_EVENTS)) 4810 amdgpu_pmu_fini(adev); 4811 if (adev->mman.discovery_bin) 4812 amdgpu_discovery_fini(adev); 4813 4814 amdgpu_reset_put_reset_domain(adev->reset_domain); 4815 adev->reset_domain = NULL; 4816 4817 kfree(adev->pci_state); 4818 4819 } 4820 4821 /** 4822 * amdgpu_device_evict_resources - evict device resources 4823 * @adev: amdgpu device object 4824 * 4825 * Evicts all ttm device resources(vram BOs, gart table) from the lru list 4826 * of the vram memory type. Mainly used for evicting device resources 4827 * at suspend time. 4828 * 4829 */ 4830 static int amdgpu_device_evict_resources(struct amdgpu_device *adev) 4831 { 4832 int ret; 4833 4834 /* No need to evict vram on APUs unless going to S4 */ 4835 if (!adev->in_s4 && (adev->flags & AMD_IS_APU)) 4836 return 0; 4837 4838 ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM); 4839 if (ret) 4840 DRM_WARN("evicting device resources failed\n"); 4841 return ret; 4842 } 4843 4844 /* 4845 * Suspend & resume. 4846 */ 4847 /** 4848 * amdgpu_device_pm_notifier - Notification block for Suspend/Hibernate events 4849 * @nb: notifier block 4850 * @mode: suspend mode 4851 * @data: data 4852 * 4853 * This function is called when the system is about to suspend or hibernate. 4854 * It is used to evict resources from the device before the system goes to 4855 * sleep while there is still access to swap. 4856 */ 4857 static int amdgpu_device_pm_notifier(struct notifier_block *nb, unsigned long mode, 4858 void *data) 4859 { 4860 struct amdgpu_device *adev = container_of(nb, struct amdgpu_device, pm_nb); 4861 int r; 4862 4863 switch (mode) { 4864 case PM_HIBERNATION_PREPARE: 4865 adev->in_s4 = true; 4866 fallthrough; 4867 case PM_SUSPEND_PREPARE: 4868 r = amdgpu_device_evict_resources(adev); 4869 /* 4870 * This is considered non-fatal at this time because 4871 * amdgpu_device_prepare() will also fatally evict resources. 4872 * See https://gitlab.freedesktop.org/drm/amd/-/issues/3781 4873 */ 4874 if (r) 4875 drm_warn(adev_to_drm(adev), "Failed to evict resources, freeze active processes if problems occur: %d\n", r); 4876 break; 4877 } 4878 4879 return NOTIFY_DONE; 4880 } 4881 4882 /** 4883 * amdgpu_device_prepare - prepare for device suspend 4884 * 4885 * @dev: drm dev pointer 4886 * 4887 * Prepare to put the hw in the suspend state (all asics). 4888 * Returns 0 for success or an error on failure. 4889 * Called at driver suspend. 4890 */ 4891 int amdgpu_device_prepare(struct drm_device *dev) 4892 { 4893 struct amdgpu_device *adev = drm_to_adev(dev); 4894 int i, r; 4895 4896 amdgpu_choose_low_power_state(adev); 4897 4898 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 4899 return 0; 4900 4901 /* Evict the majority of BOs before starting suspend sequence */ 4902 r = amdgpu_device_evict_resources(adev); 4903 if (r) 4904 goto unprepare; 4905 4906 flush_delayed_work(&adev->gfx.gfx_off_delay_work); 4907 4908 for (i = 0; i < adev->num_ip_blocks; i++) { 4909 if (!adev->ip_blocks[i].status.valid) 4910 continue; 4911 if (!adev->ip_blocks[i].version->funcs->prepare_suspend) 4912 continue; 4913 r = adev->ip_blocks[i].version->funcs->prepare_suspend(&adev->ip_blocks[i]); 4914 if (r) 4915 goto unprepare; 4916 } 4917 4918 return 0; 4919 4920 unprepare: 4921 adev->in_s0ix = adev->in_s3 = adev->in_s4 = false; 4922 4923 return r; 4924 } 4925 4926 /** 4927 * amdgpu_device_suspend - initiate device suspend 4928 * 4929 * @dev: drm dev pointer 4930 * @notify_clients: notify in-kernel DRM clients 4931 * 4932 * Puts the hw in the suspend state (all asics). 4933 * Returns 0 for success or an error on failure. 4934 * Called at driver suspend. 4935 */ 4936 int amdgpu_device_suspend(struct drm_device *dev, bool notify_clients) 4937 { 4938 struct amdgpu_device *adev = drm_to_adev(dev); 4939 int r = 0; 4940 4941 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 4942 return 0; 4943 4944 adev->in_suspend = true; 4945 4946 if (amdgpu_sriov_vf(adev)) { 4947 amdgpu_virt_fini_data_exchange(adev); 4948 r = amdgpu_virt_request_full_gpu(adev, false); 4949 if (r) 4950 return r; 4951 } 4952 4953 if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D3)) 4954 DRM_WARN("smart shift update failed\n"); 4955 4956 if (notify_clients) 4957 drm_client_dev_suspend(adev_to_drm(adev), false); 4958 4959 cancel_delayed_work_sync(&adev->delayed_init_work); 4960 4961 amdgpu_ras_suspend(adev); 4962 4963 amdgpu_device_ip_suspend_phase1(adev); 4964 4965 if (!adev->in_s0ix) 4966 amdgpu_amdkfd_suspend(adev, adev->in_runpm); 4967 4968 r = amdgpu_device_evict_resources(adev); 4969 if (r) 4970 return r; 4971 4972 amdgpu_ttm_set_buffer_funcs_status(adev, false); 4973 4974 amdgpu_fence_driver_hw_fini(adev); 4975 4976 amdgpu_device_ip_suspend_phase2(adev); 4977 4978 if (amdgpu_sriov_vf(adev)) 4979 amdgpu_virt_release_full_gpu(adev, false); 4980 4981 r = amdgpu_dpm_notify_rlc_state(adev, false); 4982 if (r) 4983 return r; 4984 4985 return 0; 4986 } 4987 4988 /** 4989 * amdgpu_device_resume - initiate device resume 4990 * 4991 * @dev: drm dev pointer 4992 * @notify_clients: notify in-kernel DRM clients 4993 * 4994 * Bring the hw back to operating state (all asics). 4995 * Returns 0 for success or an error on failure. 4996 * Called at driver resume. 4997 */ 4998 int amdgpu_device_resume(struct drm_device *dev, bool notify_clients) 4999 { 5000 struct amdgpu_device *adev = drm_to_adev(dev); 5001 int r = 0; 5002 5003 if (amdgpu_sriov_vf(adev)) { 5004 r = amdgpu_virt_request_full_gpu(adev, true); 5005 if (r) 5006 return r; 5007 } 5008 5009 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 5010 return 0; 5011 5012 if (adev->in_s0ix) 5013 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry); 5014 5015 /* post card */ 5016 if (amdgpu_device_need_post(adev)) { 5017 r = amdgpu_device_asic_init(adev); 5018 if (r) 5019 dev_err(adev->dev, "amdgpu asic init failed\n"); 5020 } 5021 5022 r = amdgpu_device_ip_resume(adev); 5023 5024 if (r) { 5025 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r); 5026 goto exit; 5027 } 5028 5029 if (!adev->in_s0ix) { 5030 r = amdgpu_amdkfd_resume(adev, adev->in_runpm); 5031 if (r) 5032 goto exit; 5033 } 5034 5035 r = amdgpu_device_ip_late_init(adev); 5036 if (r) 5037 goto exit; 5038 5039 queue_delayed_work(system_wq, &adev->delayed_init_work, 5040 msecs_to_jiffies(AMDGPU_RESUME_MS)); 5041 exit: 5042 if (amdgpu_sriov_vf(adev)) { 5043 amdgpu_virt_init_data_exchange(adev); 5044 amdgpu_virt_release_full_gpu(adev, true); 5045 } 5046 5047 if (r) 5048 return r; 5049 5050 /* Make sure IB tests flushed */ 5051 flush_delayed_work(&adev->delayed_init_work); 5052 5053 if (notify_clients) 5054 drm_client_dev_resume(adev_to_drm(adev), false); 5055 5056 amdgpu_ras_resume(adev); 5057 5058 if (adev->mode_info.num_crtc) { 5059 /* 5060 * Most of the connector probing functions try to acquire runtime pm 5061 * refs to ensure that the GPU is powered on when connector polling is 5062 * performed. Since we're calling this from a runtime PM callback, 5063 * trying to acquire rpm refs will cause us to deadlock. 5064 * 5065 * Since we're guaranteed to be holding the rpm lock, it's safe to 5066 * temporarily disable the rpm helpers so this doesn't deadlock us. 5067 */ 5068 #ifdef CONFIG_PM 5069 dev->dev->power.disable_depth++; 5070 #endif 5071 if (!adev->dc_enabled) 5072 drm_helper_hpd_irq_event(dev); 5073 else 5074 drm_kms_helper_hotplug_event(dev); 5075 #ifdef CONFIG_PM 5076 dev->dev->power.disable_depth--; 5077 #endif 5078 } 5079 adev->in_suspend = false; 5080 5081 if (adev->enable_mes) 5082 amdgpu_mes_self_test(adev); 5083 5084 if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D0)) 5085 DRM_WARN("smart shift update failed\n"); 5086 5087 return 0; 5088 } 5089 5090 /** 5091 * amdgpu_device_ip_check_soft_reset - did soft reset succeed 5092 * 5093 * @adev: amdgpu_device pointer 5094 * 5095 * The list of all the hardware IPs that make up the asic is walked and 5096 * the check_soft_reset callbacks are run. check_soft_reset determines 5097 * if the asic is still hung or not. 5098 * Returns true if any of the IPs are still in a hung state, false if not. 5099 */ 5100 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev) 5101 { 5102 int i; 5103 bool asic_hang = false; 5104 5105 if (amdgpu_sriov_vf(adev)) 5106 return true; 5107 5108 if (amdgpu_asic_need_full_reset(adev)) 5109 return true; 5110 5111 for (i = 0; i < adev->num_ip_blocks; i++) { 5112 if (!adev->ip_blocks[i].status.valid) 5113 continue; 5114 if (adev->ip_blocks[i].version->funcs->check_soft_reset) 5115 adev->ip_blocks[i].status.hang = 5116 adev->ip_blocks[i].version->funcs->check_soft_reset( 5117 &adev->ip_blocks[i]); 5118 if (adev->ip_blocks[i].status.hang) { 5119 dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name); 5120 asic_hang = true; 5121 } 5122 } 5123 return asic_hang; 5124 } 5125 5126 /** 5127 * amdgpu_device_ip_pre_soft_reset - prepare for soft reset 5128 * 5129 * @adev: amdgpu_device pointer 5130 * 5131 * The list of all the hardware IPs that make up the asic is walked and the 5132 * pre_soft_reset callbacks are run if the block is hung. pre_soft_reset 5133 * handles any IP specific hardware or software state changes that are 5134 * necessary for a soft reset to succeed. 5135 * Returns 0 on success, negative error code on failure. 5136 */ 5137 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev) 5138 { 5139 int i, r = 0; 5140 5141 for (i = 0; i < adev->num_ip_blocks; i++) { 5142 if (!adev->ip_blocks[i].status.valid) 5143 continue; 5144 if (adev->ip_blocks[i].status.hang && 5145 adev->ip_blocks[i].version->funcs->pre_soft_reset) { 5146 r = adev->ip_blocks[i].version->funcs->pre_soft_reset(&adev->ip_blocks[i]); 5147 if (r) 5148 return r; 5149 } 5150 } 5151 5152 return 0; 5153 } 5154 5155 /** 5156 * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed 5157 * 5158 * @adev: amdgpu_device pointer 5159 * 5160 * Some hardware IPs cannot be soft reset. If they are hung, a full gpu 5161 * reset is necessary to recover. 5162 * Returns true if a full asic reset is required, false if not. 5163 */ 5164 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev) 5165 { 5166 int i; 5167 5168 if (amdgpu_asic_need_full_reset(adev)) 5169 return true; 5170 5171 for (i = 0; i < adev->num_ip_blocks; i++) { 5172 if (!adev->ip_blocks[i].status.valid) 5173 continue; 5174 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) || 5175 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) || 5176 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) || 5177 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) || 5178 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) { 5179 if (adev->ip_blocks[i].status.hang) { 5180 dev_info(adev->dev, "Some block need full reset!\n"); 5181 return true; 5182 } 5183 } 5184 } 5185 return false; 5186 } 5187 5188 /** 5189 * amdgpu_device_ip_soft_reset - do a soft reset 5190 * 5191 * @adev: amdgpu_device pointer 5192 * 5193 * The list of all the hardware IPs that make up the asic is walked and the 5194 * soft_reset callbacks are run if the block is hung. soft_reset handles any 5195 * IP specific hardware or software state changes that are necessary to soft 5196 * reset the IP. 5197 * Returns 0 on success, negative error code on failure. 5198 */ 5199 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev) 5200 { 5201 int i, r = 0; 5202 5203 for (i = 0; i < adev->num_ip_blocks; i++) { 5204 if (!adev->ip_blocks[i].status.valid) 5205 continue; 5206 if (adev->ip_blocks[i].status.hang && 5207 adev->ip_blocks[i].version->funcs->soft_reset) { 5208 r = adev->ip_blocks[i].version->funcs->soft_reset(&adev->ip_blocks[i]); 5209 if (r) 5210 return r; 5211 } 5212 } 5213 5214 return 0; 5215 } 5216 5217 /** 5218 * amdgpu_device_ip_post_soft_reset - clean up from soft reset 5219 * 5220 * @adev: amdgpu_device pointer 5221 * 5222 * The list of all the hardware IPs that make up the asic is walked and the 5223 * post_soft_reset callbacks are run if the asic was hung. post_soft_reset 5224 * handles any IP specific hardware or software state changes that are 5225 * necessary after the IP has been soft reset. 5226 * Returns 0 on success, negative error code on failure. 5227 */ 5228 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev) 5229 { 5230 int i, r = 0; 5231 5232 for (i = 0; i < adev->num_ip_blocks; i++) { 5233 if (!adev->ip_blocks[i].status.valid) 5234 continue; 5235 if (adev->ip_blocks[i].status.hang && 5236 adev->ip_blocks[i].version->funcs->post_soft_reset) 5237 r = adev->ip_blocks[i].version->funcs->post_soft_reset(&adev->ip_blocks[i]); 5238 if (r) 5239 return r; 5240 } 5241 5242 return 0; 5243 } 5244 5245 /** 5246 * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf 5247 * 5248 * @adev: amdgpu_device pointer 5249 * @reset_context: amdgpu reset context pointer 5250 * 5251 * do VF FLR and reinitialize Asic 5252 * return 0 means succeeded otherwise failed 5253 */ 5254 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev, 5255 struct amdgpu_reset_context *reset_context) 5256 { 5257 int r; 5258 struct amdgpu_hive_info *hive = NULL; 5259 5260 if (test_bit(AMDGPU_HOST_FLR, &reset_context->flags)) { 5261 if (!amdgpu_ras_get_fed_status(adev)) 5262 amdgpu_virt_ready_to_reset(adev); 5263 amdgpu_virt_wait_reset(adev); 5264 clear_bit(AMDGPU_HOST_FLR, &reset_context->flags); 5265 r = amdgpu_virt_request_full_gpu(adev, true); 5266 } else { 5267 r = amdgpu_virt_reset_gpu(adev); 5268 } 5269 if (r) 5270 return r; 5271 5272 amdgpu_ras_clear_err_state(adev); 5273 amdgpu_irq_gpu_reset_resume_helper(adev); 5274 5275 /* some sw clean up VF needs to do before recover */ 5276 amdgpu_virt_post_reset(adev); 5277 5278 /* Resume IP prior to SMC */ 5279 r = amdgpu_device_ip_reinit_early_sriov(adev); 5280 if (r) 5281 return r; 5282 5283 amdgpu_virt_init_data_exchange(adev); 5284 5285 r = amdgpu_device_fw_loading(adev); 5286 if (r) 5287 return r; 5288 5289 /* now we are okay to resume SMC/CP/SDMA */ 5290 r = amdgpu_device_ip_reinit_late_sriov(adev); 5291 if (r) 5292 return r; 5293 5294 hive = amdgpu_get_xgmi_hive(adev); 5295 /* Update PSP FW topology after reset */ 5296 if (hive && adev->gmc.xgmi.num_physical_nodes > 1) 5297 r = amdgpu_xgmi_update_topology(hive, adev); 5298 if (hive) 5299 amdgpu_put_xgmi_hive(hive); 5300 if (r) 5301 return r; 5302 5303 r = amdgpu_ib_ring_tests(adev); 5304 if (r) 5305 return r; 5306 5307 if (adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) 5308 amdgpu_inc_vram_lost(adev); 5309 5310 /* need to be called during full access so we can't do it later like 5311 * bare-metal does. 5312 */ 5313 amdgpu_amdkfd_post_reset(adev); 5314 amdgpu_virt_release_full_gpu(adev, true); 5315 5316 /* Aldebaran and gfx_11_0_3 support ras in SRIOV, so need resume ras during reset */ 5317 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) || 5318 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) || 5319 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4) || 5320 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3)) 5321 amdgpu_ras_resume(adev); 5322 5323 amdgpu_virt_ras_telemetry_post_reset(adev); 5324 5325 return 0; 5326 } 5327 5328 /** 5329 * amdgpu_device_has_job_running - check if there is any unfinished job 5330 * 5331 * @adev: amdgpu_device pointer 5332 * 5333 * check if there is any job running on the device when guest driver receives 5334 * FLR notification from host driver. If there are still jobs running, then 5335 * the guest driver will not respond the FLR reset. Instead, let the job hit 5336 * the timeout and guest driver then issue the reset request. 5337 */ 5338 bool amdgpu_device_has_job_running(struct amdgpu_device *adev) 5339 { 5340 int i; 5341 5342 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 5343 struct amdgpu_ring *ring = adev->rings[i]; 5344 5345 if (!amdgpu_ring_sched_ready(ring)) 5346 continue; 5347 5348 if (amdgpu_fence_count_emitted(ring)) 5349 return true; 5350 } 5351 return false; 5352 } 5353 5354 /** 5355 * amdgpu_device_should_recover_gpu - check if we should try GPU recovery 5356 * 5357 * @adev: amdgpu_device pointer 5358 * 5359 * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover 5360 * a hung GPU. 5361 */ 5362 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev) 5363 { 5364 5365 if (amdgpu_gpu_recovery == 0) 5366 goto disabled; 5367 5368 /* Skip soft reset check in fatal error mode */ 5369 if (!amdgpu_ras_is_poison_mode_supported(adev)) 5370 return true; 5371 5372 if (amdgpu_sriov_vf(adev)) 5373 return true; 5374 5375 if (amdgpu_gpu_recovery == -1) { 5376 switch (adev->asic_type) { 5377 #ifdef CONFIG_DRM_AMDGPU_SI 5378 case CHIP_VERDE: 5379 case CHIP_TAHITI: 5380 case CHIP_PITCAIRN: 5381 case CHIP_OLAND: 5382 case CHIP_HAINAN: 5383 #endif 5384 #ifdef CONFIG_DRM_AMDGPU_CIK 5385 case CHIP_KAVERI: 5386 case CHIP_KABINI: 5387 case CHIP_MULLINS: 5388 #endif 5389 case CHIP_CARRIZO: 5390 case CHIP_STONEY: 5391 case CHIP_CYAN_SKILLFISH: 5392 goto disabled; 5393 default: 5394 break; 5395 } 5396 } 5397 5398 return true; 5399 5400 disabled: 5401 dev_info(adev->dev, "GPU recovery disabled.\n"); 5402 return false; 5403 } 5404 5405 int amdgpu_device_mode1_reset(struct amdgpu_device *adev) 5406 { 5407 u32 i; 5408 int ret = 0; 5409 5410 if (adev->bios) 5411 amdgpu_atombios_scratch_regs_engine_hung(adev, true); 5412 5413 dev_info(adev->dev, "GPU mode1 reset\n"); 5414 5415 /* Cache the state before bus master disable. The saved config space 5416 * values are used in other cases like restore after mode-2 reset. 5417 */ 5418 amdgpu_device_cache_pci_state(adev->pdev); 5419 5420 /* disable BM */ 5421 pci_clear_master(adev->pdev); 5422 5423 if (amdgpu_dpm_is_mode1_reset_supported(adev)) { 5424 dev_info(adev->dev, "GPU smu mode1 reset\n"); 5425 ret = amdgpu_dpm_mode1_reset(adev); 5426 } else { 5427 dev_info(adev->dev, "GPU psp mode1 reset\n"); 5428 ret = psp_gpu_reset(adev); 5429 } 5430 5431 if (ret) 5432 goto mode1_reset_failed; 5433 5434 amdgpu_device_load_pci_state(adev->pdev); 5435 ret = amdgpu_psp_wait_for_bootloader(adev); 5436 if (ret) 5437 goto mode1_reset_failed; 5438 5439 /* wait for asic to come out of reset */ 5440 for (i = 0; i < adev->usec_timeout; i++) { 5441 u32 memsize = adev->nbio.funcs->get_memsize(adev); 5442 5443 if (memsize != 0xffffffff) 5444 break; 5445 udelay(1); 5446 } 5447 5448 if (i >= adev->usec_timeout) { 5449 ret = -ETIMEDOUT; 5450 goto mode1_reset_failed; 5451 } 5452 5453 if (adev->bios) 5454 amdgpu_atombios_scratch_regs_engine_hung(adev, false); 5455 5456 return 0; 5457 5458 mode1_reset_failed: 5459 dev_err(adev->dev, "GPU mode1 reset failed\n"); 5460 return ret; 5461 } 5462 5463 int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, 5464 struct amdgpu_reset_context *reset_context) 5465 { 5466 int i, r = 0; 5467 struct amdgpu_job *job = NULL; 5468 struct amdgpu_device *tmp_adev = reset_context->reset_req_dev; 5469 bool need_full_reset = 5470 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 5471 5472 if (reset_context->reset_req_dev == adev) 5473 job = reset_context->job; 5474 5475 if (amdgpu_sriov_vf(adev)) 5476 amdgpu_virt_pre_reset(adev); 5477 5478 amdgpu_fence_driver_isr_toggle(adev, true); 5479 5480 /* block all schedulers and reset given job's ring */ 5481 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 5482 struct amdgpu_ring *ring = adev->rings[i]; 5483 5484 if (!amdgpu_ring_sched_ready(ring)) 5485 continue; 5486 5487 /* Clear job fence from fence drv to avoid force_completion 5488 * leave NULL and vm flush fence in fence drv 5489 */ 5490 amdgpu_fence_driver_clear_job_fences(ring); 5491 5492 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */ 5493 amdgpu_fence_driver_force_completion(ring); 5494 } 5495 5496 amdgpu_fence_driver_isr_toggle(adev, false); 5497 5498 if (job && job->vm) 5499 drm_sched_increase_karma(&job->base); 5500 5501 r = amdgpu_reset_prepare_hwcontext(adev, reset_context); 5502 /* If reset handler not implemented, continue; otherwise return */ 5503 if (r == -EOPNOTSUPP) 5504 r = 0; 5505 else 5506 return r; 5507 5508 /* Don't suspend on bare metal if we are not going to HW reset the ASIC */ 5509 if (!amdgpu_sriov_vf(adev)) { 5510 5511 if (!need_full_reset) 5512 need_full_reset = amdgpu_device_ip_need_full_reset(adev); 5513 5514 if (!need_full_reset && amdgpu_gpu_recovery && 5515 amdgpu_device_ip_check_soft_reset(adev)) { 5516 amdgpu_device_ip_pre_soft_reset(adev); 5517 r = amdgpu_device_ip_soft_reset(adev); 5518 amdgpu_device_ip_post_soft_reset(adev); 5519 if (r || amdgpu_device_ip_check_soft_reset(adev)) { 5520 dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n"); 5521 need_full_reset = true; 5522 } 5523 } 5524 5525 if (!test_bit(AMDGPU_SKIP_COREDUMP, &reset_context->flags)) { 5526 dev_info(tmp_adev->dev, "Dumping IP State\n"); 5527 /* Trigger ip dump before we reset the asic */ 5528 for (i = 0; i < tmp_adev->num_ip_blocks; i++) 5529 if (tmp_adev->ip_blocks[i].version->funcs->dump_ip_state) 5530 tmp_adev->ip_blocks[i].version->funcs 5531 ->dump_ip_state((void *)&tmp_adev->ip_blocks[i]); 5532 dev_info(tmp_adev->dev, "Dumping IP State Completed\n"); 5533 } 5534 5535 if (need_full_reset) 5536 r = amdgpu_device_ip_suspend(adev); 5537 if (need_full_reset) 5538 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 5539 else 5540 clear_bit(AMDGPU_NEED_FULL_RESET, 5541 &reset_context->flags); 5542 } 5543 5544 return r; 5545 } 5546 5547 int amdgpu_device_reinit_after_reset(struct amdgpu_reset_context *reset_context) 5548 { 5549 struct list_head *device_list_handle; 5550 bool full_reset, vram_lost = false; 5551 struct amdgpu_device *tmp_adev; 5552 int r, init_level; 5553 5554 device_list_handle = reset_context->reset_device_list; 5555 5556 if (!device_list_handle) 5557 return -EINVAL; 5558 5559 full_reset = test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 5560 5561 /** 5562 * If it's reset on init, it's default init level, otherwise keep level 5563 * as recovery level. 5564 */ 5565 if (reset_context->method == AMD_RESET_METHOD_ON_INIT) 5566 init_level = AMDGPU_INIT_LEVEL_DEFAULT; 5567 else 5568 init_level = AMDGPU_INIT_LEVEL_RESET_RECOVERY; 5569 5570 r = 0; 5571 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 5572 amdgpu_set_init_level(tmp_adev, init_level); 5573 if (full_reset) { 5574 /* post card */ 5575 amdgpu_ras_clear_err_state(tmp_adev); 5576 r = amdgpu_device_asic_init(tmp_adev); 5577 if (r) { 5578 dev_warn(tmp_adev->dev, "asic atom init failed!"); 5579 } else { 5580 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n"); 5581 5582 r = amdgpu_device_ip_resume_phase1(tmp_adev); 5583 if (r) 5584 goto out; 5585 5586 vram_lost = amdgpu_device_check_vram_lost(tmp_adev); 5587 5588 if (!test_bit(AMDGPU_SKIP_COREDUMP, &reset_context->flags)) 5589 amdgpu_coredump(tmp_adev, false, vram_lost, reset_context->job); 5590 5591 if (vram_lost) { 5592 DRM_INFO("VRAM is lost due to GPU reset!\n"); 5593 amdgpu_inc_vram_lost(tmp_adev); 5594 } 5595 5596 r = amdgpu_device_fw_loading(tmp_adev); 5597 if (r) 5598 return r; 5599 5600 r = amdgpu_xcp_restore_partition_mode( 5601 tmp_adev->xcp_mgr); 5602 if (r) 5603 goto out; 5604 5605 r = amdgpu_device_ip_resume_phase2(tmp_adev); 5606 if (r) 5607 goto out; 5608 5609 if (tmp_adev->mman.buffer_funcs_ring->sched.ready) 5610 amdgpu_ttm_set_buffer_funcs_status(tmp_adev, true); 5611 5612 r = amdgpu_device_ip_resume_phase3(tmp_adev); 5613 if (r) 5614 goto out; 5615 5616 if (vram_lost) 5617 amdgpu_device_fill_reset_magic(tmp_adev); 5618 5619 /* 5620 * Add this ASIC as tracked as reset was already 5621 * complete successfully. 5622 */ 5623 amdgpu_register_gpu_instance(tmp_adev); 5624 5625 if (!reset_context->hive && 5626 tmp_adev->gmc.xgmi.num_physical_nodes > 1) 5627 amdgpu_xgmi_add_device(tmp_adev); 5628 5629 r = amdgpu_device_ip_late_init(tmp_adev); 5630 if (r) 5631 goto out; 5632 5633 drm_client_dev_resume(adev_to_drm(tmp_adev), false); 5634 5635 /* 5636 * The GPU enters bad state once faulty pages 5637 * by ECC has reached the threshold, and ras 5638 * recovery is scheduled next. So add one check 5639 * here to break recovery if it indeed exceeds 5640 * bad page threshold, and remind user to 5641 * retire this GPU or setting one bigger 5642 * bad_page_threshold value to fix this once 5643 * probing driver again. 5644 */ 5645 if (!amdgpu_ras_is_rma(tmp_adev)) { 5646 /* must succeed. */ 5647 amdgpu_ras_resume(tmp_adev); 5648 } else { 5649 r = -EINVAL; 5650 goto out; 5651 } 5652 5653 /* Update PSP FW topology after reset */ 5654 if (reset_context->hive && 5655 tmp_adev->gmc.xgmi.num_physical_nodes > 1) 5656 r = amdgpu_xgmi_update_topology( 5657 reset_context->hive, tmp_adev); 5658 } 5659 } 5660 5661 out: 5662 if (!r) { 5663 /* IP init is complete now, set level as default */ 5664 amdgpu_set_init_level(tmp_adev, 5665 AMDGPU_INIT_LEVEL_DEFAULT); 5666 amdgpu_irq_gpu_reset_resume_helper(tmp_adev); 5667 r = amdgpu_ib_ring_tests(tmp_adev); 5668 if (r) { 5669 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r); 5670 r = -EAGAIN; 5671 goto end; 5672 } 5673 } 5674 5675 if (r) 5676 tmp_adev->asic_reset_res = r; 5677 } 5678 5679 end: 5680 return r; 5681 } 5682 5683 int amdgpu_do_asic_reset(struct list_head *device_list_handle, 5684 struct amdgpu_reset_context *reset_context) 5685 { 5686 struct amdgpu_device *tmp_adev = NULL; 5687 bool need_full_reset, skip_hw_reset; 5688 int r = 0; 5689 5690 /* Try reset handler method first */ 5691 tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device, 5692 reset_list); 5693 5694 reset_context->reset_device_list = device_list_handle; 5695 r = amdgpu_reset_perform_reset(tmp_adev, reset_context); 5696 /* If reset handler not implemented, continue; otherwise return */ 5697 if (r == -EOPNOTSUPP) 5698 r = 0; 5699 else 5700 return r; 5701 5702 /* Reset handler not implemented, use the default method */ 5703 need_full_reset = 5704 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 5705 skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags); 5706 5707 /* 5708 * ASIC reset has to be done on all XGMI hive nodes ASAP 5709 * to allow proper links negotiation in FW (within 1 sec) 5710 */ 5711 if (!skip_hw_reset && need_full_reset) { 5712 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 5713 /* For XGMI run all resets in parallel to speed up the process */ 5714 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) { 5715 if (!queue_work(system_unbound_wq, 5716 &tmp_adev->xgmi_reset_work)) 5717 r = -EALREADY; 5718 } else 5719 r = amdgpu_asic_reset(tmp_adev); 5720 5721 if (r) { 5722 dev_err(tmp_adev->dev, 5723 "ASIC reset failed with error, %d for drm dev, %s", 5724 r, adev_to_drm(tmp_adev)->unique); 5725 goto out; 5726 } 5727 } 5728 5729 /* For XGMI wait for all resets to complete before proceed */ 5730 if (!r) { 5731 list_for_each_entry(tmp_adev, device_list_handle, 5732 reset_list) { 5733 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) { 5734 flush_work(&tmp_adev->xgmi_reset_work); 5735 r = tmp_adev->asic_reset_res; 5736 if (r) 5737 break; 5738 } 5739 } 5740 } 5741 } 5742 5743 if (!r && amdgpu_ras_intr_triggered()) { 5744 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 5745 amdgpu_ras_reset_error_count(tmp_adev, 5746 AMDGPU_RAS_BLOCK__MMHUB); 5747 } 5748 5749 amdgpu_ras_intr_cleared(); 5750 } 5751 5752 r = amdgpu_device_reinit_after_reset(reset_context); 5753 if (r == -EAGAIN) 5754 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 5755 else 5756 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 5757 5758 out: 5759 return r; 5760 } 5761 5762 static void amdgpu_device_set_mp1_state(struct amdgpu_device *adev) 5763 { 5764 5765 switch (amdgpu_asic_reset_method(adev)) { 5766 case AMD_RESET_METHOD_MODE1: 5767 adev->mp1_state = PP_MP1_STATE_SHUTDOWN; 5768 break; 5769 case AMD_RESET_METHOD_MODE2: 5770 adev->mp1_state = PP_MP1_STATE_RESET; 5771 break; 5772 default: 5773 adev->mp1_state = PP_MP1_STATE_NONE; 5774 break; 5775 } 5776 } 5777 5778 static void amdgpu_device_unset_mp1_state(struct amdgpu_device *adev) 5779 { 5780 amdgpu_vf_error_trans_all(adev); 5781 adev->mp1_state = PP_MP1_STATE_NONE; 5782 } 5783 5784 static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev) 5785 { 5786 struct pci_dev *p = NULL; 5787 5788 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus), 5789 adev->pdev->bus->number, 1); 5790 if (p) { 5791 pm_runtime_enable(&(p->dev)); 5792 pm_runtime_resume(&(p->dev)); 5793 } 5794 5795 pci_dev_put(p); 5796 } 5797 5798 static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev) 5799 { 5800 enum amd_reset_method reset_method; 5801 struct pci_dev *p = NULL; 5802 u64 expires; 5803 5804 /* 5805 * For now, only BACO and mode1 reset are confirmed 5806 * to suffer the audio issue without proper suspended. 5807 */ 5808 reset_method = amdgpu_asic_reset_method(adev); 5809 if ((reset_method != AMD_RESET_METHOD_BACO) && 5810 (reset_method != AMD_RESET_METHOD_MODE1)) 5811 return -EINVAL; 5812 5813 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus), 5814 adev->pdev->bus->number, 1); 5815 if (!p) 5816 return -ENODEV; 5817 5818 expires = pm_runtime_autosuspend_expiration(&(p->dev)); 5819 if (!expires) 5820 /* 5821 * If we cannot get the audio device autosuspend delay, 5822 * a fixed 4S interval will be used. Considering 3S is 5823 * the audio controller default autosuspend delay setting. 5824 * 4S used here is guaranteed to cover that. 5825 */ 5826 expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL; 5827 5828 while (!pm_runtime_status_suspended(&(p->dev))) { 5829 if (!pm_runtime_suspend(&(p->dev))) 5830 break; 5831 5832 if (expires < ktime_get_mono_fast_ns()) { 5833 dev_warn(adev->dev, "failed to suspend display audio\n"); 5834 pci_dev_put(p); 5835 /* TODO: abort the succeeding gpu reset? */ 5836 return -ETIMEDOUT; 5837 } 5838 } 5839 5840 pm_runtime_disable(&(p->dev)); 5841 5842 pci_dev_put(p); 5843 return 0; 5844 } 5845 5846 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev) 5847 { 5848 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5849 5850 #if defined(CONFIG_DEBUG_FS) 5851 if (!amdgpu_sriov_vf(adev)) 5852 cancel_work(&adev->reset_work); 5853 #endif 5854 5855 if (adev->kfd.dev) 5856 cancel_work(&adev->kfd.reset_work); 5857 5858 if (amdgpu_sriov_vf(adev)) 5859 cancel_work(&adev->virt.flr_work); 5860 5861 if (con && adev->ras_enabled) 5862 cancel_work(&con->recovery_work); 5863 5864 } 5865 5866 static int amdgpu_device_health_check(struct list_head *device_list_handle) 5867 { 5868 struct amdgpu_device *tmp_adev; 5869 int ret = 0; 5870 u32 status; 5871 5872 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 5873 pci_read_config_dword(tmp_adev->pdev, PCI_COMMAND, &status); 5874 if (PCI_POSSIBLE_ERROR(status)) { 5875 dev_err(tmp_adev->dev, "device lost from bus!"); 5876 ret = -ENODEV; 5877 } 5878 } 5879 5880 return ret; 5881 } 5882 5883 /** 5884 * amdgpu_device_gpu_recover - reset the asic and recover scheduler 5885 * 5886 * @adev: amdgpu_device pointer 5887 * @job: which job trigger hang 5888 * @reset_context: amdgpu reset context pointer 5889 * 5890 * Attempt to reset the GPU if it has hung (all asics). 5891 * Attempt to do soft-reset or full-reset and reinitialize Asic 5892 * Returns 0 for success or an error on failure. 5893 */ 5894 5895 int amdgpu_device_gpu_recover(struct amdgpu_device *adev, 5896 struct amdgpu_job *job, 5897 struct amdgpu_reset_context *reset_context) 5898 { 5899 struct list_head device_list, *device_list_handle = NULL; 5900 bool job_signaled = false; 5901 struct amdgpu_hive_info *hive = NULL; 5902 struct amdgpu_device *tmp_adev = NULL; 5903 int i, r = 0; 5904 bool need_emergency_restart = false; 5905 bool audio_suspended = false; 5906 int retry_limit = AMDGPU_MAX_RETRY_LIMIT; 5907 5908 /* 5909 * If it reaches here because of hang/timeout and a RAS error is 5910 * detected at the same time, let RAS recovery take care of it. 5911 */ 5912 if (amdgpu_ras_is_err_state(adev, AMDGPU_RAS_BLOCK__ANY) && 5913 !amdgpu_sriov_vf(adev) && 5914 reset_context->src != AMDGPU_RESET_SRC_RAS) { 5915 dev_dbg(adev->dev, 5916 "Gpu recovery from source: %d yielding to RAS error recovery handling", 5917 reset_context->src); 5918 return 0; 5919 } 5920 /* 5921 * Special case: RAS triggered and full reset isn't supported 5922 */ 5923 need_emergency_restart = amdgpu_ras_need_emergency_restart(adev); 5924 5925 /* 5926 * Flush RAM to disk so that after reboot 5927 * the user can read log and see why the system rebooted. 5928 */ 5929 if (need_emergency_restart && amdgpu_ras_get_context(adev) && 5930 amdgpu_ras_get_context(adev)->reboot) { 5931 DRM_WARN("Emergency reboot."); 5932 5933 ksys_sync_helper(); 5934 emergency_restart(); 5935 } 5936 5937 dev_info(adev->dev, "GPU %s begin!\n", 5938 need_emergency_restart ? "jobs stop":"reset"); 5939 5940 if (!amdgpu_sriov_vf(adev)) 5941 hive = amdgpu_get_xgmi_hive(adev); 5942 if (hive) 5943 mutex_lock(&hive->hive_lock); 5944 5945 reset_context->job = job; 5946 reset_context->hive = hive; 5947 /* 5948 * Build list of devices to reset. 5949 * In case we are in XGMI hive mode, resort the device list 5950 * to put adev in the 1st position. 5951 */ 5952 INIT_LIST_HEAD(&device_list); 5953 if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1) && hive) { 5954 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { 5955 list_add_tail(&tmp_adev->reset_list, &device_list); 5956 if (adev->shutdown) 5957 tmp_adev->shutdown = true; 5958 } 5959 if (!list_is_first(&adev->reset_list, &device_list)) 5960 list_rotate_to_front(&adev->reset_list, &device_list); 5961 device_list_handle = &device_list; 5962 } else { 5963 list_add_tail(&adev->reset_list, &device_list); 5964 device_list_handle = &device_list; 5965 } 5966 5967 if (!amdgpu_sriov_vf(adev)) { 5968 r = amdgpu_device_health_check(device_list_handle); 5969 if (r) 5970 goto end_reset; 5971 } 5972 5973 /* We need to lock reset domain only once both for XGMI and single device */ 5974 tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device, 5975 reset_list); 5976 amdgpu_device_lock_reset_domain(tmp_adev->reset_domain); 5977 5978 /* block all schedulers and reset given job's ring */ 5979 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 5980 5981 amdgpu_device_set_mp1_state(tmp_adev); 5982 5983 /* 5984 * Try to put the audio codec into suspend state 5985 * before gpu reset started. 5986 * 5987 * Due to the power domain of the graphics device 5988 * is shared with AZ power domain. Without this, 5989 * we may change the audio hardware from behind 5990 * the audio driver's back. That will trigger 5991 * some audio codec errors. 5992 */ 5993 if (!amdgpu_device_suspend_display_audio(tmp_adev)) 5994 audio_suspended = true; 5995 5996 amdgpu_ras_set_error_query_ready(tmp_adev, false); 5997 5998 cancel_delayed_work_sync(&tmp_adev->delayed_init_work); 5999 6000 amdgpu_amdkfd_pre_reset(tmp_adev, reset_context); 6001 6002 /* 6003 * Mark these ASICs to be reset as untracked first 6004 * And add them back after reset completed 6005 */ 6006 amdgpu_unregister_gpu_instance(tmp_adev); 6007 6008 drm_client_dev_suspend(adev_to_drm(tmp_adev), false); 6009 6010 /* disable ras on ALL IPs */ 6011 if (!need_emergency_restart && 6012 amdgpu_device_ip_need_full_reset(tmp_adev)) 6013 amdgpu_ras_suspend(tmp_adev); 6014 6015 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 6016 struct amdgpu_ring *ring = tmp_adev->rings[i]; 6017 6018 if (!amdgpu_ring_sched_ready(ring)) 6019 continue; 6020 6021 drm_sched_stop(&ring->sched, job ? &job->base : NULL); 6022 6023 if (need_emergency_restart) 6024 amdgpu_job_stop_all_jobs_on_sched(&ring->sched); 6025 } 6026 atomic_inc(&tmp_adev->gpu_reset_counter); 6027 } 6028 6029 if (need_emergency_restart) 6030 goto skip_sched_resume; 6031 6032 /* 6033 * Must check guilty signal here since after this point all old 6034 * HW fences are force signaled. 6035 * 6036 * job->base holds a reference to parent fence 6037 */ 6038 if (job && dma_fence_is_signaled(&job->hw_fence)) { 6039 job_signaled = true; 6040 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset"); 6041 goto skip_hw_reset; 6042 } 6043 6044 retry: /* Rest of adevs pre asic reset from XGMI hive. */ 6045 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 6046 r = amdgpu_device_pre_asic_reset(tmp_adev, reset_context); 6047 /*TODO Should we stop ?*/ 6048 if (r) { 6049 dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ", 6050 r, adev_to_drm(tmp_adev)->unique); 6051 tmp_adev->asic_reset_res = r; 6052 } 6053 } 6054 6055 /* Actual ASIC resets if needed.*/ 6056 /* Host driver will handle XGMI hive reset for SRIOV */ 6057 if (amdgpu_sriov_vf(adev)) { 6058 if (amdgpu_ras_get_fed_status(adev) || amdgpu_virt_rcvd_ras_interrupt(adev)) { 6059 dev_dbg(adev->dev, "Detected RAS error, wait for FLR completion\n"); 6060 amdgpu_ras_set_fed(adev, true); 6061 set_bit(AMDGPU_HOST_FLR, &reset_context->flags); 6062 } 6063 6064 r = amdgpu_device_reset_sriov(adev, reset_context); 6065 if (AMDGPU_RETRY_SRIOV_RESET(r) && (retry_limit--) > 0) { 6066 amdgpu_virt_release_full_gpu(adev, true); 6067 goto retry; 6068 } 6069 if (r) 6070 adev->asic_reset_res = r; 6071 } else { 6072 r = amdgpu_do_asic_reset(device_list_handle, reset_context); 6073 if (r && r == -EAGAIN) 6074 goto retry; 6075 } 6076 6077 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 6078 /* 6079 * Drop any pending non scheduler resets queued before reset is done. 6080 * Any reset scheduled after this point would be valid. Scheduler resets 6081 * were already dropped during drm_sched_stop and no new ones can come 6082 * in before drm_sched_start. 6083 */ 6084 amdgpu_device_stop_pending_resets(tmp_adev); 6085 } 6086 6087 skip_hw_reset: 6088 6089 /* Post ASIC reset for all devs .*/ 6090 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 6091 6092 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 6093 struct amdgpu_ring *ring = tmp_adev->rings[i]; 6094 6095 if (!amdgpu_ring_sched_ready(ring)) 6096 continue; 6097 6098 drm_sched_start(&ring->sched, 0); 6099 } 6100 6101 if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled) 6102 drm_helper_resume_force_mode(adev_to_drm(tmp_adev)); 6103 6104 if (tmp_adev->asic_reset_res) 6105 r = tmp_adev->asic_reset_res; 6106 6107 tmp_adev->asic_reset_res = 0; 6108 6109 if (r) { 6110 /* bad news, how to tell it to userspace ? 6111 * for ras error, we should report GPU bad status instead of 6112 * reset failure 6113 */ 6114 if (reset_context->src != AMDGPU_RESET_SRC_RAS || 6115 !amdgpu_ras_eeprom_check_err_threshold(tmp_adev)) 6116 dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", 6117 atomic_read(&tmp_adev->gpu_reset_counter)); 6118 amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r); 6119 } else { 6120 dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter)); 6121 if (amdgpu_acpi_smart_shift_update(adev_to_drm(tmp_adev), AMDGPU_SS_DEV_D0)) 6122 DRM_WARN("smart shift update failed\n"); 6123 } 6124 } 6125 6126 skip_sched_resume: 6127 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 6128 /* unlock kfd: SRIOV would do it separately */ 6129 if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev)) 6130 amdgpu_amdkfd_post_reset(tmp_adev); 6131 6132 /* kfd_post_reset will do nothing if kfd device is not initialized, 6133 * need to bring up kfd here if it's not be initialized before 6134 */ 6135 if (!adev->kfd.init_complete) 6136 amdgpu_amdkfd_device_init(adev); 6137 6138 if (audio_suspended) 6139 amdgpu_device_resume_display_audio(tmp_adev); 6140 6141 amdgpu_device_unset_mp1_state(tmp_adev); 6142 6143 amdgpu_ras_set_error_query_ready(tmp_adev, true); 6144 } 6145 6146 tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device, 6147 reset_list); 6148 amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain); 6149 6150 end_reset: 6151 if (hive) { 6152 mutex_unlock(&hive->hive_lock); 6153 amdgpu_put_xgmi_hive(hive); 6154 } 6155 6156 if (r) 6157 dev_info(adev->dev, "GPU reset end with ret = %d\n", r); 6158 6159 atomic_set(&adev->reset_domain->reset_res, r); 6160 return r; 6161 } 6162 6163 /** 6164 * amdgpu_device_partner_bandwidth - find the bandwidth of appropriate partner 6165 * 6166 * @adev: amdgpu_device pointer 6167 * @speed: pointer to the speed of the link 6168 * @width: pointer to the width of the link 6169 * 6170 * Evaluate the hierarchy to find the speed and bandwidth capabilities of the 6171 * first physical partner to an AMD dGPU. 6172 * This will exclude any virtual switches and links. 6173 */ 6174 static void amdgpu_device_partner_bandwidth(struct amdgpu_device *adev, 6175 enum pci_bus_speed *speed, 6176 enum pcie_link_width *width) 6177 { 6178 struct pci_dev *parent = adev->pdev; 6179 6180 if (!speed || !width) 6181 return; 6182 6183 *speed = PCI_SPEED_UNKNOWN; 6184 *width = PCIE_LNK_WIDTH_UNKNOWN; 6185 6186 if (amdgpu_device_pcie_dynamic_switching_supported(adev)) { 6187 while ((parent = pci_upstream_bridge(parent))) { 6188 /* skip upstream/downstream switches internal to dGPU*/ 6189 if (parent->vendor == PCI_VENDOR_ID_ATI) 6190 continue; 6191 *speed = pcie_get_speed_cap(parent); 6192 *width = pcie_get_width_cap(parent); 6193 break; 6194 } 6195 } else { 6196 /* use the current speeds rather than max if switching is not supported */ 6197 pcie_bandwidth_available(adev->pdev, NULL, speed, width); 6198 } 6199 } 6200 6201 /** 6202 * amdgpu_device_gpu_bandwidth - find the bandwidth of the GPU 6203 * 6204 * @adev: amdgpu_device pointer 6205 * @speed: pointer to the speed of the link 6206 * @width: pointer to the width of the link 6207 * 6208 * Evaluate the hierarchy to find the speed and bandwidth capabilities of the 6209 * AMD dGPU which may be a virtual upstream bridge. 6210 */ 6211 static void amdgpu_device_gpu_bandwidth(struct amdgpu_device *adev, 6212 enum pci_bus_speed *speed, 6213 enum pcie_link_width *width) 6214 { 6215 struct pci_dev *parent = adev->pdev; 6216 6217 if (!speed || !width) 6218 return; 6219 6220 parent = pci_upstream_bridge(parent); 6221 if (parent && parent->vendor == PCI_VENDOR_ID_ATI) { 6222 /* use the upstream/downstream switches internal to dGPU */ 6223 *speed = pcie_get_speed_cap(parent); 6224 *width = pcie_get_width_cap(parent); 6225 while ((parent = pci_upstream_bridge(parent))) { 6226 if (parent->vendor == PCI_VENDOR_ID_ATI) { 6227 /* use the upstream/downstream switches internal to dGPU */ 6228 *speed = pcie_get_speed_cap(parent); 6229 *width = pcie_get_width_cap(parent); 6230 } 6231 } 6232 } else { 6233 /* use the device itself */ 6234 *speed = pcie_get_speed_cap(adev->pdev); 6235 *width = pcie_get_width_cap(adev->pdev); 6236 } 6237 } 6238 6239 /** 6240 * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot 6241 * 6242 * @adev: amdgpu_device pointer 6243 * 6244 * Fetches and stores in the driver the PCIE capabilities (gen speed 6245 * and lanes) of the slot the device is in. Handles APUs and 6246 * virtualized environments where PCIE config space may not be available. 6247 */ 6248 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev) 6249 { 6250 enum pci_bus_speed speed_cap, platform_speed_cap; 6251 enum pcie_link_width platform_link_width, link_width; 6252 6253 if (amdgpu_pcie_gen_cap) 6254 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap; 6255 6256 if (amdgpu_pcie_lane_cap) 6257 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap; 6258 6259 /* covers APUs as well */ 6260 if (pci_is_root_bus(adev->pdev->bus) && !amdgpu_passthrough(adev)) { 6261 if (adev->pm.pcie_gen_mask == 0) 6262 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK; 6263 if (adev->pm.pcie_mlw_mask == 0) 6264 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK; 6265 return; 6266 } 6267 6268 if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask) 6269 return; 6270 6271 amdgpu_device_partner_bandwidth(adev, &platform_speed_cap, 6272 &platform_link_width); 6273 amdgpu_device_gpu_bandwidth(adev, &speed_cap, &link_width); 6274 6275 if (adev->pm.pcie_gen_mask == 0) { 6276 /* asic caps */ 6277 if (speed_cap == PCI_SPEED_UNKNOWN) { 6278 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6279 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 6280 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3); 6281 } else { 6282 if (speed_cap == PCIE_SPEED_32_0GT) 6283 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6284 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 6285 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 | 6286 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 | 6287 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5); 6288 else if (speed_cap == PCIE_SPEED_16_0GT) 6289 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6290 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 6291 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 | 6292 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4); 6293 else if (speed_cap == PCIE_SPEED_8_0GT) 6294 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6295 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 6296 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3); 6297 else if (speed_cap == PCIE_SPEED_5_0GT) 6298 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6299 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2); 6300 else 6301 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1; 6302 } 6303 /* platform caps */ 6304 if (platform_speed_cap == PCI_SPEED_UNKNOWN) { 6305 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6306 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2); 6307 } else { 6308 if (platform_speed_cap == PCIE_SPEED_32_0GT) 6309 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6310 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | 6311 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 | 6312 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 | 6313 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5); 6314 else if (platform_speed_cap == PCIE_SPEED_16_0GT) 6315 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6316 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | 6317 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 | 6318 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4); 6319 else if (platform_speed_cap == PCIE_SPEED_8_0GT) 6320 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6321 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | 6322 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3); 6323 else if (platform_speed_cap == PCIE_SPEED_5_0GT) 6324 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6325 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2); 6326 else 6327 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1; 6328 6329 } 6330 } 6331 if (adev->pm.pcie_mlw_mask == 0) { 6332 /* asic caps */ 6333 if (link_width == PCIE_LNK_WIDTH_UNKNOWN) { 6334 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_ASIC_PCIE_MLW_MASK; 6335 } else { 6336 switch (link_width) { 6337 case PCIE_LNK_X32: 6338 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X32 | 6339 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X16 | 6340 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X12 | 6341 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 | 6342 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 | 6343 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 | 6344 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1); 6345 break; 6346 case PCIE_LNK_X16: 6347 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X16 | 6348 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X12 | 6349 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 | 6350 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 | 6351 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 | 6352 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1); 6353 break; 6354 case PCIE_LNK_X12: 6355 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X12 | 6356 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 | 6357 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 | 6358 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 | 6359 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1); 6360 break; 6361 case PCIE_LNK_X8: 6362 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 | 6363 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 | 6364 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 | 6365 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1); 6366 break; 6367 case PCIE_LNK_X4: 6368 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 | 6369 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 | 6370 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1); 6371 break; 6372 case PCIE_LNK_X2: 6373 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 | 6374 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1); 6375 break; 6376 case PCIE_LNK_X1: 6377 adev->pm.pcie_mlw_mask |= CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1; 6378 break; 6379 default: 6380 break; 6381 } 6382 } 6383 /* platform caps */ 6384 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) { 6385 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK; 6386 } else { 6387 switch (platform_link_width) { 6388 case PCIE_LNK_X32: 6389 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 | 6390 CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 | 6391 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | 6392 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 6393 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 6394 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 6395 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 6396 break; 6397 case PCIE_LNK_X16: 6398 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 | 6399 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | 6400 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 6401 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 6402 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 6403 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 6404 break; 6405 case PCIE_LNK_X12: 6406 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | 6407 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 6408 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 6409 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 6410 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 6411 break; 6412 case PCIE_LNK_X8: 6413 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 6414 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 6415 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 6416 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 6417 break; 6418 case PCIE_LNK_X4: 6419 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 6420 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 6421 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 6422 break; 6423 case PCIE_LNK_X2: 6424 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 6425 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 6426 break; 6427 case PCIE_LNK_X1: 6428 adev->pm.pcie_mlw_mask |= CAIL_PCIE_LINK_WIDTH_SUPPORT_X1; 6429 break; 6430 default: 6431 break; 6432 } 6433 } 6434 } 6435 } 6436 6437 /** 6438 * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR 6439 * 6440 * @adev: amdgpu_device pointer 6441 * @peer_adev: amdgpu_device pointer for peer device trying to access @adev 6442 * 6443 * Return true if @peer_adev can access (DMA) @adev through the PCIe 6444 * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of 6445 * @peer_adev. 6446 */ 6447 bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev, 6448 struct amdgpu_device *peer_adev) 6449 { 6450 #ifdef CONFIG_HSA_AMD_P2P 6451 bool p2p_access = 6452 !adev->gmc.xgmi.connected_to_cpu && 6453 !(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0); 6454 if (!p2p_access) 6455 dev_info(adev->dev, "PCIe P2P access from peer device %s is not supported by the chipset\n", 6456 pci_name(peer_adev->pdev)); 6457 6458 bool is_large_bar = adev->gmc.visible_vram_size && 6459 adev->gmc.real_vram_size == adev->gmc.visible_vram_size; 6460 bool p2p_addressable = amdgpu_device_check_iommu_remap(peer_adev); 6461 6462 if (!p2p_addressable) { 6463 uint64_t address_mask = peer_adev->dev->dma_mask ? 6464 ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1); 6465 resource_size_t aper_limit = 6466 adev->gmc.aper_base + adev->gmc.aper_size - 1; 6467 6468 p2p_addressable = !(adev->gmc.aper_base & address_mask || 6469 aper_limit & address_mask); 6470 } 6471 return pcie_p2p && is_large_bar && p2p_access && p2p_addressable; 6472 #else 6473 return false; 6474 #endif 6475 } 6476 6477 int amdgpu_device_baco_enter(struct drm_device *dev) 6478 { 6479 struct amdgpu_device *adev = drm_to_adev(dev); 6480 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 6481 6482 if (!amdgpu_device_supports_baco(dev)) 6483 return -ENOTSUPP; 6484 6485 if (ras && adev->ras_enabled && 6486 adev->nbio.funcs->enable_doorbell_interrupt) 6487 adev->nbio.funcs->enable_doorbell_interrupt(adev, false); 6488 6489 return amdgpu_dpm_baco_enter(adev); 6490 } 6491 6492 int amdgpu_device_baco_exit(struct drm_device *dev) 6493 { 6494 struct amdgpu_device *adev = drm_to_adev(dev); 6495 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 6496 int ret = 0; 6497 6498 if (!amdgpu_device_supports_baco(dev)) 6499 return -ENOTSUPP; 6500 6501 ret = amdgpu_dpm_baco_exit(adev); 6502 if (ret) 6503 return ret; 6504 6505 if (ras && adev->ras_enabled && 6506 adev->nbio.funcs->enable_doorbell_interrupt) 6507 adev->nbio.funcs->enable_doorbell_interrupt(adev, true); 6508 6509 if (amdgpu_passthrough(adev) && adev->nbio.funcs && 6510 adev->nbio.funcs->clear_doorbell_interrupt) 6511 adev->nbio.funcs->clear_doorbell_interrupt(adev); 6512 6513 return 0; 6514 } 6515 6516 /** 6517 * amdgpu_pci_error_detected - Called when a PCI error is detected. 6518 * @pdev: PCI device struct 6519 * @state: PCI channel state 6520 * 6521 * Description: Called when a PCI error is detected. 6522 * 6523 * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT. 6524 */ 6525 pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state) 6526 { 6527 struct drm_device *dev = pci_get_drvdata(pdev); 6528 struct amdgpu_device *adev = drm_to_adev(dev); 6529 int i; 6530 6531 DRM_INFO("PCI error: detected callback, state(%d)!!\n", state); 6532 6533 if (adev->gmc.xgmi.num_physical_nodes > 1) { 6534 DRM_WARN("No support for XGMI hive yet..."); 6535 return PCI_ERS_RESULT_DISCONNECT; 6536 } 6537 6538 adev->pci_channel_state = state; 6539 6540 switch (state) { 6541 case pci_channel_io_normal: 6542 return PCI_ERS_RESULT_CAN_RECOVER; 6543 /* Fatal error, prepare for slot reset */ 6544 case pci_channel_io_frozen: 6545 /* 6546 * Locking adev->reset_domain->sem will prevent any external access 6547 * to GPU during PCI error recovery 6548 */ 6549 amdgpu_device_lock_reset_domain(adev->reset_domain); 6550 amdgpu_device_set_mp1_state(adev); 6551 6552 /* 6553 * Block any work scheduling as we do for regular GPU reset 6554 * for the duration of the recovery 6555 */ 6556 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 6557 struct amdgpu_ring *ring = adev->rings[i]; 6558 6559 if (!amdgpu_ring_sched_ready(ring)) 6560 continue; 6561 6562 drm_sched_stop(&ring->sched, NULL); 6563 } 6564 atomic_inc(&adev->gpu_reset_counter); 6565 return PCI_ERS_RESULT_NEED_RESET; 6566 case pci_channel_io_perm_failure: 6567 /* Permanent error, prepare for device removal */ 6568 return PCI_ERS_RESULT_DISCONNECT; 6569 } 6570 6571 return PCI_ERS_RESULT_NEED_RESET; 6572 } 6573 6574 /** 6575 * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers 6576 * @pdev: pointer to PCI device 6577 */ 6578 pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev) 6579 { 6580 6581 DRM_INFO("PCI error: mmio enabled callback!!\n"); 6582 6583 /* TODO - dump whatever for debugging purposes */ 6584 6585 /* This called only if amdgpu_pci_error_detected returns 6586 * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still 6587 * works, no need to reset slot. 6588 */ 6589 6590 return PCI_ERS_RESULT_RECOVERED; 6591 } 6592 6593 /** 6594 * amdgpu_pci_slot_reset - Called when PCI slot has been reset. 6595 * @pdev: PCI device struct 6596 * 6597 * Description: This routine is called by the pci error recovery 6598 * code after the PCI slot has been reset, just before we 6599 * should resume normal operations. 6600 */ 6601 pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev) 6602 { 6603 struct drm_device *dev = pci_get_drvdata(pdev); 6604 struct amdgpu_device *adev = drm_to_adev(dev); 6605 int r, i; 6606 struct amdgpu_reset_context reset_context; 6607 u32 memsize; 6608 struct list_head device_list; 6609 6610 /* PCI error slot reset should be skipped During RAS recovery */ 6611 if ((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) || 6612 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4)) && 6613 amdgpu_ras_in_recovery(adev)) 6614 return PCI_ERS_RESULT_RECOVERED; 6615 6616 DRM_INFO("PCI error: slot reset callback!!\n"); 6617 6618 memset(&reset_context, 0, sizeof(reset_context)); 6619 6620 INIT_LIST_HEAD(&device_list); 6621 list_add_tail(&adev->reset_list, &device_list); 6622 6623 /* wait for asic to come out of reset */ 6624 msleep(500); 6625 6626 /* Restore PCI confspace */ 6627 amdgpu_device_load_pci_state(pdev); 6628 6629 /* confirm ASIC came out of reset */ 6630 for (i = 0; i < adev->usec_timeout; i++) { 6631 memsize = amdgpu_asic_get_config_memsize(adev); 6632 6633 if (memsize != 0xffffffff) 6634 break; 6635 udelay(1); 6636 } 6637 if (memsize == 0xffffffff) { 6638 r = -ETIME; 6639 goto out; 6640 } 6641 6642 reset_context.method = AMD_RESET_METHOD_NONE; 6643 reset_context.reset_req_dev = adev; 6644 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 6645 set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags); 6646 6647 adev->no_hw_access = true; 6648 r = amdgpu_device_pre_asic_reset(adev, &reset_context); 6649 adev->no_hw_access = false; 6650 if (r) 6651 goto out; 6652 6653 r = amdgpu_do_asic_reset(&device_list, &reset_context); 6654 6655 out: 6656 if (!r) { 6657 if (amdgpu_device_cache_pci_state(adev->pdev)) 6658 pci_restore_state(adev->pdev); 6659 6660 DRM_INFO("PCIe error recovery succeeded\n"); 6661 } else { 6662 DRM_ERROR("PCIe error recovery failed, err:%d", r); 6663 amdgpu_device_unset_mp1_state(adev); 6664 amdgpu_device_unlock_reset_domain(adev->reset_domain); 6665 } 6666 6667 return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED; 6668 } 6669 6670 /** 6671 * amdgpu_pci_resume() - resume normal ops after PCI reset 6672 * @pdev: pointer to PCI device 6673 * 6674 * Called when the error recovery driver tells us that its 6675 * OK to resume normal operation. 6676 */ 6677 void amdgpu_pci_resume(struct pci_dev *pdev) 6678 { 6679 struct drm_device *dev = pci_get_drvdata(pdev); 6680 struct amdgpu_device *adev = drm_to_adev(dev); 6681 int i; 6682 6683 6684 DRM_INFO("PCI error: resume callback!!\n"); 6685 6686 /* Only continue execution for the case of pci_channel_io_frozen */ 6687 if (adev->pci_channel_state != pci_channel_io_frozen) 6688 return; 6689 6690 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 6691 struct amdgpu_ring *ring = adev->rings[i]; 6692 6693 if (!amdgpu_ring_sched_ready(ring)) 6694 continue; 6695 6696 drm_sched_start(&ring->sched, 0); 6697 } 6698 6699 amdgpu_device_unset_mp1_state(adev); 6700 amdgpu_device_unlock_reset_domain(adev->reset_domain); 6701 } 6702 6703 bool amdgpu_device_cache_pci_state(struct pci_dev *pdev) 6704 { 6705 struct drm_device *dev = pci_get_drvdata(pdev); 6706 struct amdgpu_device *adev = drm_to_adev(dev); 6707 int r; 6708 6709 if (amdgpu_sriov_vf(adev)) 6710 return false; 6711 6712 r = pci_save_state(pdev); 6713 if (!r) { 6714 kfree(adev->pci_state); 6715 6716 adev->pci_state = pci_store_saved_state(pdev); 6717 6718 if (!adev->pci_state) { 6719 DRM_ERROR("Failed to store PCI saved state"); 6720 return false; 6721 } 6722 } else { 6723 DRM_WARN("Failed to save PCI state, err:%d\n", r); 6724 return false; 6725 } 6726 6727 return true; 6728 } 6729 6730 bool amdgpu_device_load_pci_state(struct pci_dev *pdev) 6731 { 6732 struct drm_device *dev = pci_get_drvdata(pdev); 6733 struct amdgpu_device *adev = drm_to_adev(dev); 6734 int r; 6735 6736 if (!adev->pci_state) 6737 return false; 6738 6739 r = pci_load_saved_state(pdev, adev->pci_state); 6740 6741 if (!r) { 6742 pci_restore_state(pdev); 6743 } else { 6744 DRM_WARN("Failed to load PCI state, err:%d\n", r); 6745 return false; 6746 } 6747 6748 return true; 6749 } 6750 6751 void amdgpu_device_flush_hdp(struct amdgpu_device *adev, 6752 struct amdgpu_ring *ring) 6753 { 6754 #ifdef CONFIG_X86_64 6755 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) 6756 return; 6757 #endif 6758 if (adev->gmc.xgmi.connected_to_cpu) 6759 return; 6760 6761 if (ring && ring->funcs->emit_hdp_flush) 6762 amdgpu_ring_emit_hdp_flush(ring); 6763 else 6764 amdgpu_asic_flush_hdp(adev, ring); 6765 } 6766 6767 void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev, 6768 struct amdgpu_ring *ring) 6769 { 6770 #ifdef CONFIG_X86_64 6771 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) 6772 return; 6773 #endif 6774 if (adev->gmc.xgmi.connected_to_cpu) 6775 return; 6776 6777 amdgpu_asic_invalidate_hdp(adev, ring); 6778 } 6779 6780 int amdgpu_in_reset(struct amdgpu_device *adev) 6781 { 6782 return atomic_read(&adev->reset_domain->in_gpu_reset); 6783 } 6784 6785 /** 6786 * amdgpu_device_halt() - bring hardware to some kind of halt state 6787 * 6788 * @adev: amdgpu_device pointer 6789 * 6790 * Bring hardware to some kind of halt state so that no one can touch it 6791 * any more. It will help to maintain error context when error occurred. 6792 * Compare to a simple hang, the system will keep stable at least for SSH 6793 * access. Then it should be trivial to inspect the hardware state and 6794 * see what's going on. Implemented as following: 6795 * 6796 * 1. drm_dev_unplug() makes device inaccessible to user space(IOCTLs, etc), 6797 * clears all CPU mappings to device, disallows remappings through page faults 6798 * 2. amdgpu_irq_disable_all() disables all interrupts 6799 * 3. amdgpu_fence_driver_hw_fini() signals all HW fences 6800 * 4. set adev->no_hw_access to avoid potential crashes after setp 5 6801 * 5. amdgpu_device_unmap_mmio() clears all MMIO mappings 6802 * 6. pci_disable_device() and pci_wait_for_pending_transaction() 6803 * flush any in flight DMA operations 6804 */ 6805 void amdgpu_device_halt(struct amdgpu_device *adev) 6806 { 6807 struct pci_dev *pdev = adev->pdev; 6808 struct drm_device *ddev = adev_to_drm(adev); 6809 6810 amdgpu_xcp_dev_unplug(adev); 6811 drm_dev_unplug(ddev); 6812 6813 amdgpu_irq_disable_all(adev); 6814 6815 amdgpu_fence_driver_hw_fini(adev); 6816 6817 adev->no_hw_access = true; 6818 6819 amdgpu_device_unmap_mmio(adev); 6820 6821 pci_disable_device(pdev); 6822 pci_wait_for_pending_transaction(pdev); 6823 } 6824 6825 u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev, 6826 u32 reg) 6827 { 6828 unsigned long flags, address, data; 6829 u32 r; 6830 6831 address = adev->nbio.funcs->get_pcie_port_index_offset(adev); 6832 data = adev->nbio.funcs->get_pcie_port_data_offset(adev); 6833 6834 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 6835 WREG32(address, reg * 4); 6836 (void)RREG32(address); 6837 r = RREG32(data); 6838 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 6839 return r; 6840 } 6841 6842 void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev, 6843 u32 reg, u32 v) 6844 { 6845 unsigned long flags, address, data; 6846 6847 address = adev->nbio.funcs->get_pcie_port_index_offset(adev); 6848 data = adev->nbio.funcs->get_pcie_port_data_offset(adev); 6849 6850 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 6851 WREG32(address, reg * 4); 6852 (void)RREG32(address); 6853 WREG32(data, v); 6854 (void)RREG32(data); 6855 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 6856 } 6857 6858 /** 6859 * amdgpu_device_get_gang - return a reference to the current gang 6860 * @adev: amdgpu_device pointer 6861 * 6862 * Returns: A new reference to the current gang leader. 6863 */ 6864 struct dma_fence *amdgpu_device_get_gang(struct amdgpu_device *adev) 6865 { 6866 struct dma_fence *fence; 6867 6868 rcu_read_lock(); 6869 fence = dma_fence_get_rcu_safe(&adev->gang_submit); 6870 rcu_read_unlock(); 6871 return fence; 6872 } 6873 6874 /** 6875 * amdgpu_device_switch_gang - switch to a new gang 6876 * @adev: amdgpu_device pointer 6877 * @gang: the gang to switch to 6878 * 6879 * Try to switch to a new gang. 6880 * Returns: NULL if we switched to the new gang or a reference to the current 6881 * gang leader. 6882 */ 6883 struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev, 6884 struct dma_fence *gang) 6885 { 6886 struct dma_fence *old = NULL; 6887 6888 do { 6889 dma_fence_put(old); 6890 old = amdgpu_device_get_gang(adev); 6891 if (old == gang) 6892 break; 6893 6894 if (!dma_fence_is_signaled(old)) 6895 return old; 6896 6897 } while (cmpxchg((struct dma_fence __force **)&adev->gang_submit, 6898 old, gang) != old); 6899 6900 dma_fence_put(old); 6901 return NULL; 6902 } 6903 6904 bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev) 6905 { 6906 switch (adev->asic_type) { 6907 #ifdef CONFIG_DRM_AMDGPU_SI 6908 case CHIP_HAINAN: 6909 #endif 6910 case CHIP_TOPAZ: 6911 /* chips with no display hardware */ 6912 return false; 6913 #ifdef CONFIG_DRM_AMDGPU_SI 6914 case CHIP_TAHITI: 6915 case CHIP_PITCAIRN: 6916 case CHIP_VERDE: 6917 case CHIP_OLAND: 6918 #endif 6919 #ifdef CONFIG_DRM_AMDGPU_CIK 6920 case CHIP_BONAIRE: 6921 case CHIP_HAWAII: 6922 case CHIP_KAVERI: 6923 case CHIP_KABINI: 6924 case CHIP_MULLINS: 6925 #endif 6926 case CHIP_TONGA: 6927 case CHIP_FIJI: 6928 case CHIP_POLARIS10: 6929 case CHIP_POLARIS11: 6930 case CHIP_POLARIS12: 6931 case CHIP_VEGAM: 6932 case CHIP_CARRIZO: 6933 case CHIP_STONEY: 6934 /* chips with display hardware */ 6935 return true; 6936 default: 6937 /* IP discovery */ 6938 if (!amdgpu_ip_version(adev, DCE_HWIP, 0) || 6939 (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK)) 6940 return false; 6941 return true; 6942 } 6943 } 6944 6945 uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev, 6946 uint32_t inst, uint32_t reg_addr, char reg_name[], 6947 uint32_t expected_value, uint32_t mask) 6948 { 6949 uint32_t ret = 0; 6950 uint32_t old_ = 0; 6951 uint32_t tmp_ = RREG32(reg_addr); 6952 uint32_t loop = adev->usec_timeout; 6953 6954 while ((tmp_ & (mask)) != (expected_value)) { 6955 if (old_ != tmp_) { 6956 loop = adev->usec_timeout; 6957 old_ = tmp_; 6958 } else 6959 udelay(1); 6960 tmp_ = RREG32(reg_addr); 6961 loop--; 6962 if (!loop) { 6963 DRM_WARN("Register(%d) [%s] failed to reach value 0x%08x != 0x%08xn", 6964 inst, reg_name, (uint32_t)expected_value, 6965 (uint32_t)(tmp_ & (mask))); 6966 ret = -ETIMEDOUT; 6967 break; 6968 } 6969 } 6970 return ret; 6971 } 6972 6973 ssize_t amdgpu_get_soft_full_reset_mask(struct amdgpu_ring *ring) 6974 { 6975 ssize_t size = 0; 6976 6977 if (!ring || !ring->adev) 6978 return size; 6979 6980 if (amdgpu_device_should_recover_gpu(ring->adev)) 6981 size |= AMDGPU_RESET_TYPE_FULL; 6982 6983 if (unlikely(!ring->adev->debug_disable_soft_recovery) && 6984 !amdgpu_sriov_vf(ring->adev) && ring->funcs->soft_recovery) 6985 size |= AMDGPU_RESET_TYPE_SOFT_RESET; 6986 6987 return size; 6988 } 6989 6990 ssize_t amdgpu_show_reset_mask(char *buf, uint32_t supported_reset) 6991 { 6992 ssize_t size = 0; 6993 6994 if (supported_reset == 0) { 6995 size += sysfs_emit_at(buf, size, "unsupported"); 6996 size += sysfs_emit_at(buf, size, "\n"); 6997 return size; 6998 6999 } 7000 7001 if (supported_reset & AMDGPU_RESET_TYPE_SOFT_RESET) 7002 size += sysfs_emit_at(buf, size, "soft "); 7003 7004 if (supported_reset & AMDGPU_RESET_TYPE_PER_QUEUE) 7005 size += sysfs_emit_at(buf, size, "queue "); 7006 7007 if (supported_reset & AMDGPU_RESET_TYPE_PER_PIPE) 7008 size += sysfs_emit_at(buf, size, "pipe "); 7009 7010 if (supported_reset & AMDGPU_RESET_TYPE_FULL) 7011 size += sysfs_emit_at(buf, size, "full "); 7012 7013 size += sysfs_emit_at(buf, size, "\n"); 7014 return size; 7015 } 7016