1 /* 2 * Copyright 2008 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: Dave Airlie 25 * Alex Deucher 26 * Jerome Glisse 27 */ 28 29 #include <linux/aperture.h> 30 #include <linux/power_supply.h> 31 #include <linux/kthread.h> 32 #include <linux/module.h> 33 #include <linux/console.h> 34 #include <linux/slab.h> 35 #include <linux/iommu.h> 36 #include <linux/pci.h> 37 #include <linux/pci-p2pdma.h> 38 #include <linux/apple-gmux.h> 39 #include <linux/nospec.h> 40 41 #include <drm/drm_atomic_helper.h> 42 #include <drm/drm_client_event.h> 43 #include <drm/drm_crtc_helper.h> 44 #include <drm/drm_probe_helper.h> 45 #include <drm/amdgpu_drm.h> 46 #include <linux/device.h> 47 #include <linux/vgaarb.h> 48 #include <linux/vga_switcheroo.h> 49 #include <linux/efi.h> 50 #include "amdgpu.h" 51 #include "amdgpu_trace.h" 52 #include "amdgpu_i2c.h" 53 #include "atom.h" 54 #include "amdgpu_atombios.h" 55 #include "amdgpu_atomfirmware.h" 56 #include "amd_pcie.h" 57 #ifdef CONFIG_DRM_AMDGPU_SI 58 #include "si.h" 59 #endif 60 #ifdef CONFIG_DRM_AMDGPU_CIK 61 #include "cik.h" 62 #endif 63 #include "vi.h" 64 #include "soc15.h" 65 #include "nv.h" 66 #include "bif/bif_4_1_d.h" 67 #include <linux/firmware.h> 68 #include "amdgpu_vf_error.h" 69 70 #include "amdgpu_amdkfd.h" 71 #include "amdgpu_ip.h" 72 #include "amdgpu_pm.h" 73 74 #include "amdgpu_xgmi.h" 75 #include "amdgpu_ras.h" 76 #include "amdgpu_wb.h" 77 #include "amdgpu_ras_mgr.h" 78 #include "amdgpu_pmu.h" 79 #include "amdgpu_smu.h" 80 #include "amdgpu_fru_eeprom.h" 81 #include "amdgpu_reset.h" 82 #include "amdgpu_virt.h" 83 #include "amdgpu_dev_coredump.h" 84 85 #include <linux/suspend.h> 86 #include <drm/task_barrier.h> 87 #include <linux/pm_runtime.h> 88 89 #include <drm/drm_drv.h> 90 91 #if IS_ENABLED(CONFIG_X86) 92 #include <asm/intel-family.h> 93 #include <asm/cpu_device_id.h> 94 #endif 95 96 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin"); 97 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin"); 98 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin"); 99 MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin"); 100 MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin"); 101 MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin"); 102 MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin"); 103 MODULE_FIRMWARE("amdgpu/cyan_skillfish_gpu_info.bin"); 104 105 #define AMDGPU_RESUME_MS 2000 106 #define AMDGPU_MAX_RETRY_LIMIT 2 107 #define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL) 108 #define AMDGPU_PCIE_INDEX_FALLBACK (0x38 >> 2) 109 #define AMDGPU_PCIE_INDEX_HI_FALLBACK (0x44 >> 2) 110 #define AMDGPU_PCIE_DATA_FALLBACK (0x3C >> 2) 111 112 #define AMDGPU_VBIOS_SKIP (1U << 0) 113 #define AMDGPU_VBIOS_OPTIONAL (1U << 1) 114 115 static const struct drm_driver amdgpu_kms_driver; 116 117 const char *amdgpu_asic_name[] = { 118 "TAHITI", 119 "PITCAIRN", 120 "VERDE", 121 "OLAND", 122 "HAINAN", 123 "BONAIRE", 124 "KAVERI", 125 "KABINI", 126 "HAWAII", 127 "MULLINS", 128 "TOPAZ", 129 "TONGA", 130 "FIJI", 131 "CARRIZO", 132 "STONEY", 133 "POLARIS10", 134 "POLARIS11", 135 "POLARIS12", 136 "VEGAM", 137 "VEGA10", 138 "VEGA12", 139 "VEGA20", 140 "RAVEN", 141 "ARCTURUS", 142 "RENOIR", 143 "ALDEBARAN", 144 "NAVI10", 145 "CYAN_SKILLFISH", 146 "NAVI14", 147 "NAVI12", 148 "SIENNA_CICHLID", 149 "NAVY_FLOUNDER", 150 "VANGOGH", 151 "DIMGREY_CAVEFISH", 152 "BEIGE_GOBY", 153 "YELLOW_CARP", 154 "IP DISCOVERY", 155 "LAST", 156 }; 157 158 #define AMDGPU_IP_BLK_MASK_ALL GENMASK(AMD_IP_BLOCK_TYPE_NUM - 1, 0) 159 /* 160 * Default init level where all blocks are expected to be initialized. This is 161 * the level of initialization expected by default and also after a full reset 162 * of the device. 163 */ 164 struct amdgpu_init_level amdgpu_init_default = { 165 .level = AMDGPU_INIT_LEVEL_DEFAULT, 166 .hwini_ip_block_mask = AMDGPU_IP_BLK_MASK_ALL, 167 }; 168 169 struct amdgpu_init_level amdgpu_init_recovery = { 170 .level = AMDGPU_INIT_LEVEL_RESET_RECOVERY, 171 .hwini_ip_block_mask = AMDGPU_IP_BLK_MASK_ALL, 172 }; 173 174 /* 175 * Minimal blocks needed to be initialized before a XGMI hive can be reset. This 176 * is used for cases like reset on initialization where the entire hive needs to 177 * be reset before first use. 178 */ 179 struct amdgpu_init_level amdgpu_init_minimal_xgmi = { 180 .level = AMDGPU_INIT_LEVEL_MINIMAL_XGMI, 181 .hwini_ip_block_mask = 182 BIT(AMD_IP_BLOCK_TYPE_GMC) | BIT(AMD_IP_BLOCK_TYPE_SMC) | 183 BIT(AMD_IP_BLOCK_TYPE_COMMON) | BIT(AMD_IP_BLOCK_TYPE_IH) | 184 BIT(AMD_IP_BLOCK_TYPE_PSP) 185 }; 186 187 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev); 188 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev); 189 static int amdgpu_device_ip_resume_phase3(struct amdgpu_device *adev); 190 191 static void amdgpu_device_load_switch_state(struct amdgpu_device *adev); 192 193 static inline bool amdgpu_ip_member_of_hwini(struct amdgpu_device *adev, 194 enum amd_ip_block_type block) 195 { 196 return (adev->init_lvl->hwini_ip_block_mask & (1U << block)) != 0; 197 } 198 199 void amdgpu_set_init_level(struct amdgpu_device *adev, 200 enum amdgpu_init_lvl_id lvl) 201 { 202 switch (lvl) { 203 case AMDGPU_INIT_LEVEL_MINIMAL_XGMI: 204 adev->init_lvl = &amdgpu_init_minimal_xgmi; 205 break; 206 case AMDGPU_INIT_LEVEL_RESET_RECOVERY: 207 adev->init_lvl = &amdgpu_init_recovery; 208 break; 209 case AMDGPU_INIT_LEVEL_DEFAULT: 210 fallthrough; 211 default: 212 adev->init_lvl = &amdgpu_init_default; 213 break; 214 } 215 } 216 217 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev); 218 static int amdgpu_device_pm_notifier(struct notifier_block *nb, unsigned long mode, 219 void *data); 220 221 /** 222 * DOC: pcie_replay_count 223 * 224 * The amdgpu driver provides a sysfs API for reporting the total number 225 * of PCIe replays (NAKs). 226 * The file pcie_replay_count is used for this and returns the total 227 * number of replays as a sum of the NAKs generated and NAKs received. 228 */ 229 230 static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev, 231 struct device_attribute *attr, char *buf) 232 { 233 struct drm_device *ddev = dev_get_drvdata(dev); 234 struct amdgpu_device *adev = drm_to_adev(ddev); 235 uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev); 236 237 return sysfs_emit(buf, "%llu\n", cnt); 238 } 239 240 static DEVICE_ATTR(pcie_replay_count, 0444, 241 amdgpu_device_get_pcie_replay_count, NULL); 242 243 static int amdgpu_device_attr_sysfs_init(struct amdgpu_device *adev) 244 { 245 int ret = 0; 246 247 if (amdgpu_nbio_is_replay_cnt_supported(adev)) 248 ret = sysfs_create_file(&adev->dev->kobj, 249 &dev_attr_pcie_replay_count.attr); 250 251 if (!ret) 252 ret = amdgpu_discovery_mem_reserved_info_sysfs_init(adev); 253 254 return ret; 255 } 256 257 static void amdgpu_device_attr_sysfs_fini(struct amdgpu_device *adev) 258 { 259 if (amdgpu_nbio_is_replay_cnt_supported(adev)) 260 sysfs_remove_file(&adev->dev->kobj, 261 &dev_attr_pcie_replay_count.attr); 262 263 amdgpu_discovery_mem_reserved_info_sysfs_fini(adev); 264 } 265 266 static ssize_t amdgpu_sysfs_reg_state_get(struct file *f, struct kobject *kobj, 267 const struct bin_attribute *attr, char *buf, 268 loff_t ppos, size_t count) 269 { 270 struct device *dev = kobj_to_dev(kobj); 271 struct drm_device *ddev = dev_get_drvdata(dev); 272 struct amdgpu_device *adev = drm_to_adev(ddev); 273 ssize_t bytes_read; 274 275 switch (ppos) { 276 case AMDGPU_SYS_REG_STATE_XGMI: 277 bytes_read = amdgpu_asic_get_reg_state( 278 adev, AMDGPU_REG_STATE_TYPE_XGMI, buf, count); 279 break; 280 case AMDGPU_SYS_REG_STATE_WAFL: 281 bytes_read = amdgpu_asic_get_reg_state( 282 adev, AMDGPU_REG_STATE_TYPE_WAFL, buf, count); 283 break; 284 case AMDGPU_SYS_REG_STATE_PCIE: 285 bytes_read = amdgpu_asic_get_reg_state( 286 adev, AMDGPU_REG_STATE_TYPE_PCIE, buf, count); 287 break; 288 case AMDGPU_SYS_REG_STATE_USR: 289 bytes_read = amdgpu_asic_get_reg_state( 290 adev, AMDGPU_REG_STATE_TYPE_USR, buf, count); 291 break; 292 case AMDGPU_SYS_REG_STATE_USR_1: 293 bytes_read = amdgpu_asic_get_reg_state( 294 adev, AMDGPU_REG_STATE_TYPE_USR_1, buf, count); 295 break; 296 default: 297 return -EINVAL; 298 } 299 300 return bytes_read; 301 } 302 303 static const BIN_ATTR(reg_state, 0444, amdgpu_sysfs_reg_state_get, NULL, 304 AMDGPU_SYS_REG_STATE_END); 305 306 int amdgpu_reg_state_sysfs_init(struct amdgpu_device *adev) 307 { 308 int ret; 309 310 if (!amdgpu_asic_get_reg_state_supported(adev)) 311 return 0; 312 313 ret = sysfs_create_bin_file(&adev->dev->kobj, &bin_attr_reg_state); 314 315 return ret; 316 } 317 318 void amdgpu_reg_state_sysfs_fini(struct amdgpu_device *adev) 319 { 320 if (!amdgpu_asic_get_reg_state_supported(adev)) 321 return; 322 sysfs_remove_bin_file(&adev->dev->kobj, &bin_attr_reg_state); 323 } 324 325 /** 326 * DOC: board_info 327 * 328 * The amdgpu driver provides a sysfs API for giving board related information. 329 * It provides the form factor information in the format 330 * 331 * type : form factor 332 * 333 * Possible form factor values 334 * 335 * - "cem" - PCIE CEM card 336 * - "oam" - Open Compute Accelerator Module 337 * - "eam" - Enhanced Accelerator Module 338 * - "unknown" - Not known 339 * 340 */ 341 342 static ssize_t amdgpu_device_get_board_info(struct device *dev, 343 struct device_attribute *attr, 344 char *buf) 345 { 346 struct drm_device *ddev = dev_get_drvdata(dev); 347 struct amdgpu_device *adev = drm_to_adev(ddev); 348 enum amdgpu_pkg_type pkg_type = AMDGPU_PKG_TYPE_CEM; 349 const char *pkg; 350 351 if (adev->smuio.funcs && adev->smuio.funcs->get_pkg_type) 352 pkg_type = adev->smuio.funcs->get_pkg_type(adev); 353 354 switch (pkg_type) { 355 case AMDGPU_PKG_TYPE_CEM: 356 pkg = "cem"; 357 break; 358 case AMDGPU_PKG_TYPE_OAM: 359 pkg = "oam"; 360 break; 361 case AMDGPU_PKG_TYPE_BB: 362 pkg = "eam"; 363 break; 364 default: 365 pkg = "unknown"; 366 break; 367 } 368 369 return sysfs_emit(buf, "%s : %s\n", "type", pkg); 370 } 371 372 static DEVICE_ATTR(board_info, 0444, amdgpu_device_get_board_info, NULL); 373 374 static struct attribute *amdgpu_board_attrs[] = { 375 &dev_attr_board_info.attr, 376 NULL, 377 }; 378 379 static umode_t amdgpu_board_attrs_is_visible(struct kobject *kobj, 380 struct attribute *attr, int n) 381 { 382 struct device *dev = kobj_to_dev(kobj); 383 struct drm_device *ddev = dev_get_drvdata(dev); 384 struct amdgpu_device *adev = drm_to_adev(ddev); 385 386 if (adev->flags & AMD_IS_APU) 387 return 0; 388 389 return attr->mode; 390 } 391 392 static const struct attribute_group amdgpu_board_attrs_group = { 393 .attrs = amdgpu_board_attrs, 394 .is_visible = amdgpu_board_attrs_is_visible 395 }; 396 397 /** 398 * DOC: uma/carveout_options 399 * 400 * This is a read-only file that lists all available UMA allocation 401 * options and their corresponding indices. Example output:: 402 * 403 * $ cat uma/carveout_options 404 * 0: Minimum (512 MB) 405 * 1: (1 GB) 406 * 2: (2 GB) 407 * 3: (4 GB) 408 * 4: (6 GB) 409 * 5: (8 GB) 410 * 6: (12 GB) 411 * 7: Medium (16 GB) 412 * 8: (24 GB) 413 * 9: High (32 GB) 414 */ 415 static ssize_t carveout_options_show(struct device *dev, 416 struct device_attribute *attr, 417 char *buf) 418 { 419 struct drm_device *ddev = dev_get_drvdata(dev); 420 struct amdgpu_device *adev = drm_to_adev(ddev); 421 struct amdgpu_uma_carveout_info *uma_info = &adev->uma_info; 422 uint32_t memory_carved; 423 ssize_t size = 0; 424 425 if (!uma_info || !uma_info->num_entries) 426 return -ENODEV; 427 428 for (int i = 0; i < uma_info->num_entries; i++) { 429 memory_carved = uma_info->entries[i].memory_carved_mb; 430 if (memory_carved >= SZ_1G/SZ_1M) { 431 size += sysfs_emit_at(buf, size, "%d: %s (%u GB)\n", 432 i, 433 uma_info->entries[i].name, 434 memory_carved >> 10); 435 } else { 436 size += sysfs_emit_at(buf, size, "%d: %s (%u MB)\n", 437 i, 438 uma_info->entries[i].name, 439 memory_carved); 440 } 441 } 442 443 return size; 444 } 445 static DEVICE_ATTR_RO(carveout_options); 446 447 /** 448 * DOC: uma/carveout 449 * 450 * This file is both readable and writable. When read, it shows the 451 * index of the current setting. Writing a valid index to this file 452 * allows users to change the UMA carveout size to the selected option 453 * on the next boot. 454 * 455 * The available options and their corresponding indices can be read 456 * from the uma/carveout_options file. 457 */ 458 static ssize_t carveout_show(struct device *dev, 459 struct device_attribute *attr, 460 char *buf) 461 { 462 struct drm_device *ddev = dev_get_drvdata(dev); 463 struct amdgpu_device *adev = drm_to_adev(ddev); 464 465 return sysfs_emit(buf, "%u\n", adev->uma_info.uma_option_index); 466 } 467 468 static ssize_t carveout_store(struct device *dev, 469 struct device_attribute *attr, 470 const char *buf, size_t count) 471 { 472 struct drm_device *ddev = dev_get_drvdata(dev); 473 struct amdgpu_device *adev = drm_to_adev(ddev); 474 struct amdgpu_uma_carveout_info *uma_info = &adev->uma_info; 475 struct amdgpu_uma_carveout_option *opt; 476 unsigned long val; 477 uint8_t flags; 478 int r; 479 480 r = kstrtoul(buf, 10, &val); 481 if (r) 482 return r; 483 484 if (val >= uma_info->num_entries) 485 return -EINVAL; 486 487 val = array_index_nospec(val, uma_info->num_entries); 488 opt = &uma_info->entries[val]; 489 490 if (!(opt->flags & AMDGPU_UMA_FLAG_AUTO) && 491 !(opt->flags & AMDGPU_UMA_FLAG_CUSTOM)) { 492 drm_err_once(ddev, "Option %lu not supported due to lack of Custom/Auto flag", val); 493 return -EINVAL; 494 } 495 496 flags = opt->flags; 497 flags &= ~((flags & AMDGPU_UMA_FLAG_AUTO) >> 1); 498 499 guard(mutex)(&uma_info->update_lock); 500 501 r = amdgpu_acpi_set_uma_allocation_size(adev, val, flags); 502 if (r) 503 return r; 504 505 uma_info->uma_option_index = val; 506 507 return count; 508 } 509 static DEVICE_ATTR_RW(carveout); 510 511 static struct attribute *amdgpu_uma_attrs[] = { 512 &dev_attr_carveout.attr, 513 &dev_attr_carveout_options.attr, 514 NULL 515 }; 516 517 const struct attribute_group amdgpu_uma_attr_group = { 518 .name = "uma", 519 .attrs = amdgpu_uma_attrs 520 }; 521 522 static void amdgpu_uma_sysfs_init(struct amdgpu_device *adev) 523 { 524 int rc; 525 526 if (!(adev->flags & AMD_IS_APU)) 527 return; 528 529 if (!amdgpu_acpi_is_set_uma_allocation_size_supported()) 530 return; 531 532 rc = amdgpu_atomfirmware_get_uma_carveout_info(adev, &adev->uma_info); 533 if (rc) { 534 drm_dbg(adev_to_drm(adev), 535 "Failed to parse UMA carveout info from VBIOS: %d\n", rc); 536 goto out_info; 537 } 538 539 mutex_init(&adev->uma_info.update_lock); 540 541 rc = devm_device_add_group(adev->dev, &amdgpu_uma_attr_group); 542 if (rc) { 543 drm_dbg(adev_to_drm(adev), "Failed to add UMA carveout sysfs interfaces %d\n", rc); 544 goto out_attr; 545 } 546 547 return; 548 549 out_attr: 550 mutex_destroy(&adev->uma_info.update_lock); 551 out_info: 552 return; 553 } 554 555 static void amdgpu_uma_sysfs_fini(struct amdgpu_device *adev) 556 { 557 struct amdgpu_uma_carveout_info *uma_info = &adev->uma_info; 558 559 if (!amdgpu_acpi_is_set_uma_allocation_size_supported()) 560 return; 561 562 mutex_destroy(&uma_info->update_lock); 563 uma_info->num_entries = 0; 564 } 565 566 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev); 567 568 /** 569 * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control 570 * 571 * @adev: amdgpu device pointer 572 * 573 * Returns true if the device is a dGPU with ATPX power control, 574 * otherwise return false. 575 */ 576 bool amdgpu_device_supports_px(struct amdgpu_device *adev) 577 { 578 if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid()) 579 return true; 580 return false; 581 } 582 583 /** 584 * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources 585 * 586 * @adev: amdgpu device pointer 587 * 588 * Returns true if the device is a dGPU with ACPI power control, 589 * otherwise return false. 590 */ 591 bool amdgpu_device_supports_boco(struct amdgpu_device *adev) 592 { 593 if (!IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE)) 594 return false; 595 596 /* BOCO needs a platform power resource (_PR3) to cut the power */ 597 return adev->has_pr3; 598 } 599 600 /** 601 * amdgpu_device_supports_baco - Does the device support BACO 602 * 603 * @adev: amdgpu device pointer 604 * 605 * Return: 606 * 1 if the device supports BACO; 607 * 3 if the device supports MACO (only works if BACO is supported) 608 * otherwise return 0. 609 */ 610 int amdgpu_device_supports_baco(struct amdgpu_device *adev) 611 { 612 return amdgpu_asic_supports_baco(adev); 613 } 614 615 void amdgpu_device_detect_runtime_pm_mode(struct amdgpu_device *adev) 616 { 617 int bamaco_support; 618 619 adev->pm.rpm_mode = AMDGPU_RUNPM_NONE; 620 if (pci_is_thunderbolt_attached(adev->pdev) || 621 dev_is_removable(&adev->pdev->dev)) { 622 dev_info(adev->dev, 623 "Runtime PM disabled for externally attached device\n"); 624 return; 625 } 626 627 bamaco_support = amdgpu_device_supports_baco(adev); 628 629 switch (amdgpu_runtime_pm) { 630 case 2: 631 if (bamaco_support & MACO_SUPPORT) { 632 adev->pm.rpm_mode = AMDGPU_RUNPM_BAMACO; 633 dev_info(adev->dev, "Forcing BAMACO for runtime pm\n"); 634 } else if (bamaco_support == BACO_SUPPORT) { 635 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO; 636 dev_info(adev->dev, "Requested mode BAMACO not available,fallback to use BACO\n"); 637 } 638 break; 639 case 1: 640 if (bamaco_support & BACO_SUPPORT) { 641 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO; 642 dev_info(adev->dev, "Forcing BACO for runtime pm\n"); 643 } 644 break; 645 case -1: 646 case -2: 647 if (amdgpu_device_supports_px(adev)) { 648 /* enable PX as runtime mode */ 649 adev->pm.rpm_mode = AMDGPU_RUNPM_PX; 650 dev_info(adev->dev, "Using ATPX for runtime pm\n"); 651 } else if (amdgpu_device_supports_boco(adev)) { 652 /* enable boco as runtime mode */ 653 adev->pm.rpm_mode = AMDGPU_RUNPM_BOCO; 654 dev_info(adev->dev, "Using BOCO for runtime pm\n"); 655 } else { 656 if (!bamaco_support) 657 goto no_runtime_pm; 658 659 switch (adev->asic_type) { 660 case CHIP_VEGA20: 661 case CHIP_ARCTURUS: 662 /* BACO are not supported on vega20 and arctrus */ 663 break; 664 case CHIP_VEGA10: 665 /* enable BACO as runpm mode if noretry=0 */ 666 if (!adev->gmc.noretry && !amdgpu_passthrough(adev)) 667 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO; 668 break; 669 default: 670 /* enable BACO as runpm mode on CI+ */ 671 if (!amdgpu_passthrough(adev)) 672 adev->pm.rpm_mode = AMDGPU_RUNPM_BACO; 673 break; 674 } 675 676 if (adev->pm.rpm_mode == AMDGPU_RUNPM_BACO) { 677 if (bamaco_support & MACO_SUPPORT) { 678 adev->pm.rpm_mode = AMDGPU_RUNPM_BAMACO; 679 dev_info(adev->dev, "Using BAMACO for runtime pm\n"); 680 } else { 681 dev_info(adev->dev, "Using BACO for runtime pm\n"); 682 } 683 } 684 } 685 break; 686 case 0: 687 dev_info(adev->dev, "runtime pm is manually disabled\n"); 688 break; 689 default: 690 break; 691 } 692 693 no_runtime_pm: 694 if (adev->pm.rpm_mode == AMDGPU_RUNPM_NONE) 695 dev_info(adev->dev, "Runtime PM not available\n"); 696 } 697 /** 698 * amdgpu_device_supports_smart_shift - Is the device dGPU with 699 * smart shift support 700 * 701 * @adev: amdgpu device pointer 702 * 703 * Returns true if the device is a dGPU with Smart Shift support, 704 * otherwise returns false. 705 */ 706 bool amdgpu_device_supports_smart_shift(struct amdgpu_device *adev) 707 { 708 return (amdgpu_device_supports_boco(adev) && 709 amdgpu_acpi_is_power_shift_control_supported()); 710 } 711 712 /* 713 * VRAM access helper functions 714 */ 715 716 /** 717 * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA 718 * 719 * @adev: amdgpu_device pointer 720 * @pos: offset of the buffer in vram 721 * @buf: virtual address of the buffer in system memory 722 * @size: read/write size, sizeof(@buf) must > @size 723 * @write: true - write to vram, otherwise - read from vram 724 */ 725 void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos, 726 void *buf, size_t size, bool write) 727 { 728 unsigned long flags; 729 uint32_t hi = ~0, tmp = 0; 730 uint32_t *data = buf; 731 uint64_t last; 732 int idx; 733 734 if (!drm_dev_enter(adev_to_drm(adev), &idx)) 735 return; 736 737 if (!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4)) { 738 dev_err(adev->dev, "unaligned pos/size (pos=0x%llx, size=0x%zx)\n", 739 pos, size); 740 drm_dev_exit(idx); 741 return; 742 } 743 744 spin_lock_irqsave(&adev->mmio_idx_lock, flags); 745 for (last = pos + size; pos < last; pos += 4) { 746 tmp = pos >> 31; 747 748 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000); 749 if (tmp != hi) { 750 WREG32_NO_KIQ(mmMM_INDEX_HI, tmp); 751 hi = tmp; 752 } 753 if (write) 754 WREG32_NO_KIQ(mmMM_DATA, *data++); 755 else 756 *data++ = RREG32_NO_KIQ(mmMM_DATA); 757 } 758 759 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); 760 drm_dev_exit(idx); 761 } 762 763 #ifdef CONFIG_64BIT 764 /* 765 * During early SR-IOV VF init, host-provided init data can live in FB before 766 * the normal VRAM aperture mapping is ready. Use a temporary BAR0 mapping for 767 * reads only, and verify it matches the VRAM aperture when aperture information 768 * is already available. 769 */ 770 static int amdgpu_device_read_fb_via_bar0(struct amdgpu_device *adev, 771 u64 offset, void *buf, size_t size) 772 { 773 resource_size_t aper_base, aper_size, bar_start, bar_size, map_base; 774 void __iomem *vram; 775 size_t map_offset, map_size; 776 unsigned long flags; 777 u64 end; 778 779 if (!buf || !size) 780 return -EINVAL; 781 782 if (!amdgpu_sriov_vf(adev)) 783 return -EINVAL; 784 785 flags = pci_resource_flags(adev->pdev, 0); 786 if ((flags & IORESOURCE_UNSET) || !(flags & IORESOURCE_MEM)) 787 return -EINVAL; 788 789 bar_start = pci_resource_start(adev->pdev, 0); 790 bar_size = pci_resource_len(adev->pdev, 0); 791 if (!bar_size) 792 return -ENODEV; 793 794 aper_base = adev->gmc.aper_base; 795 aper_size = adev->gmc.visible_vram_size ? adev->gmc.visible_vram_size : 796 adev->gmc.aper_size; 797 798 if (aper_base || aper_size) { 799 if (aper_base != bar_start || aper_size > bar_size) 800 return -EINVAL; 801 } else { 802 aper_base = bar_start; 803 aper_size = bar_size; 804 } 805 806 if (check_add_overflow(offset, size, &end) || end > aper_size) 807 return -EINVAL; 808 809 map_offset = offset_in_page(offset); 810 map_base = aper_base + (offset & PAGE_MASK); 811 map_size = PAGE_ALIGN(map_offset + size); 812 813 vram = ioremap_wc(map_base, map_size); 814 if (!vram) 815 return -ENOMEM; 816 817 memcpy_fromio(buf, (u8 __iomem *)vram + map_offset, size); 818 iounmap(vram); 819 820 return 0; 821 } 822 #endif 823 824 /** 825 * amdgpu_device_aper_access - access vram by vram aperture 826 * 827 * @adev: amdgpu_device pointer 828 * @pos: offset of the buffer in vram 829 * @buf: virtual address of the buffer in system memory 830 * @size: read/write size, sizeof(@buf) must > @size 831 * @write: true - write to vram, otherwise - read from vram 832 * 833 * The return value means how many bytes have been transferred. 834 */ 835 size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos, 836 void *buf, size_t size, bool write) 837 { 838 #ifdef CONFIG_64BIT 839 void __iomem *addr; 840 size_t count = 0; 841 uint64_t last; 842 843 if (!adev->mman.aper_base_kaddr) { 844 /* Writes still require the regular aperture/MM path. */ 845 if (!write && !amdgpu_device_read_fb_via_bar0(adev, pos, buf, size)) 846 return size; 847 return 0; 848 } 849 850 last = min(pos + size, adev->gmc.visible_vram_size); 851 if (last > pos) { 852 addr = adev->mman.aper_base_kaddr + pos; 853 count = last - pos; 854 855 if (write) { 856 memcpy_toio(addr, buf, count); 857 /* Make sure HDP write cache flush happens without any reordering 858 * after the system memory contents are sent over PCIe device 859 */ 860 mb(); 861 amdgpu_device_flush_hdp(adev, NULL); 862 } else { 863 amdgpu_device_invalidate_hdp(adev, NULL); 864 /* Make sure HDP read cache is invalidated before issuing a read 865 * to the PCIe device 866 */ 867 mb(); 868 memcpy_fromio(buf, addr, count); 869 } 870 871 } 872 873 return count; 874 #else 875 return 0; 876 #endif 877 } 878 879 /** 880 * amdgpu_device_vram_access - read/write a buffer in vram 881 * 882 * @adev: amdgpu_device pointer 883 * @pos: offset of the buffer in vram 884 * @buf: virtual address of the buffer in system memory 885 * @size: read/write size, sizeof(@buf) must > @size 886 * @write: true - write to vram, otherwise - read from vram 887 */ 888 void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos, 889 void *buf, size_t size, bool write) 890 { 891 size_t count; 892 893 /* try to using vram apreature to access vram first */ 894 count = amdgpu_device_aper_access(adev, pos, buf, size, write); 895 size -= count; 896 if (size) { 897 /* using MM to access rest vram */ 898 pos += count; 899 buf += count; 900 amdgpu_device_mm_access(adev, pos, buf, size, write); 901 } 902 } 903 904 /* 905 * register access helper functions. 906 */ 907 908 /* Check if hw access should be skipped because of hotplug or device error */ 909 bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev) 910 { 911 if (adev->no_hw_access) 912 return true; 913 914 #ifdef CONFIG_LOCKDEP 915 /* 916 * This is a bit complicated to understand, so worth a comment. What we assert 917 * here is that the GPU reset is not running on another thread in parallel. 918 * 919 * For this we trylock the read side of the reset semaphore, if that succeeds 920 * we know that the reset is not running in parallel. 921 * 922 * If the trylock fails we assert that we are either already holding the read 923 * side of the lock or are the reset thread itself and hold the write side of 924 * the lock. 925 */ 926 if (in_task()) { 927 if (down_read_trylock(&adev->reset_domain->sem)) 928 up_read(&adev->reset_domain->sem); 929 else 930 lockdep_assert_held(&adev->reset_domain->sem); 931 } 932 #endif 933 return false; 934 } 935 936 /** 937 * amdgpu_device_get_rev_id - query device rev_id 938 * 939 * @adev: amdgpu_device pointer 940 * 941 * Return device rev_id 942 */ 943 u32 amdgpu_device_get_rev_id(struct amdgpu_device *adev) 944 { 945 return adev->nbio.funcs->get_rev_id(adev); 946 } 947 948 static uint32_t amdgpu_device_get_vbios_flags(struct amdgpu_device *adev) 949 { 950 if (amdgpu_virt_vram_is_spm(adev)) 951 return AMDGPU_VBIOS_OPTIONAL; 952 953 if (hweight32(adev->aid_mask) && (adev->flags & AMD_IS_APU)) 954 return AMDGPU_VBIOS_SKIP; 955 956 if (hweight32(adev->aid_mask) && amdgpu_passthrough(adev)) 957 return AMDGPU_VBIOS_OPTIONAL; 958 959 return 0; 960 } 961 962 /** 963 * amdgpu_device_asic_init - Wrapper for atom asic_init 964 * 965 * @adev: amdgpu_device pointer 966 * 967 * Does any asic specific work and then calls atom asic init. 968 */ 969 static int amdgpu_device_asic_init(struct amdgpu_device *adev) 970 { 971 uint32_t flags; 972 bool optional; 973 int ret; 974 975 amdgpu_asic_pre_asic_init(adev); 976 flags = amdgpu_device_get_vbios_flags(adev); 977 optional = !!(flags & (AMDGPU_VBIOS_OPTIONAL | AMDGPU_VBIOS_SKIP)); 978 979 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) || 980 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4) || 981 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0) || 982 amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0)) { 983 amdgpu_psp_wait_for_bootloader(adev); 984 if (optional && !adev->bios) 985 return 0; 986 987 ret = amdgpu_atomfirmware_asic_init(adev, true); 988 return ret; 989 } else { 990 if (optional && !adev->bios) 991 return 0; 992 993 return amdgpu_atom_asic_init(adev->mode_info.atom_context); 994 } 995 996 return 0; 997 } 998 999 /** 1000 * amdgpu_device_mem_scratch_init - allocate the VRAM scratch page 1001 * 1002 * @adev: amdgpu_device pointer 1003 * 1004 * Allocates a scratch page of VRAM for use by various things in the 1005 * driver. 1006 */ 1007 static int amdgpu_device_mem_scratch_init(struct amdgpu_device *adev) 1008 { 1009 return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, PAGE_SIZE, 1010 AMDGPU_GEM_DOMAIN_VRAM | 1011 AMDGPU_GEM_DOMAIN_GTT, 1012 &adev->mem_scratch.robj, 1013 &adev->mem_scratch.gpu_addr, 1014 (void **)&adev->mem_scratch.ptr); 1015 } 1016 1017 /** 1018 * amdgpu_device_mem_scratch_fini - Free the VRAM scratch page 1019 * 1020 * @adev: amdgpu_device pointer 1021 * 1022 * Frees the VRAM scratch page. 1023 */ 1024 static void amdgpu_device_mem_scratch_fini(struct amdgpu_device *adev) 1025 { 1026 amdgpu_bo_free_kernel(&adev->mem_scratch.robj, NULL, NULL); 1027 } 1028 1029 /** 1030 * amdgpu_device_program_register_sequence - program an array of registers. 1031 * 1032 * @adev: amdgpu_device pointer 1033 * @registers: pointer to the register array 1034 * @array_size: size of the register array 1035 * 1036 * Programs an array or registers with and or masks. 1037 * This is a helper for setting golden registers. 1038 */ 1039 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev, 1040 const u32 *registers, 1041 const u32 array_size) 1042 { 1043 u32 tmp, reg, and_mask, or_mask; 1044 int i; 1045 1046 if (array_size % 3) 1047 return; 1048 1049 for (i = 0; i < array_size; i += 3) { 1050 reg = registers[i + 0]; 1051 and_mask = registers[i + 1]; 1052 or_mask = registers[i + 2]; 1053 1054 if (and_mask == 0xffffffff) { 1055 tmp = or_mask; 1056 } else { 1057 tmp = RREG32(reg); 1058 tmp &= ~and_mask; 1059 if (adev->family >= AMDGPU_FAMILY_AI) 1060 tmp |= (or_mask & and_mask); 1061 else 1062 tmp |= or_mask; 1063 } 1064 WREG32(reg, tmp); 1065 } 1066 } 1067 1068 /** 1069 * amdgpu_device_pci_config_reset - reset the GPU 1070 * 1071 * @adev: amdgpu_device pointer 1072 * 1073 * Resets the GPU using the pci config reset sequence. 1074 * Only applicable to asics prior to vega10. 1075 */ 1076 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev) 1077 { 1078 pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA); 1079 } 1080 1081 /** 1082 * amdgpu_device_pci_reset - reset the GPU using generic PCI means 1083 * 1084 * @adev: amdgpu_device pointer 1085 * 1086 * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.). 1087 */ 1088 int amdgpu_device_pci_reset(struct amdgpu_device *adev) 1089 { 1090 return pci_reset_function(adev->pdev); 1091 } 1092 1093 /** 1094 * amdgpu_device_resize_fb_bar - try to resize FB BAR 1095 * 1096 * @adev: amdgpu_device pointer 1097 * 1098 * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not 1099 * to fail, but if any of the BARs is not accessible after the size we abort 1100 * driver loading by returning -ENODEV. 1101 */ 1102 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev) 1103 { 1104 int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size); 1105 struct pci_bus *root; 1106 struct resource *res; 1107 int max_size, r; 1108 unsigned int i; 1109 u16 cmd; 1110 1111 if (!IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT)) 1112 return 0; 1113 1114 /* Bypass for VF */ 1115 if (amdgpu_sriov_vf(adev)) 1116 return 0; 1117 1118 if (!amdgpu_rebar) 1119 return 0; 1120 1121 /* resizing on Dell G5 SE platforms causes problems with runtime pm */ 1122 if ((amdgpu_runtime_pm != 0) && 1123 adev->pdev->vendor == PCI_VENDOR_ID_ATI && 1124 adev->pdev->device == 0x731f && 1125 adev->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL) 1126 return 0; 1127 1128 /* PCI_EXT_CAP_ID_VNDR extended capability is located at 0x100 */ 1129 if (!pci_find_ext_capability(adev->pdev, PCI_EXT_CAP_ID_VNDR)) 1130 dev_warn( 1131 adev->dev, 1132 "System can't access extended configuration space, please check!!\n"); 1133 1134 /* skip if the bios has already enabled large BAR */ 1135 if (adev->gmc.real_vram_size && 1136 (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size)) 1137 return 0; 1138 1139 /* Check if the root BUS has 64bit memory resources */ 1140 root = adev->pdev->bus; 1141 while (root->parent) 1142 root = root->parent; 1143 1144 pci_bus_for_each_resource(root, res, i) { 1145 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) && 1146 res->start > 0x100000000ull) 1147 break; 1148 } 1149 1150 /* Trying to resize is pointless without a root hub window above 4GB */ 1151 if (!res) 1152 return 0; 1153 1154 /* Limit the BAR size to what is available */ 1155 max_size = pci_rebar_get_max_size(adev->pdev, 0); 1156 if (max_size < 0) 1157 return 0; 1158 rbar_size = min(max_size, rbar_size); 1159 1160 /* Disable memory decoding while we change the BAR addresses and size */ 1161 pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd); 1162 pci_write_config_word(adev->pdev, PCI_COMMAND, 1163 cmd & ~PCI_COMMAND_MEMORY); 1164 1165 /* Tear down doorbell as resizing will release BARs */ 1166 amdgpu_doorbell_fini(adev); 1167 1168 r = pci_resize_resource(adev->pdev, 0, rbar_size, 1169 (adev->asic_type >= CHIP_BONAIRE) ? 1 << 5 1170 : 1 << 2); 1171 if (r == -ENOSPC) 1172 dev_info(adev->dev, 1173 "Not enough PCI address space for a large BAR."); 1174 else if (r && r != -ENOTSUPP) 1175 dev_err(adev->dev, "Problem resizing BAR0 (%d).", r); 1176 1177 /* When the doorbell or fb BAR isn't available we have no chance of 1178 * using the device. 1179 */ 1180 r = amdgpu_doorbell_init(adev); 1181 if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET)) 1182 return -ENODEV; 1183 1184 pci_write_config_word(adev->pdev, PCI_COMMAND, cmd); 1185 1186 return 0; 1187 } 1188 1189 /* 1190 * GPU helpers function. 1191 */ 1192 /** 1193 * amdgpu_device_need_post - check if the hw need post or not 1194 * 1195 * @adev: amdgpu_device pointer 1196 * 1197 * Check if the asic has been initialized (all asics) at driver startup 1198 * or post is needed if hw reset is performed. 1199 * Returns true if need or false if not. 1200 */ 1201 bool amdgpu_device_need_post(struct amdgpu_device *adev) 1202 { 1203 uint32_t reg, flags; 1204 1205 if (amdgpu_sriov_vf(adev)) 1206 return false; 1207 1208 flags = amdgpu_device_get_vbios_flags(adev); 1209 if (flags & AMDGPU_VBIOS_SKIP) 1210 return false; 1211 if ((flags & AMDGPU_VBIOS_OPTIONAL) && !adev->bios) 1212 return false; 1213 1214 if (amdgpu_passthrough(adev)) { 1215 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot 1216 * some old smc fw still need driver do vPost otherwise gpu hang, while 1217 * those smc fw version above 22.15 doesn't have this flaw, so we force 1218 * vpost executed for smc version below 22.15 1219 */ 1220 if (adev->asic_type == CHIP_FIJI) { 1221 int err; 1222 uint32_t fw_ver; 1223 1224 err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev); 1225 /* force vPost if error occurred */ 1226 if (err) 1227 return true; 1228 1229 fw_ver = *((uint32_t *)adev->pm.fw->data + 69); 1230 release_firmware(adev->pm.fw); 1231 if (fw_ver < 0x00160e00) 1232 return true; 1233 } 1234 } 1235 1236 /* Don't post if we need to reset whole hive on init */ 1237 if (adev->init_lvl->level == AMDGPU_INIT_LEVEL_MINIMAL_XGMI) 1238 return false; 1239 1240 if (adev->has_hw_reset) { 1241 adev->has_hw_reset = false; 1242 return true; 1243 } 1244 1245 /* bios scratch used on CIK+ */ 1246 if (adev->asic_type >= CHIP_BONAIRE) 1247 return amdgpu_atombios_scratch_need_asic_init(adev); 1248 1249 /* check MEM_SIZE for older asics */ 1250 reg = amdgpu_asic_get_config_memsize(adev); 1251 1252 if ((reg != 0) && (reg != 0xffffffff)) 1253 return false; 1254 1255 return true; 1256 } 1257 1258 /* 1259 * Check whether seamless boot is supported. 1260 * 1261 * So far we only support seamless boot on DCE 3.0 or later. 1262 * If users report that it works on older ASICS as well, we may 1263 * loosen this. 1264 */ 1265 bool amdgpu_device_seamless_boot_supported(struct amdgpu_device *adev) 1266 { 1267 switch (amdgpu_seamless) { 1268 case -1: 1269 break; 1270 case 1: 1271 return true; 1272 case 0: 1273 return false; 1274 default: 1275 dev_err(adev->dev, "Invalid value for amdgpu.seamless: %d\n", 1276 amdgpu_seamless); 1277 return false; 1278 } 1279 1280 if (!(adev->flags & AMD_IS_APU)) 1281 return false; 1282 1283 if (adev->mman.keep_stolen_vga_memory) 1284 return false; 1285 1286 return amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(3, 0, 0); 1287 } 1288 1289 #if IS_ENABLED(CONFIG_X86) 1290 static const struct x86_cpu_id amdgpu_pcie_dynamic_switching_quirks[] = { 1291 /* 1292 * Intel hosts such as Rocket Lake, Alder Lake, Raptor Lake and Sapphire Rapids 1293 * don't support dynamic speed switching. Until we have confirmation from Intel 1294 * that a specific host supports it, it's safer that we keep it disabled for all. 1295 * 1296 * https://edc.intel.com/content/www/us/en/design/products/platforms/details/raptor-lake-s/13th-generation-core-processors-datasheet-volume-1-of-2/005/pci-express-support/ 1297 * https://gitlab.freedesktop.org/drm/amd/-/issues/2663 1298 */ 1299 X86_MATCH_VENDOR_FAM(INTEL, X86_FAMILY_ANY, NULL), 1300 /* 1301 * AMD Ryzen Pinnacle Ridge (Zen+, family 0x17 model 0x08) CPUs don't 1302 * support PCIe dynamic speed switching. 1303 * https://gitlab.freedesktop.org/drm/amd/-/work_items/5436 1304 */ 1305 X86_MATCH_VENDOR_FAM_MODEL(AMD, 0x17, 0x08, NULL), 1306 {} 1307 }; 1308 1309 static bool amdgpu_device_pcie_dynamic_switching_supported(struct amdgpu_device *adev) 1310 { 1311 /* eGPU change speeds based on USB4 fabric conditions */ 1312 if (dev_is_removable(adev->dev)) 1313 return true; 1314 1315 /* Hosts have problems with dynamic speed switching */ 1316 if (x86_match_cpu(amdgpu_pcie_dynamic_switching_quirks)) 1317 return false; 1318 1319 return true; 1320 } 1321 #else 1322 static inline bool amdgpu_device_pcie_dynamic_switching_supported(struct amdgpu_device *adev) 1323 { 1324 return true; 1325 } 1326 #endif 1327 1328 static bool amdgpu_device_aspm_support_quirk(struct amdgpu_device *adev) 1329 { 1330 /* Enabling ASPM causes randoms hangs on Tahiti and Oland on Zen4. 1331 * It's unclear if this is a platform-specific or GPU-specific issue. 1332 * Disable ASPM on SI for the time being. 1333 */ 1334 if (adev->family == AMDGPU_FAMILY_SI || 1335 (!(adev->pm.pp_feature & PP_PCIE_DPM_MASK) && adev->family == AMDGPU_FAMILY_VI)) 1336 return true; 1337 1338 #if IS_ENABLED(CONFIG_X86) 1339 struct cpuinfo_x86 *c = &cpu_data(0); 1340 1341 if (c->x86_vendor == X86_VENDOR_INTEL) { 1342 switch (c->x86_model) { 1343 case VFM_MODEL(INTEL_ALDERLAKE): 1344 case VFM_MODEL(INTEL_ALDERLAKE_L): 1345 case VFM_MODEL(INTEL_RAPTORLAKE): 1346 case VFM_MODEL(INTEL_RAPTORLAKE_P): 1347 case VFM_MODEL(INTEL_RAPTORLAKE_S): 1348 case VFM_MODEL(INTEL_TIGERLAKE): 1349 case VFM_MODEL(INTEL_TIGERLAKE_L): 1350 return true; 1351 default: 1352 return false; 1353 } 1354 } else { 1355 return false; 1356 } 1357 #else 1358 return false; 1359 #endif 1360 } 1361 1362 /** 1363 * amdgpu_device_should_use_aspm - check if the device should program ASPM 1364 * 1365 * @adev: amdgpu_device pointer 1366 * 1367 * Confirm whether the module parameter and pcie bridge agree that ASPM should 1368 * be set for this device. 1369 * 1370 * Returns true if it should be used or false if not. 1371 */ 1372 bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev) 1373 { 1374 struct pci_dev *aspm_pdev, *parent; 1375 bool enabled; 1376 1377 switch (amdgpu_aspm) { 1378 case -1: 1379 break; 1380 case 0: 1381 return false; 1382 case 1: 1383 return true; 1384 default: 1385 return false; 1386 } 1387 if (adev->flags & AMD_IS_APU) 1388 return false; 1389 if (amdgpu_device_aspm_support_quirk(adev)) 1390 return false; 1391 1392 /* 1393 * pcie_aspm_enabled() checks the link between its argument and 1394 * the immediate upstream bridge. Use SWUS for dGPUs with an 1395 * internal switch so that this is the host-facing link. 1396 */ 1397 aspm_pdev = adev->link_dev; 1398 parent = pci_upstream_bridge(aspm_pdev); 1399 if (!parent) { 1400 dev_dbg(adev->dev, "ASPM: no upstream PCIe link for %s\n", 1401 pci_name(aspm_pdev)); 1402 return false; 1403 } 1404 1405 enabled = pcie_aspm_enabled(aspm_pdev); 1406 /* Report the exact link used for the automatic ASPM decision. */ 1407 dev_dbg(adev->dev, "ASPM: link %s <-> %s is %s\n", 1408 pci_name(parent), pci_name(aspm_pdev), 1409 enabled ? "enabled" : "disabled"); 1410 1411 return enabled; 1412 } 1413 1414 /* if we get transitioned to only one device, take VGA back */ 1415 /** 1416 * amdgpu_device_vga_set_decode - enable/disable vga decode 1417 * 1418 * @data: driver private data (adev) 1419 * @state: enable/disable vga decode 1420 * 1421 * Enable/disable vga decode (all asics). 1422 * Returns VGA resource flags. 1423 */ 1424 static unsigned int amdgpu_device_vga_set_decode(void *data, bool state) 1425 { 1426 struct amdgpu_device *adev = data; 1427 1428 amdgpu_asic_set_vga_state(adev, state); 1429 if (state) 1430 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM | 1431 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 1432 else 1433 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 1434 } 1435 1436 /** 1437 * amdgpu_device_check_block_size - validate the vm block size 1438 * 1439 * @adev: amdgpu_device pointer 1440 * 1441 * Validates the vm block size specified via module parameter. 1442 * The vm block size defines number of bits in page table versus page directory, 1443 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the 1444 * page table and the remaining bits are in the page directory. 1445 */ 1446 static void amdgpu_device_check_block_size(struct amdgpu_device *adev) 1447 { 1448 /* defines number of bits in page table versus page directory, 1449 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the 1450 * page table and the remaining bits are in the page directory 1451 */ 1452 if (amdgpu_vm_block_size == -1) 1453 return; 1454 1455 if (amdgpu_vm_block_size < 9) { 1456 dev_warn(adev->dev, "VM page table size (%d) too small\n", 1457 amdgpu_vm_block_size); 1458 amdgpu_vm_block_size = -1; 1459 } 1460 } 1461 1462 /** 1463 * amdgpu_device_check_vm_size - validate the vm size 1464 * 1465 * @adev: amdgpu_device pointer 1466 * 1467 * Validates the vm size in GB specified via module parameter. 1468 * The VM size is the size of the GPU virtual memory space in GB. 1469 */ 1470 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev) 1471 { 1472 /* no need to check the default value */ 1473 if (amdgpu_vm_size == -1) 1474 return; 1475 1476 if (amdgpu_vm_size < 1) { 1477 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n", 1478 amdgpu_vm_size); 1479 amdgpu_vm_size = -1; 1480 } 1481 } 1482 1483 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev) 1484 { 1485 struct sysinfo si; 1486 bool is_os_64 = (sizeof(void *) == 8); 1487 uint64_t total_memory; 1488 uint64_t dram_size_seven_GB = 0x1B8000000; 1489 uint64_t dram_size_three_GB = 0xB8000000; 1490 1491 if (amdgpu_smu_memory_pool_size == 0) 1492 return; 1493 1494 if (!is_os_64) { 1495 dev_warn(adev->dev, "Not 64-bit OS, feature not supported\n"); 1496 goto def_value; 1497 } 1498 si_meminfo(&si); 1499 total_memory = (uint64_t)si.totalram * si.mem_unit; 1500 1501 if ((amdgpu_smu_memory_pool_size == 1) || 1502 (amdgpu_smu_memory_pool_size == 2)) { 1503 if (total_memory < dram_size_three_GB) 1504 goto def_value1; 1505 } else if ((amdgpu_smu_memory_pool_size == 4) || 1506 (amdgpu_smu_memory_pool_size == 8)) { 1507 if (total_memory < dram_size_seven_GB) 1508 goto def_value1; 1509 } else { 1510 dev_warn(adev->dev, "Smu memory pool size not supported\n"); 1511 goto def_value; 1512 } 1513 adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28; 1514 1515 return; 1516 1517 def_value1: 1518 dev_warn(adev->dev, "No enough system memory\n"); 1519 def_value: 1520 adev->pm.smu_prv_buffer_size = 0; 1521 } 1522 1523 static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev) 1524 { 1525 if (!(adev->flags & AMD_IS_APU) || 1526 adev->asic_type < CHIP_RAVEN) 1527 return 0; 1528 1529 switch (adev->asic_type) { 1530 case CHIP_RAVEN: 1531 if (adev->pdev->device == 0x15dd) 1532 adev->apu_flags |= AMD_APU_IS_RAVEN; 1533 if (adev->pdev->device == 0x15d8) 1534 adev->apu_flags |= AMD_APU_IS_PICASSO; 1535 break; 1536 case CHIP_RENOIR: 1537 if ((adev->pdev->device == 0x1636) || 1538 (adev->pdev->device == 0x164c)) 1539 adev->apu_flags |= AMD_APU_IS_RENOIR; 1540 else 1541 adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE; 1542 break; 1543 case CHIP_VANGOGH: 1544 adev->apu_flags |= AMD_APU_IS_VANGOGH; 1545 break; 1546 case CHIP_YELLOW_CARP: 1547 break; 1548 case CHIP_CYAN_SKILLFISH: 1549 if ((adev->pdev->device == 0x13FE) || 1550 (adev->pdev->device == 0x143F)) 1551 adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2; 1552 break; 1553 default: 1554 break; 1555 } 1556 1557 return 0; 1558 } 1559 1560 /** 1561 * amdgpu_device_check_arguments - validate module params 1562 * 1563 * @adev: amdgpu_device pointer 1564 * 1565 * Validates certain module parameters and updates 1566 * the associated values used by the driver (all asics). 1567 */ 1568 static int amdgpu_device_check_arguments(struct amdgpu_device *adev) 1569 { 1570 int i; 1571 1572 if (amdgpu_sched_jobs < 4) { 1573 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n", 1574 amdgpu_sched_jobs); 1575 amdgpu_sched_jobs = 4; 1576 } else if (!is_power_of_2(amdgpu_sched_jobs)) { 1577 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n", 1578 amdgpu_sched_jobs); 1579 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs); 1580 } 1581 1582 if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) { 1583 /* gart size must be greater or equal to 32M */ 1584 dev_warn(adev->dev, "gart size (%d) too small\n", 1585 amdgpu_gart_size); 1586 amdgpu_gart_size = -1; 1587 } 1588 1589 if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) { 1590 /* gtt size must be greater or equal to 32M */ 1591 dev_warn(adev->dev, "gtt size (%d) too small\n", 1592 amdgpu_gtt_size); 1593 amdgpu_gtt_size = -1; 1594 } 1595 1596 /* valid range is between 4 and 9 inclusive */ 1597 if (amdgpu_vm_fragment_size != -1 && 1598 (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) { 1599 dev_warn(adev->dev, "valid range is between 4 and 9\n"); 1600 amdgpu_vm_fragment_size = -1; 1601 } 1602 1603 if (amdgpu_sched_hw_submission < 2) { 1604 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n", 1605 amdgpu_sched_hw_submission); 1606 amdgpu_sched_hw_submission = 2; 1607 } else if (!is_power_of_2(amdgpu_sched_hw_submission)) { 1608 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n", 1609 amdgpu_sched_hw_submission); 1610 amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission); 1611 } 1612 1613 if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) { 1614 dev_warn(adev->dev, "invalid option for reset method, reverting to default\n"); 1615 amdgpu_reset_method = -1; 1616 } 1617 1618 amdgpu_device_check_smu_prv_buffer_size(adev); 1619 1620 amdgpu_device_check_vm_size(adev); 1621 1622 amdgpu_device_check_block_size(adev); 1623 1624 adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type); 1625 1626 for (i = 0; i < MAX_XCP; i++) { 1627 switch (amdgpu_enforce_isolation) { 1628 case -1: 1629 case 0: 1630 default: 1631 /* disable */ 1632 adev->enforce_isolation[i] = AMDGPU_ENFORCE_ISOLATION_DISABLE; 1633 break; 1634 case 1: 1635 /* enable */ 1636 adev->enforce_isolation[i] = 1637 AMDGPU_ENFORCE_ISOLATION_ENABLE; 1638 break; 1639 case 2: 1640 /* enable legacy mode */ 1641 adev->enforce_isolation[i] = 1642 AMDGPU_ENFORCE_ISOLATION_ENABLE_LEGACY; 1643 break; 1644 case 3: 1645 /* enable only process isolation without submitting cleaner shader */ 1646 adev->enforce_isolation[i] = 1647 AMDGPU_ENFORCE_ISOLATION_NO_CLEANER_SHADER; 1648 break; 1649 } 1650 } 1651 1652 return 0; 1653 } 1654 1655 /** 1656 * amdgpu_switcheroo_set_state - set switcheroo state 1657 * 1658 * @pdev: pci dev pointer 1659 * @state: vga_switcheroo state 1660 * 1661 * Callback for the switcheroo driver. Suspends or resumes 1662 * the asics before or after it is powered up using ACPI methods. 1663 */ 1664 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, 1665 enum vga_switcheroo_state state) 1666 { 1667 struct drm_device *dev = pci_get_drvdata(pdev); 1668 int r; 1669 1670 if (amdgpu_device_supports_px(drm_to_adev(dev)) && 1671 state == VGA_SWITCHEROO_OFF) 1672 return; 1673 1674 if (state == VGA_SWITCHEROO_ON) { 1675 pr_info("switched on\n"); 1676 /* don't suspend or resume card normally */ 1677 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; 1678 1679 pci_set_power_state(pdev, PCI_D0); 1680 amdgpu_device_load_pci_state(pdev); 1681 r = pci_enable_device(pdev); 1682 if (r) 1683 dev_warn(&pdev->dev, "pci_enable_device failed (%d)\n", 1684 r); 1685 amdgpu_device_resume(dev, true); 1686 1687 dev->switch_power_state = DRM_SWITCH_POWER_ON; 1688 } else { 1689 dev_info(&pdev->dev, "switched off\n"); 1690 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; 1691 amdgpu_device_prepare(dev); 1692 amdgpu_device_suspend(dev, true); 1693 amdgpu_device_cache_pci_state(pdev); 1694 /* Shut down the device */ 1695 pci_disable_device(pdev); 1696 pci_set_power_state(pdev, PCI_D3cold); 1697 dev->switch_power_state = DRM_SWITCH_POWER_OFF; 1698 } 1699 } 1700 1701 /** 1702 * amdgpu_switcheroo_can_switch - see if switcheroo state can change 1703 * 1704 * @pdev: pci dev pointer 1705 * 1706 * Callback for the switcheroo driver. Check of the switcheroo 1707 * state can be changed. 1708 * Returns true if the state can be changed, false if not. 1709 */ 1710 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev) 1711 { 1712 struct drm_device *dev = pci_get_drvdata(pdev); 1713 1714 /* 1715 * FIXME: open_count is protected by drm_global_mutex but that would lead to 1716 * locking inversion with the driver load path. And the access here is 1717 * completely racy anyway. So don't bother with locking for now. 1718 */ 1719 return atomic_read(&dev->open_count) == 0; 1720 } 1721 1722 static void amdgpu_switcheroo_pre_switch(struct pci_dev *pdev) 1723 { 1724 struct drm_device *dev = pci_get_drvdata(pdev); 1725 1726 drm_client_dev_acquire_outputs(dev); 1727 } 1728 1729 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = { 1730 .set_gpu_state = amdgpu_switcheroo_set_state, 1731 .can_switch = amdgpu_switcheroo_can_switch, 1732 .pre_switch = amdgpu_switcheroo_pre_switch, 1733 }; 1734 1735 /** 1736 * amdgpu_device_enable_virtual_display - enable virtual display feature 1737 * 1738 * @adev: amdgpu_device pointer 1739 * 1740 * Enabled the virtual display feature if the user has enabled it via 1741 * the module parameter virtual_display. This feature provides a virtual 1742 * display hardware on headless boards or in virtualized environments. 1743 * This function parses and validates the configuration string specified by 1744 * the user and configures the virtual display configuration (number of 1745 * virtual connectors, crtcs, etc.) specified. 1746 */ 1747 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev) 1748 { 1749 adev->enable_virtual_display = false; 1750 1751 if (amdgpu_virtual_display) { 1752 const char *pci_address_name = pci_name(adev->pdev); 1753 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname; 1754 1755 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL); 1756 pciaddstr_tmp = pciaddstr; 1757 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) { 1758 pciaddname = strsep(&pciaddname_tmp, ","); 1759 if (!strcmp("all", pciaddname) 1760 || !strcmp(pci_address_name, pciaddname)) { 1761 long num_crtc; 1762 int res = -1; 1763 1764 adev->enable_virtual_display = true; 1765 1766 if (pciaddname_tmp) 1767 res = kstrtol(pciaddname_tmp, 10, 1768 &num_crtc); 1769 1770 if (!res) { 1771 if (num_crtc < 1) 1772 num_crtc = 1; 1773 if (num_crtc > 6) 1774 num_crtc = 6; 1775 adev->mode_info.num_crtc = num_crtc; 1776 } else { 1777 adev->mode_info.num_crtc = 1; 1778 } 1779 break; 1780 } 1781 } 1782 1783 dev_info( 1784 adev->dev, 1785 "virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n", 1786 amdgpu_virtual_display, pci_address_name, 1787 adev->enable_virtual_display, adev->mode_info.num_crtc); 1788 1789 kfree(pciaddstr); 1790 } 1791 } 1792 1793 void amdgpu_device_set_sriov_virtual_display(struct amdgpu_device *adev) 1794 { 1795 if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) { 1796 adev->mode_info.num_crtc = 1; 1797 adev->enable_virtual_display = true; 1798 dev_info(adev->dev, "virtual_display:%d, num_crtc:%d\n", 1799 adev->enable_virtual_display, 1800 adev->mode_info.num_crtc); 1801 } 1802 } 1803 1804 /** 1805 * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware 1806 * 1807 * @adev: amdgpu_device pointer 1808 * 1809 * Parses the asic configuration parameters specified in the gpu info 1810 * firmware and makes them available to the driver for use in configuring 1811 * the asic. 1812 * Returns 0 on success, -EINVAL on failure. 1813 */ 1814 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev) 1815 { 1816 const char *chip_name; 1817 int err; 1818 const struct gpu_info_firmware_header_v1_0 *hdr; 1819 1820 adev->firmware.gpu_info_fw = NULL; 1821 1822 switch (adev->asic_type) { 1823 default: 1824 return 0; 1825 case CHIP_VEGA10: 1826 chip_name = "vega10"; 1827 break; 1828 case CHIP_VEGA12: 1829 chip_name = "vega12"; 1830 break; 1831 case CHIP_RAVEN: 1832 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 1833 chip_name = "raven2"; 1834 else if (adev->apu_flags & AMD_APU_IS_PICASSO) 1835 chip_name = "picasso"; 1836 else 1837 chip_name = "raven"; 1838 break; 1839 case CHIP_ARCTURUS: 1840 chip_name = "arcturus"; 1841 break; 1842 case CHIP_NAVI12: 1843 if (adev->discovery.bin) 1844 return 0; 1845 chip_name = "navi12"; 1846 break; 1847 case CHIP_CYAN_SKILLFISH: 1848 if (adev->discovery.bin) 1849 return 0; 1850 chip_name = "cyan_skillfish"; 1851 break; 1852 } 1853 1854 err = amdgpu_ucode_request(adev, &adev->firmware.gpu_info_fw, 1855 AMDGPU_UCODE_OPTIONAL, 1856 "amdgpu/%s_gpu_info.bin", chip_name); 1857 if (err) { 1858 dev_err(adev->dev, 1859 "Failed to get gpu_info firmware \"%s_gpu_info.bin\"\n", 1860 chip_name); 1861 goto out; 1862 } 1863 1864 hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data; 1865 amdgpu_ucode_print_gpu_info_hdr(&hdr->header); 1866 1867 switch (hdr->version_major) { 1868 case 1: 1869 { 1870 const struct gpu_info_firmware_v1_0 *gpu_info_fw = 1871 (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data + 1872 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 1873 1874 /* 1875 * Should be dropped when DAL no longer needs it. 1876 */ 1877 if (adev->asic_type == CHIP_NAVI12) 1878 goto parse_soc_bounding_box; 1879 1880 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se); 1881 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh); 1882 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se); 1883 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se); 1884 adev->gfx.config.max_texture_channel_caches = 1885 le32_to_cpu(gpu_info_fw->gc_num_tccs); 1886 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs); 1887 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds); 1888 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth); 1889 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth); 1890 adev->gfx.config.double_offchip_lds_buf = 1891 le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer); 1892 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size); 1893 adev->gfx.cu_info.max_waves_per_simd = 1894 le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd); 1895 adev->gfx.cu_info.max_scratch_slots_per_cu = 1896 le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu); 1897 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size); 1898 if (hdr->version_minor >= 1) { 1899 const struct gpu_info_firmware_v1_1 *gpu_info_fw = 1900 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data + 1901 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 1902 adev->gfx.config.num_sc_per_sh = 1903 le32_to_cpu(gpu_info_fw->num_sc_per_sh); 1904 adev->gfx.config.num_packer_per_sc = 1905 le32_to_cpu(gpu_info_fw->num_packer_per_sc); 1906 } 1907 1908 parse_soc_bounding_box: 1909 /* 1910 * soc bounding box info is not integrated in disocovery table, 1911 * we always need to parse it from gpu info firmware if needed. 1912 */ 1913 if (hdr->version_minor == 2) { 1914 const struct gpu_info_firmware_v1_2 *gpu_info_fw = 1915 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data + 1916 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 1917 adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box; 1918 } 1919 break; 1920 } 1921 default: 1922 dev_err(adev->dev, 1923 "Unsupported gpu_info table %d\n", hdr->header.ucode_version); 1924 err = -EINVAL; 1925 goto out; 1926 } 1927 out: 1928 return err; 1929 } 1930 1931 static void amdgpu_uid_init(struct amdgpu_device *adev) 1932 { 1933 /* Initialize the UID for the device */ 1934 adev->uid_info = kzalloc_obj(struct amdgpu_uid); 1935 if (!adev->uid_info) { 1936 dev_warn(adev->dev, "Failed to allocate memory for UID\n"); 1937 return; 1938 } 1939 adev->uid_info->adev = adev; 1940 } 1941 1942 static void amdgpu_uid_fini(struct amdgpu_device *adev) 1943 { 1944 /* Free the UID memory */ 1945 kfree(adev->uid_info); 1946 adev->uid_info = NULL; 1947 } 1948 1949 static void amdgpu_device_init_pcie_links(struct amdgpu_device *adev) 1950 { 1951 struct pci_dev *swds, *swus; 1952 1953 adev->link_dev = adev->pdev; 1954 adev->link_partner = pci_upstream_bridge(adev->link_dev); 1955 1956 /* Skip an upstream/downstream switch internal to the dGPU. */ 1957 swds = adev->link_partner; 1958 if (!swds || swds->vendor != PCI_VENDOR_ID_ATI || 1959 pci_pcie_type(swds) != PCI_EXP_TYPE_DOWNSTREAM) 1960 return; 1961 1962 swus = pci_upstream_bridge(swds); 1963 if (!swus || 1964 (swus->vendor != PCI_VENDOR_ID_ATI && 1965 swus->vendor != PCI_VENDOR_ID_AMD) || 1966 pci_pcie_type(swus) != PCI_EXP_TYPE_UPSTREAM) 1967 return; 1968 1969 adev->link_dev = swus; 1970 adev->link_partner = pci_upstream_bridge(swus); 1971 } 1972 1973 /** 1974 * amdgpu_device_ip_early_init - run early init for hardware IPs 1975 * 1976 * @adev: amdgpu_device pointer 1977 * 1978 * Early initialization pass for hardware IPs. The hardware IPs that make 1979 * up each asic are discovered each IP's early_init callback is run. This 1980 * is the first stage in initializing the asic. 1981 * Returns 0 on success, negative error code on failure. 1982 */ 1983 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev) 1984 { 1985 struct amdgpu_ip_block *ip_block; 1986 bool total, skip_bios, early_full_gpu_access = false; 1987 uint32_t bios_flags; 1988 int i, r; 1989 1990 amdgpu_device_enable_virtual_display(adev); 1991 1992 if (amdgpu_sriov_vf(adev)) { 1993 /* 1994 * Legacy hosts do not provide init data before early init, so 1995 * keep the original early full GPU access request for them. Newer 1996 * hosts publish the init data through VF FB, which lets us defer 1997 * full GPU access until after non-GPU early init work is done. 1998 */ 1999 early_full_gpu_access = (adev->virt.req_init_data_ver == 0); 2000 if (early_full_gpu_access) { 2001 r = amdgpu_virt_request_full_gpu(adev, true); 2002 if (r) 2003 return r; 2004 } 2005 2006 r = amdgpu_virt_init_critical_region(adev); 2007 if (r) 2008 return r; 2009 } 2010 2011 switch (adev->asic_type) { 2012 #ifdef CONFIG_DRM_AMDGPU_SI 2013 case CHIP_VERDE: 2014 case CHIP_TAHITI: 2015 case CHIP_PITCAIRN: 2016 case CHIP_OLAND: 2017 case CHIP_HAINAN: 2018 adev->family = AMDGPU_FAMILY_SI; 2019 r = si_set_ip_blocks(adev); 2020 if (r) 2021 return r; 2022 break; 2023 #endif 2024 #ifdef CONFIG_DRM_AMDGPU_CIK 2025 case CHIP_BONAIRE: 2026 case CHIP_HAWAII: 2027 case CHIP_KAVERI: 2028 case CHIP_KABINI: 2029 case CHIP_MULLINS: 2030 if (adev->flags & AMD_IS_APU) 2031 adev->family = AMDGPU_FAMILY_KV; 2032 else 2033 adev->family = AMDGPU_FAMILY_CI; 2034 2035 r = cik_set_ip_blocks(adev); 2036 if (r) 2037 return r; 2038 break; 2039 #endif 2040 case CHIP_TOPAZ: 2041 case CHIP_TONGA: 2042 case CHIP_FIJI: 2043 case CHIP_POLARIS10: 2044 case CHIP_POLARIS11: 2045 case CHIP_POLARIS12: 2046 case CHIP_VEGAM: 2047 case CHIP_CARRIZO: 2048 case CHIP_STONEY: 2049 if (adev->flags & AMD_IS_APU) 2050 adev->family = AMDGPU_FAMILY_CZ; 2051 else 2052 adev->family = AMDGPU_FAMILY_VI; 2053 2054 r = vi_set_ip_blocks(adev); 2055 if (r) 2056 return r; 2057 break; 2058 default: 2059 r = amdgpu_discovery_set_ip_blocks(adev); 2060 if (r) { 2061 adev->num_ip_blocks = 0; 2062 return r; 2063 } 2064 break; 2065 } 2066 2067 /* Check for IP version 9.4.3 with A0 hardware */ 2068 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) && 2069 !amdgpu_device_get_rev_id(adev)) { 2070 dev_err(adev->dev, "Unsupported A0 hardware\n"); 2071 return -ENODEV; /* device unsupported - no device error */ 2072 } 2073 2074 if (amdgpu_has_atpx() && 2075 (amdgpu_is_atpx_hybrid() || 2076 amdgpu_has_atpx_dgpu_power_cntl()) && 2077 ((adev->flags & AMD_IS_APU) == 0) && 2078 !dev_is_removable(&adev->pdev->dev)) 2079 adev->flags |= AMD_IS_PX; 2080 2081 if (!(adev->flags & AMD_IS_APU)) { 2082 adev->has_pr3 = adev->link_partner && 2083 pci_pr3_present(adev->link_partner); 2084 2085 if ((amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev)) && 2086 !pci_resource_len(adev->pdev, 0)) 2087 adev->virt.caps |= AMDGPU_VIRT_VRAM_IS_SPM; 2088 } 2089 2090 adev->pm.pp_feature = amdgpu_pp_feature_mask; 2091 if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS) 2092 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 2093 if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID) 2094 adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK; 2095 if (!amdgpu_device_pcie_dynamic_switching_supported(adev)) 2096 adev->pm.pp_feature &= ~PP_PCIE_DPM_MASK; 2097 2098 adev->virt.is_xgmi_node_migrate_enabled = false; 2099 if (amdgpu_sriov_vf(adev)) { 2100 adev->virt.is_xgmi_node_migrate_enabled = 2101 amdgpu_ip_version((adev), GC_HWIP, 0) == IP_VERSION(9, 4, 4); 2102 } 2103 2104 total = true; 2105 for (i = 0; i < adev->num_ip_blocks; i++) { 2106 ip_block = &adev->ip_blocks[i]; 2107 2108 if ((amdgpu_ip_block_mask & (1 << i)) == 0) { 2109 dev_warn(adev->dev, "disabled ip block: %d <%s>\n", i, 2110 adev->ip_blocks[i].version->funcs->name); 2111 adev->ip_blocks[i].status.valid = false; 2112 } else if (ip_block->version->funcs->early_init) { 2113 r = ip_block->version->funcs->early_init(ip_block); 2114 if (r == -ENOENT) { 2115 adev->ip_blocks[i].status.valid = false; 2116 } else if (r) { 2117 dev_err(adev->dev, 2118 "early_init of IP block <%s> failed %d\n", 2119 adev->ip_blocks[i].version->funcs->name, 2120 r); 2121 total = false; 2122 } else { 2123 adev->ip_blocks[i].status.valid = true; 2124 } 2125 } else { 2126 adev->ip_blocks[i].status.valid = true; 2127 } 2128 /* get the vbios after the asic_funcs are set up */ 2129 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) { 2130 r = amdgpu_device_parse_gpu_info_fw(adev); 2131 if (r) 2132 return r; 2133 2134 bios_flags = amdgpu_device_get_vbios_flags(adev); 2135 skip_bios = !!(bios_flags & AMDGPU_VBIOS_SKIP); 2136 /* Read BIOS */ 2137 if (!skip_bios) { 2138 bool optional = 2139 !!(bios_flags & AMDGPU_VBIOS_OPTIONAL); 2140 if (!amdgpu_get_bios(adev) && !optional) 2141 return -EINVAL; 2142 2143 if (optional && !adev->bios) 2144 dev_info( 2145 adev->dev, 2146 "VBIOS image optional, proceeding without VBIOS image"); 2147 2148 if (adev->bios) { 2149 r = amdgpu_atombios_init(adev); 2150 if (r) { 2151 dev_err(adev->dev, 2152 "amdgpu_atombios_init failed\n"); 2153 amdgpu_vf_error_put( 2154 adev, 2155 AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 2156 0, 0); 2157 return r; 2158 } 2159 } 2160 } 2161 2162 /*get pf2vf msg info at it's earliest time*/ 2163 if (amdgpu_sriov_vf(adev)) 2164 amdgpu_virt_init_data_exchange(adev); 2165 2166 } 2167 } 2168 if (!total) 2169 return -ENODEV; 2170 2171 /* Request full GPU access only for the remaining SR-IOV init work. */ 2172 if (amdgpu_sriov_vf(adev) && !early_full_gpu_access) { 2173 r = amdgpu_virt_request_full_gpu(adev, true); 2174 if (r) 2175 return r; 2176 } 2177 2178 if (adev->gmc.xgmi.supported) 2179 amdgpu_xgmi_early_init(adev); 2180 2181 if (amdgpu_is_multi_aid(adev)) 2182 amdgpu_uid_init(adev); 2183 ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX); 2184 if (ip_block->status.valid != false) 2185 amdgpu_amdkfd_device_probe(adev); 2186 2187 adev->cg_flags &= amdgpu_cg_mask; 2188 adev->pg_flags &= amdgpu_pg_mask; 2189 2190 amdgpu_smu_early_init(adev); 2191 2192 return 0; 2193 } 2194 2195 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev) 2196 { 2197 int i, r; 2198 2199 for (i = 0; i < adev->num_ip_blocks; i++) { 2200 if (!adev->ip_blocks[i].status.sw) 2201 continue; 2202 if (adev->ip_blocks[i].status.hw) 2203 continue; 2204 if (!amdgpu_ip_member_of_hwini( 2205 adev, adev->ip_blocks[i].version->type)) 2206 continue; 2207 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || 2208 (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) || 2209 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) { 2210 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]); 2211 if (r) { 2212 dev_err(adev->dev, 2213 "hw_init of IP block <%s> failed %d\n", 2214 adev->ip_blocks[i].version->funcs->name, 2215 r); 2216 return r; 2217 } 2218 adev->ip_blocks[i].status.hw = true; 2219 } 2220 } 2221 2222 return 0; 2223 } 2224 2225 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev) 2226 { 2227 int i, r; 2228 2229 for (i = 0; i < adev->num_ip_blocks; i++) { 2230 if (!adev->ip_blocks[i].status.sw) 2231 continue; 2232 if (adev->ip_blocks[i].status.hw) 2233 continue; 2234 if (!amdgpu_ip_member_of_hwini( 2235 adev, adev->ip_blocks[i].version->type)) 2236 continue; 2237 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]); 2238 if (r) { 2239 dev_err(adev->dev, 2240 "hw_init of IP block <%s> failed %d\n", 2241 adev->ip_blocks[i].version->funcs->name, r); 2242 return r; 2243 } 2244 adev->ip_blocks[i].status.hw = true; 2245 } 2246 2247 return 0; 2248 } 2249 2250 static int amdgpu_device_fw_loading(struct amdgpu_device *adev) 2251 { 2252 int r = 0; 2253 int i; 2254 uint32_t smu_version; 2255 2256 if (adev->asic_type >= CHIP_VEGA10) { 2257 for (i = 0; i < adev->num_ip_blocks; i++) { 2258 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP) 2259 continue; 2260 2261 if (!amdgpu_ip_member_of_hwini(adev, 2262 AMD_IP_BLOCK_TYPE_PSP)) 2263 break; 2264 2265 if (!adev->ip_blocks[i].status.sw) 2266 continue; 2267 2268 /* no need to do the fw loading again if already done*/ 2269 if (adev->ip_blocks[i].status.hw == true) 2270 break; 2271 2272 if (amdgpu_in_reset(adev) || adev->in_suspend) { 2273 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]); 2274 if (r) 2275 return r; 2276 } else { 2277 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]); 2278 if (r) { 2279 dev_err(adev->dev, 2280 "hw_init of IP block <%s> failed %d\n", 2281 adev->ip_blocks[i] 2282 .version->funcs->name, 2283 r); 2284 return r; 2285 } 2286 adev->ip_blocks[i].status.hw = true; 2287 } 2288 break; 2289 } 2290 } 2291 2292 if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA) 2293 r = amdgpu_pm_load_smu_firmware(adev, &smu_version); 2294 2295 return r; 2296 } 2297 2298 static int amdgpu_device_init_schedulers(struct amdgpu_device *adev) 2299 { 2300 struct drm_sched_init_args args = { 2301 .ops = &amdgpu_sched_ops, 2302 .num_rqs = DRM_SCHED_PRIORITY_COUNT, 2303 .timeout_wq = adev->reset_domain->wq, 2304 .dev = adev->dev, 2305 }; 2306 long timeout; 2307 int r, i; 2308 2309 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 2310 struct amdgpu_ring *ring = adev->rings[i]; 2311 2312 /* No need to setup the GPU scheduler for rings that don't need it */ 2313 if (!ring || ring->no_scheduler) 2314 continue; 2315 2316 switch (ring->funcs->type) { 2317 case AMDGPU_RING_TYPE_GFX: 2318 timeout = adev->gfx_timeout; 2319 break; 2320 case AMDGPU_RING_TYPE_COMPUTE: 2321 timeout = adev->compute_timeout; 2322 break; 2323 case AMDGPU_RING_TYPE_SDMA: 2324 timeout = adev->sdma_timeout; 2325 break; 2326 default: 2327 timeout = adev->video_timeout; 2328 break; 2329 } 2330 2331 args.timeout = timeout; 2332 args.credit_limit = ring->num_hw_submission; 2333 args.score = ring->sched_score; 2334 args.name = ring->name; 2335 2336 r = drm_sched_init(&ring->sched, &args); 2337 if (r) { 2338 dev_err(adev->dev, 2339 "Failed to create scheduler on ring %s.\n", 2340 ring->name); 2341 return r; 2342 } 2343 r = amdgpu_uvd_entity_init(adev, ring); 2344 if (r) { 2345 dev_err(adev->dev, 2346 "Failed to create UVD scheduling entity on ring %s.\n", 2347 ring->name); 2348 return r; 2349 } 2350 r = amdgpu_vce_entity_init(adev, ring); 2351 if (r) { 2352 dev_err(adev->dev, 2353 "Failed to create VCE scheduling entity on ring %s.\n", 2354 ring->name); 2355 return r; 2356 } 2357 } 2358 2359 if (adev->xcp_mgr) 2360 amdgpu_xcp_update_partition_sched_list(adev); 2361 2362 return 0; 2363 } 2364 2365 2366 /** 2367 * amdgpu_device_ip_init - run init for hardware IPs 2368 * 2369 * @adev: amdgpu_device pointer 2370 * 2371 * Main initialization pass for hardware IPs. The list of all the hardware 2372 * IPs that make up the asic is walked and the sw_init and hw_init callbacks 2373 * are run. sw_init initializes the software state associated with each IP 2374 * and hw_init initializes the hardware associated with each IP. 2375 * Returns 0 on success, negative error code on failure. 2376 */ 2377 static int amdgpu_device_ip_init(struct amdgpu_device *adev) 2378 { 2379 bool init_badpage; 2380 int i, r; 2381 2382 r = amdgpu_ras_init(adev); 2383 if (r) 2384 return r; 2385 2386 for (i = 0; i < adev->num_ip_blocks; i++) { 2387 if (!adev->ip_blocks[i].status.valid) 2388 continue; 2389 if (adev->ip_blocks[i].version->funcs->sw_init) { 2390 r = adev->ip_blocks[i].version->funcs->sw_init(&adev->ip_blocks[i]); 2391 if (r) { 2392 dev_err(adev->dev, 2393 "sw_init of IP block <%s> failed %d\n", 2394 adev->ip_blocks[i].version->funcs->name, 2395 r); 2396 goto init_failed; 2397 } 2398 } 2399 adev->ip_blocks[i].status.sw = true; 2400 2401 if (!amdgpu_ip_member_of_hwini( 2402 adev, adev->ip_blocks[i].version->type)) 2403 continue; 2404 2405 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) { 2406 /* need to do common hw init early so everything is set up for gmc */ 2407 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]); 2408 if (r) { 2409 dev_err(adev->dev, "hw_init %d failed %d\n", i, 2410 r); 2411 goto init_failed; 2412 } 2413 adev->ip_blocks[i].status.hw = true; 2414 } else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { 2415 /* need to do gmc hw init early so we can allocate gpu mem */ 2416 /* Try to reserve bad pages early */ 2417 if (amdgpu_sriov_vf(adev)) 2418 amdgpu_virt_exchange_data(adev); 2419 2420 r = amdgpu_device_mem_scratch_init(adev); 2421 if (r) { 2422 dev_err(adev->dev, 2423 "amdgpu_mem_scratch_init failed %d\n", 2424 r); 2425 goto init_failed; 2426 } 2427 r = adev->ip_blocks[i].version->funcs->hw_init(&adev->ip_blocks[i]); 2428 if (r) { 2429 dev_err(adev->dev, "hw_init %d failed %d\n", i, 2430 r); 2431 goto init_failed; 2432 } 2433 r = amdgpu_wb_init(adev); 2434 if (r) { 2435 dev_err(adev->dev, 2436 "amdgpu_wb_init failed %d\n", r); 2437 goto init_failed; 2438 } 2439 adev->ip_blocks[i].status.hw = true; 2440 2441 /* right after GMC hw init, we create CSA */ 2442 if (adev->gfx.mcbp) { 2443 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj, 2444 AMDGPU_GEM_DOMAIN_VRAM | 2445 AMDGPU_GEM_DOMAIN_GTT, 2446 AMDGPU_CSA_SIZE); 2447 if (r) { 2448 dev_err(adev->dev, 2449 "allocate CSA failed %d\n", r); 2450 goto init_failed; 2451 } 2452 } 2453 2454 r = amdgpu_seq64_init(adev); 2455 if (r) { 2456 dev_err(adev->dev, "allocate seq64 failed %d\n", 2457 r); 2458 goto init_failed; 2459 } 2460 } 2461 } 2462 2463 if (amdgpu_sriov_vf(adev)) 2464 amdgpu_virt_init_data_exchange(adev); 2465 2466 r = amdgpu_ib_pool_init(adev); 2467 if (r) { 2468 dev_err(adev->dev, "IB initialization failed (%d).\n", r); 2469 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r); 2470 goto init_failed; 2471 } 2472 2473 r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/ 2474 if (r) 2475 goto init_failed; 2476 2477 r = amdgpu_device_ip_hw_init_phase1(adev); 2478 if (r) 2479 goto init_failed; 2480 2481 r = amdgpu_device_fw_loading(adev); 2482 if (r) 2483 goto init_failed; 2484 2485 r = amdgpu_device_ip_hw_init_phase2(adev); 2486 if (r) 2487 goto init_failed; 2488 2489 /* 2490 * retired pages will be loaded from eeprom and reserved here, 2491 * it should be called after amdgpu_device_ip_hw_init_phase2 since 2492 * for some ASICs the RAS EEPROM code relies on SMU fully functioning 2493 * for I2C communication which only true at this point. 2494 * 2495 * amdgpu_ras_recovery_init may fail, but the upper only cares the 2496 * failure from bad gpu situation and stop amdgpu init process 2497 * accordingly. For other failed cases, it will still release all 2498 * the resource and print error message, rather than returning one 2499 * negative value to upper level. 2500 * 2501 * Note: theoretically, this should be called before all vram allocations 2502 * to protect retired page from abusing 2503 */ 2504 init_badpage = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI); 2505 r = amdgpu_ras_recovery_init(adev, init_badpage); 2506 if (r) 2507 goto init_failed; 2508 2509 /** 2510 * In case of XGMI grab extra reference for reset domain for this device 2511 */ 2512 if (adev->gmc.xgmi.num_physical_nodes > 1 && 2513 adev->gmc.xgmi.supported) { 2514 if (amdgpu_xgmi_add_device(adev) == 0) { 2515 if (!amdgpu_sriov_vf(adev)) { 2516 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 2517 2518 if (WARN_ON(!hive)) { 2519 r = -ENOENT; 2520 goto init_failed; 2521 } 2522 2523 if (!hive->reset_domain || 2524 !amdgpu_reset_get_reset_domain(hive->reset_domain)) { 2525 r = -ENOENT; 2526 amdgpu_put_xgmi_hive(hive); 2527 goto init_failed; 2528 } 2529 2530 /* Drop the early temporary reset domain we created for device */ 2531 amdgpu_reset_put_reset_domain(adev->reset_domain); 2532 adev->reset_domain = hive->reset_domain; 2533 amdgpu_put_xgmi_hive(hive); 2534 } 2535 } 2536 } 2537 2538 r = amdgpu_device_init_schedulers(adev); 2539 if (r) 2540 goto init_failed; 2541 2542 /* If SDMA is not brought up during hwini, the ttm buffer funcs enablement 2543 * is delayed after reset-on-init completes. 2544 */ 2545 if (amdgpu_ip_member_of_hwini(adev, AMD_IP_BLOCK_TYPE_SDMA)) 2546 amdgpu_ttm_enable_buffer_funcs(adev); 2547 2548 /* Don't init kfd if whole hive need to be reset during init */ 2549 if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) { 2550 amdgpu_amdkfd_device_init(adev); 2551 } 2552 2553 amdgpu_fru_get_product_info(adev); 2554 2555 r = amdgpu_cper_init(adev); 2556 2557 init_failed: 2558 2559 return r; 2560 } 2561 2562 /** 2563 * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer 2564 * 2565 * @adev: amdgpu_device pointer 2566 * 2567 * Writes a reset magic value to the gart pointer in VRAM. The driver calls 2568 * this function before a GPU reset. If the value is retained after a 2569 * GPU reset, VRAM has not been lost. Some GPU resets may destroy VRAM contents. 2570 */ 2571 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev) 2572 { 2573 memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM); 2574 } 2575 2576 /** 2577 * amdgpu_device_check_vram_lost - check if vram is valid 2578 * 2579 * @adev: amdgpu_device pointer 2580 * 2581 * Checks the reset magic value written to the gart pointer in VRAM. 2582 * The driver calls this after a GPU reset to see if the contents of 2583 * VRAM is lost or now. 2584 * returns true if vram is lost, false if not. 2585 */ 2586 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev) 2587 { 2588 if (memcmp(adev->gart.ptr, adev->reset_magic, 2589 AMDGPU_RESET_MAGIC_NUM)) 2590 return true; 2591 2592 if (!amdgpu_in_reset(adev)) 2593 return false; 2594 2595 /* 2596 * For all ASICs with baco/mode1 reset, the VRAM is 2597 * always assumed to be lost. 2598 */ 2599 switch (amdgpu_asic_reset_method(adev)) { 2600 case AMD_RESET_METHOD_LEGACY: 2601 case AMD_RESET_METHOD_LINK: 2602 case AMD_RESET_METHOD_BACO: 2603 case AMD_RESET_METHOD_MODE1: 2604 return true; 2605 default: 2606 return false; 2607 } 2608 } 2609 2610 /** 2611 * amdgpu_device_set_cg_state - set clockgating for amdgpu device 2612 * 2613 * @adev: amdgpu_device pointer 2614 * @state: clockgating state (gate or ungate) 2615 * 2616 * The list of all the hardware IPs that make up the asic is walked and the 2617 * set_clockgating_state callbacks are run. 2618 * Late initialization pass enabling clockgating for hardware IPs. 2619 * Fini or suspend, pass disabling clockgating for hardware IPs. 2620 * Returns 0 on success, negative error code on failure. 2621 */ 2622 2623 int amdgpu_device_set_cg_state(struct amdgpu_device *adev, 2624 enum amd_clockgating_state state) 2625 { 2626 int i, j, r; 2627 2628 if (amdgpu_emu_mode == 1) 2629 return 0; 2630 2631 for (j = 0; j < adev->num_ip_blocks; j++) { 2632 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; 2633 if (!adev->ip_blocks[i].status.late_initialized) 2634 continue; 2635 if (!adev->ip_blocks[i].version) 2636 continue; 2637 /* skip CG for GFX, SDMA on S0ix */ 2638 if (adev->in_s0ix && 2639 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX || 2640 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA)) 2641 continue; 2642 /* skip CG for VCE/UVD, it's handled specially */ 2643 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && 2644 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && 2645 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && 2646 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG && 2647 adev->ip_blocks[i].version->funcs->set_clockgating_state) { 2648 /* enable clockgating to save power */ 2649 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(&adev->ip_blocks[i], 2650 state); 2651 if (r) { 2652 dev_err(adev->dev, 2653 "set_clockgating_state(gate) of IP block <%s> failed %d\n", 2654 adev->ip_blocks[i].version->funcs->name, 2655 r); 2656 return r; 2657 } 2658 } 2659 } 2660 2661 return 0; 2662 } 2663 2664 int amdgpu_device_set_pg_state(struct amdgpu_device *adev, 2665 enum amd_powergating_state state) 2666 { 2667 int i, j, r; 2668 2669 if (amdgpu_emu_mode == 1) 2670 return 0; 2671 2672 for (j = 0; j < adev->num_ip_blocks; j++) { 2673 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; 2674 if (!adev->ip_blocks[i].status.late_initialized) 2675 continue; 2676 if (!adev->ip_blocks[i].version) 2677 continue; 2678 /* skip PG for GFX, SDMA on S0ix */ 2679 if (adev->in_s0ix && 2680 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX || 2681 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA)) 2682 continue; 2683 /* skip CG for VCE/UVD, it's handled specially */ 2684 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && 2685 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && 2686 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && 2687 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG && 2688 adev->ip_blocks[i].version->funcs->set_powergating_state) { 2689 /* enable powergating to save power */ 2690 r = adev->ip_blocks[i].version->funcs->set_powergating_state(&adev->ip_blocks[i], 2691 state); 2692 if (r) { 2693 dev_err(adev->dev, 2694 "set_powergating_state(gate) of IP block <%s> failed %d\n", 2695 adev->ip_blocks[i].version->funcs->name, 2696 r); 2697 return r; 2698 } 2699 } 2700 } 2701 return 0; 2702 } 2703 2704 static int amdgpu_device_enable_mgpu_fan_boost(void) 2705 { 2706 struct amdgpu_gpu_instance *gpu_ins; 2707 struct amdgpu_device *adev; 2708 int i, ret = 0; 2709 2710 mutex_lock(&mgpu_info.mutex); 2711 2712 /* 2713 * MGPU fan boost feature should be enabled 2714 * only when there are two or more dGPUs in 2715 * the system 2716 */ 2717 if (mgpu_info.num_dgpu < 2) 2718 goto out; 2719 2720 for (i = 0; i < mgpu_info.num_dgpu; i++) { 2721 gpu_ins = &(mgpu_info.gpu_ins[i]); 2722 adev = gpu_ins->adev; 2723 if (!(adev->flags & AMD_IS_APU || amdgpu_sriov_multi_vf_mode(adev)) && 2724 !gpu_ins->mgpu_fan_enabled) { 2725 ret = amdgpu_dpm_enable_mgpu_fan_boost(adev); 2726 if (ret) 2727 break; 2728 2729 gpu_ins->mgpu_fan_enabled = 1; 2730 } 2731 } 2732 2733 out: 2734 mutex_unlock(&mgpu_info.mutex); 2735 2736 return ret; 2737 } 2738 2739 /** 2740 * amdgpu_device_ip_late_init - run late init for hardware IPs 2741 * 2742 * @adev: amdgpu_device pointer 2743 * 2744 * Late initialization pass for hardware IPs. The list of all the hardware 2745 * IPs that make up the asic is walked and the late_init callbacks are run. 2746 * late_init covers any special initialization that an IP requires 2747 * after all of the have been initialized or something that needs to happen 2748 * late in the init process. 2749 * Returns 0 on success, negative error code on failure. 2750 */ 2751 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev) 2752 { 2753 struct amdgpu_gpu_instance *gpu_instance; 2754 int i = 0, r; 2755 2756 for (i = 0; i < adev->num_ip_blocks; i++) { 2757 if (!adev->ip_blocks[i].status.hw) 2758 continue; 2759 if (adev->ip_blocks[i].version->funcs->late_init) { 2760 r = adev->ip_blocks[i].version->funcs->late_init(&adev->ip_blocks[i]); 2761 if (r) { 2762 dev_err(adev->dev, 2763 "late_init of IP block <%s> failed %d\n", 2764 adev->ip_blocks[i].version->funcs->name, 2765 r); 2766 return r; 2767 } 2768 } 2769 adev->ip_blocks[i].status.late_initialized = true; 2770 } 2771 2772 r = amdgpu_ras_late_init(adev); 2773 if (r) { 2774 dev_err(adev->dev, "amdgpu_ras_late_init failed %d", r); 2775 return r; 2776 } 2777 2778 if (!amdgpu_reset_in_recovery(adev)) 2779 amdgpu_ras_set_error_query_ready(adev, true); 2780 2781 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE); 2782 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE); 2783 2784 r = amdgpu_dpm_reapply_umd_profile(adev); 2785 if (r) { 2786 dev_err(adev->dev, "Failed to restore UMD profile state: %d\n", r); 2787 return r; 2788 } 2789 2790 amdgpu_device_fill_reset_magic(adev); 2791 2792 r = amdgpu_device_enable_mgpu_fan_boost(); 2793 if (r) 2794 dev_err(adev->dev, "enable mgpu fan boost failed (%d).\n", r); 2795 2796 /* For passthrough configuration on arcturus and aldebaran, enable special handling SBR */ 2797 if (amdgpu_passthrough(adev) && 2798 ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1) || 2799 adev->asic_type == CHIP_ALDEBARAN)) 2800 amdgpu_dpm_handle_passthrough_sbr(adev, true); 2801 2802 if (adev->gmc.xgmi.num_physical_nodes > 1 && adev->gmc.xgmi.supported) { 2803 mutex_lock(&mgpu_info.mutex); 2804 2805 /* 2806 * Reset device p-state to low as this was booted with high. 2807 * 2808 * This should be performed only after all devices from the same 2809 * hive get initialized. 2810 * 2811 * However, it's unknown how many device in the hive in advance. 2812 * As this is counted one by one during devices initializations. 2813 * 2814 * So, we wait for all XGMI interlinked devices initialized. 2815 * This may bring some delays as those devices may come from 2816 * different hives. But that should be OK. 2817 */ 2818 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) { 2819 for (i = 0; i < mgpu_info.num_gpu; i++) { 2820 gpu_instance = &(mgpu_info.gpu_ins[i]); 2821 if (gpu_instance->adev->flags & AMD_IS_APU) 2822 continue; 2823 2824 r = amdgpu_xgmi_set_pstate(gpu_instance->adev, 2825 AMDGPU_XGMI_PSTATE_MIN); 2826 if (r) { 2827 dev_err(adev->dev, 2828 "pstate setting failed (%d).\n", 2829 r); 2830 break; 2831 } 2832 } 2833 } 2834 2835 mutex_unlock(&mgpu_info.mutex); 2836 } 2837 2838 return 0; 2839 } 2840 2841 static void amdgpu_ip_block_hw_fini(struct amdgpu_ip_block *ip_block) 2842 { 2843 struct amdgpu_device *adev = ip_block->adev; 2844 int r; 2845 2846 if (!ip_block->version->funcs->hw_fini) { 2847 dev_err(adev->dev, "hw_fini of IP block <%s> not defined\n", 2848 ip_block->version->funcs->name); 2849 } else { 2850 r = ip_block->version->funcs->hw_fini(ip_block); 2851 /* XXX handle errors */ 2852 if (r) { 2853 dev_dbg(adev->dev, 2854 "hw_fini of IP block <%s> failed %d\n", 2855 ip_block->version->funcs->name, r); 2856 } 2857 } 2858 2859 ip_block->status.hw = false; 2860 } 2861 2862 /** 2863 * amdgpu_device_smu_fini_early - smu hw_fini wrapper 2864 * 2865 * @adev: amdgpu_device pointer 2866 * 2867 * For ASICs need to disable SMC first 2868 */ 2869 static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev) 2870 { 2871 int i; 2872 2873 if (amdgpu_ip_version(adev, GC_HWIP, 0) > IP_VERSION(9, 0, 0)) 2874 return; 2875 2876 for (i = 0; i < adev->num_ip_blocks; i++) { 2877 if (!adev->ip_blocks[i].status.hw) 2878 continue; 2879 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) { 2880 amdgpu_ip_block_hw_fini(&adev->ip_blocks[i]); 2881 break; 2882 } 2883 } 2884 } 2885 2886 static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev) 2887 { 2888 int i, r; 2889 2890 for (i = 0; i < adev->num_ip_blocks; i++) { 2891 if (!adev->ip_blocks[i].version) 2892 continue; 2893 if (!adev->ip_blocks[i].version->funcs->early_fini) 2894 continue; 2895 2896 r = adev->ip_blocks[i].version->funcs->early_fini(&adev->ip_blocks[i]); 2897 if (r) { 2898 dev_dbg(adev->dev, 2899 "early_fini of IP block <%s> failed %d\n", 2900 adev->ip_blocks[i].version->funcs->name, r); 2901 } 2902 } 2903 2904 amdgpu_amdkfd_suspend(adev, true); 2905 amdgpu_amdkfd_teardown_processes(adev); 2906 amdgpu_userq_suspend(adev); 2907 2908 /* Workaround for ASICs need to disable SMC first */ 2909 amdgpu_device_smu_fini_early(adev); 2910 2911 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 2912 if (!adev->ip_blocks[i].status.hw) 2913 continue; 2914 2915 amdgpu_ip_block_hw_fini(&adev->ip_blocks[i]); 2916 } 2917 2918 if (amdgpu_sriov_vf(adev)) { 2919 if (amdgpu_virt_release_full_gpu(adev, false)) 2920 dev_err(adev->dev, 2921 "failed to release exclusive mode on fini\n"); 2922 } 2923 2924 /* 2925 * Driver reload on the APU can fail due to firmware validation because 2926 * the PSP is always running, as it is shared across the whole SoC. 2927 * This same issue does not occur on dGPU because it has a mechanism 2928 * that checks whether the PSP is running. A solution for those issues 2929 * in the APU is to trigger a GPU reset, but this should be done during 2930 * the unload phase to avoid adding boot latency and screen flicker. 2931 * GFX V11 has GC block as default off IP. Every time AMDGPU driver sends 2932 * a request to PMFW to unload MP1, PMFW will put GC in reset and power down 2933 * the voltage. Hence, skipping reset for APUs with GFX V11 or later. 2934 */ 2935 if ((adev->flags & AMD_IS_APU) && !adev->gmc.is_app_apu && 2936 amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(11, 0, 0)) { 2937 r = amdgpu_asic_reset(adev); 2938 if (r) 2939 dev_err(adev->dev, "asic reset on %s failed\n", __func__); 2940 } 2941 2942 return 0; 2943 } 2944 2945 /** 2946 * amdgpu_device_ip_fini - run fini for hardware IPs 2947 * 2948 * @adev: amdgpu_device pointer 2949 * 2950 * Main teardown pass for hardware IPs. The list of all the hardware 2951 * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks 2952 * are run. hw_fini tears down the hardware associated with each IP 2953 * and sw_fini tears down any software state associated with each IP. 2954 * Returns 0 on success, negative error code on failure. 2955 */ 2956 static int amdgpu_device_ip_fini(struct amdgpu_device *adev) 2957 { 2958 int i, r; 2959 2960 amdgpu_cper_fini(adev); 2961 2962 if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done) 2963 amdgpu_virt_release_ras_err_handler_data(adev); 2964 2965 if (adev->gmc.xgmi.num_physical_nodes > 1 && adev->gmc.xgmi.supported) 2966 amdgpu_xgmi_remove_device(adev); 2967 2968 amdgpu_amdkfd_device_fini_sw(adev); 2969 2970 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 2971 if (!adev->ip_blocks[i].status.sw) 2972 continue; 2973 2974 if (!adev->ip_blocks[i].version) 2975 continue; 2976 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { 2977 amdgpu_ucode_free_bo(adev); 2978 amdgpu_free_static_csa(&adev->virt.csa_obj); 2979 amdgpu_wb_fini(adev); 2980 amdgpu_device_mem_scratch_fini(adev); 2981 amdgpu_ib_pool_fini(adev); 2982 amdgpu_seq64_fini(adev); 2983 amdgpu_doorbell_fini(adev); 2984 } 2985 if (adev->ip_blocks[i].version->funcs->sw_fini) { 2986 r = adev->ip_blocks[i].version->funcs->sw_fini(&adev->ip_blocks[i]); 2987 /* XXX handle errors */ 2988 if (r) { 2989 dev_dbg(adev->dev, 2990 "sw_fini of IP block <%s> failed %d\n", 2991 adev->ip_blocks[i].version->funcs->name, 2992 r); 2993 } 2994 } 2995 adev->ip_blocks[i].status.sw = false; 2996 adev->ip_blocks[i].status.valid = false; 2997 } 2998 2999 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 3000 if (!adev->ip_blocks[i].status.late_initialized) 3001 continue; 3002 if (!adev->ip_blocks[i].version) 3003 continue; 3004 if (adev->ip_blocks[i].version->funcs->late_fini) 3005 adev->ip_blocks[i].version->funcs->late_fini(&adev->ip_blocks[i]); 3006 adev->ip_blocks[i].status.late_initialized = false; 3007 } 3008 3009 amdgpu_ras_fini(adev); 3010 amdgpu_uid_fini(adev); 3011 3012 return 0; 3013 } 3014 3015 /** 3016 * amdgpu_device_delayed_init_work_handler - work handler for IB tests 3017 * 3018 * @work: work_struct. 3019 */ 3020 static void amdgpu_device_delayed_init_work_handler(struct work_struct *work) 3021 { 3022 struct amdgpu_device *adev = 3023 container_of(work, struct amdgpu_device, delayed_init_work.work); 3024 int r; 3025 3026 r = amdgpu_ib_ring_tests(adev); 3027 if (r) 3028 dev_err(adev->dev, "ib ring test failed (%d).\n", r); 3029 } 3030 3031 static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work) 3032 { 3033 struct amdgpu_device *adev = 3034 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work); 3035 3036 WARN_ON_ONCE(adev->gfx.gfx_off_state); 3037 WARN_ON_ONCE(adev->gfx.gfx_off_req_count); 3038 3039 if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true, 0)) 3040 adev->gfx.gfx_off_state = true; 3041 } 3042 3043 /** 3044 * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1) 3045 * 3046 * @adev: amdgpu_device pointer 3047 * 3048 * Main suspend function for hardware IPs. The list of all the hardware 3049 * IPs that make up the asic is walked, clockgating is disabled and the 3050 * suspend callbacks are run. suspend puts the hardware and software state 3051 * in each IP into a state suitable for suspend. 3052 * Returns 0 on success, negative error code on failure. 3053 */ 3054 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev) 3055 { 3056 int i, r, rec; 3057 3058 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); 3059 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); 3060 3061 /* 3062 * Per PMFW team's suggestion, driver needs to handle gfxoff 3063 * and df cstate features disablement for gpu reset(e.g. Mode1Reset) 3064 * scenario. Add the missing df cstate disablement here. 3065 */ 3066 if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW)) 3067 dev_warn(adev->dev, "Failed to disallow df cstate"); 3068 3069 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 3070 if (!adev->ip_blocks[i].status.valid) 3071 continue; 3072 3073 /* displays are handled separately */ 3074 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE) 3075 continue; 3076 3077 r = amdgpu_ip_block_suspend(&adev->ip_blocks[i]); 3078 if (r) 3079 goto unwind; 3080 } 3081 3082 return 0; 3083 unwind: 3084 rec = amdgpu_device_ip_resume_phase3(adev); 3085 if (rec) 3086 dev_err(adev->dev, 3087 "amdgpu_device_ip_resume_phase3 failed during unwind: %d\n", 3088 rec); 3089 3090 amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_ALLOW); 3091 3092 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE); 3093 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE); 3094 3095 return r; 3096 } 3097 3098 /** 3099 * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2) 3100 * 3101 * @adev: amdgpu_device pointer 3102 * 3103 * Main suspend function for hardware IPs. The list of all the hardware 3104 * IPs that make up the asic is walked, clockgating is disabled and the 3105 * suspend callbacks are run. suspend puts the hardware and software state 3106 * in each IP into a state suitable for suspend. 3107 * Returns 0 on success, negative error code on failure. 3108 */ 3109 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev) 3110 { 3111 int i, r, rec; 3112 3113 if (adev->in_s0ix) 3114 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D3Entry); 3115 3116 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 3117 if (!adev->ip_blocks[i].status.valid || !adev->ip_blocks[i].status.hw) 3118 continue; 3119 /* displays are handled in phase1 */ 3120 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) 3121 continue; 3122 /* PSP lost connection when err_event_athub occurs */ 3123 if (amdgpu_ras_intr_triggered() && 3124 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) { 3125 adev->ip_blocks[i].status.hw = false; 3126 continue; 3127 } 3128 3129 /* skip unnecessary suspend if we do not initialize them yet */ 3130 if (!amdgpu_ip_member_of_hwini( 3131 adev, adev->ip_blocks[i].version->type)) 3132 continue; 3133 3134 /* Since we skip suspend for S0i3, we need to cancel the delayed 3135 * idle work here as the suspend callback never gets called. 3136 */ 3137 if (adev->in_s0ix && 3138 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX && 3139 amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 0, 0)) 3140 cancel_delayed_work_sync(&adev->gfx.idle_work); 3141 /* skip suspend of gfx/mes and psp for S0ix 3142 * gfx is in gfxoff state, so on resume it will exit gfxoff just 3143 * like at runtime. PSP is also part of the always on hardware 3144 * so no need to suspend it. 3145 */ 3146 if (adev->in_s0ix && 3147 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP || 3148 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX || 3149 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_MES)) 3150 continue; 3151 3152 /* SDMA 5.x+ is part of GFX power domain so it's covered by GFXOFF */ 3153 if (adev->in_s0ix && 3154 (amdgpu_ip_version(adev, SDMA0_HWIP, 0) >= 3155 IP_VERSION(5, 0, 0)) && 3156 (adev->ip_blocks[i].version->type == 3157 AMD_IP_BLOCK_TYPE_SDMA)) 3158 continue; 3159 3160 /* Once swPSP provides the IMU, RLC FW binaries to TOS during cold-boot. 3161 * These are in TMR, hence are expected to be reused by PSP-TOS to reload 3162 * from this location and RLC Autoload automatically also gets loaded 3163 * from here based on PMFW -> PSP message during re-init sequence. 3164 * Therefore, the psp suspend & resume should be skipped to avoid destroy 3165 * the TMR and reload FWs again for IMU enabled APU ASICs. 3166 */ 3167 if (amdgpu_in_reset(adev) && 3168 (adev->flags & AMD_IS_APU) && adev->gfx.imu.funcs && 3169 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) 3170 continue; 3171 3172 r = amdgpu_ip_block_suspend(&adev->ip_blocks[i]); 3173 if (r) 3174 goto unwind; 3175 3176 /* handle putting the SMC in the appropriate state */ 3177 if (!amdgpu_sriov_vf(adev)) { 3178 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) { 3179 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state); 3180 if (r) { 3181 dev_err(adev->dev, 3182 "SMC failed to set mp1 state %d, %d\n", 3183 adev->mp1_state, r); 3184 goto unwind; 3185 } 3186 } 3187 } 3188 } 3189 3190 return 0; 3191 unwind: 3192 /* suspend phase 2 = resume phase 1 + resume phase 2 */ 3193 rec = amdgpu_device_ip_resume_phase1(adev); 3194 if (rec) { 3195 dev_err(adev->dev, 3196 "amdgpu_device_ip_resume_phase1 failed during unwind: %d\n", 3197 rec); 3198 return r; 3199 } 3200 3201 rec = amdgpu_device_fw_loading(adev); 3202 if (rec) { 3203 dev_err(adev->dev, 3204 "amdgpu_device_fw_loading failed during unwind: %d\n", 3205 rec); 3206 return r; 3207 } 3208 3209 rec = amdgpu_device_ip_resume_phase2(adev); 3210 if (rec) { 3211 dev_err(adev->dev, 3212 "amdgpu_device_ip_resume_phase2 failed during unwind: %d\n", 3213 rec); 3214 return r; 3215 } 3216 3217 return r; 3218 } 3219 3220 /** 3221 * amdgpu_device_ip_suspend - run suspend for hardware IPs 3222 * 3223 * @adev: amdgpu_device pointer 3224 * 3225 * Main suspend function for hardware IPs. The list of all the hardware 3226 * IPs that make up the asic is walked, clockgating is disabled and the 3227 * suspend callbacks are run. suspend puts the hardware and software state 3228 * in each IP into a state suitable for suspend. 3229 * Returns 0 on success, negative error code on failure. 3230 */ 3231 static int amdgpu_device_ip_suspend(struct amdgpu_device *adev) 3232 { 3233 int r; 3234 3235 if (amdgpu_sriov_vf(adev)) { 3236 amdgpu_virt_fini_data_exchange(adev); 3237 amdgpu_virt_request_full_gpu(adev, false); 3238 } 3239 3240 r = amdgpu_device_ip_suspend_phase1(adev); 3241 if (r) 3242 return r; 3243 r = amdgpu_device_ip_suspend_phase2(adev); 3244 3245 if (amdgpu_sriov_vf(adev)) 3246 amdgpu_virt_release_full_gpu(adev, false); 3247 3248 return r; 3249 } 3250 3251 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev) 3252 { 3253 int i, r; 3254 3255 static enum amd_ip_block_type ip_order[] = { 3256 AMD_IP_BLOCK_TYPE_COMMON, 3257 AMD_IP_BLOCK_TYPE_GMC, 3258 AMD_IP_BLOCK_TYPE_PSP, 3259 AMD_IP_BLOCK_TYPE_IH, 3260 }; 3261 3262 for (i = 0; i < adev->num_ip_blocks; i++) { 3263 int j; 3264 struct amdgpu_ip_block *block; 3265 3266 block = &adev->ip_blocks[i]; 3267 block->status.hw = false; 3268 3269 for (j = 0; j < ARRAY_SIZE(ip_order); j++) { 3270 3271 if (block->version->type != ip_order[j] || 3272 !block->status.valid) 3273 continue; 3274 3275 r = block->version->funcs->hw_init(&adev->ip_blocks[i]); 3276 if (r) { 3277 dev_err(adev->dev, "RE-INIT-early: %s failed\n", 3278 block->version->funcs->name); 3279 return r; 3280 } 3281 block->status.hw = true; 3282 } 3283 } 3284 3285 return 0; 3286 } 3287 3288 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev) 3289 { 3290 struct amdgpu_ip_block *block; 3291 int i, r = 0; 3292 3293 static enum amd_ip_block_type ip_order[] = { 3294 AMD_IP_BLOCK_TYPE_SMC, 3295 AMD_IP_BLOCK_TYPE_DCE, 3296 AMD_IP_BLOCK_TYPE_GFX, 3297 AMD_IP_BLOCK_TYPE_SDMA, 3298 AMD_IP_BLOCK_TYPE_MES, 3299 AMD_IP_BLOCK_TYPE_UVD, 3300 AMD_IP_BLOCK_TYPE_VCE, 3301 AMD_IP_BLOCK_TYPE_VCN, 3302 AMD_IP_BLOCK_TYPE_JPEG 3303 }; 3304 3305 for (i = 0; i < ARRAY_SIZE(ip_order); i++) { 3306 block = amdgpu_device_ip_get_ip_block(adev, ip_order[i]); 3307 3308 if (!block) 3309 continue; 3310 3311 if (block->status.valid && !block->status.hw) { 3312 if (block->version->type == AMD_IP_BLOCK_TYPE_SMC) { 3313 r = amdgpu_ip_block_resume(block); 3314 } else { 3315 r = block->version->funcs->hw_init(block); 3316 } 3317 3318 if (r) { 3319 dev_err(adev->dev, "RE-INIT-late: %s failed\n", 3320 block->version->funcs->name); 3321 break; 3322 } 3323 block->status.hw = true; 3324 } 3325 } 3326 3327 return r; 3328 } 3329 3330 /** 3331 * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs 3332 * 3333 * @adev: amdgpu_device pointer 3334 * 3335 * First resume function for hardware IPs. The list of all the hardware 3336 * IPs that make up the asic is walked and the resume callbacks are run for 3337 * COMMON, GMC, and IH. resume puts the hardware into a functional state 3338 * after a suspend and updates the software state as necessary. This 3339 * function is also used for restoring the GPU after a GPU reset. 3340 * Returns 0 on success, negative error code on failure. 3341 */ 3342 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev) 3343 { 3344 int i, r; 3345 3346 for (i = 0; i < adev->num_ip_blocks; i++) { 3347 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw) 3348 continue; 3349 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || 3350 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || 3351 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH || 3352 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) { 3353 3354 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]); 3355 if (r) 3356 return r; 3357 } 3358 } 3359 3360 return 0; 3361 } 3362 3363 /** 3364 * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs 3365 * 3366 * @adev: amdgpu_device pointer 3367 * 3368 * Second resume function for hardware IPs. The list of all the hardware 3369 * IPs that make up the asic is walked and the resume callbacks are run for 3370 * all blocks except COMMON, GMC, and IH. resume puts the hardware into a 3371 * functional state after a suspend and updates the software state as 3372 * necessary. This function is also used for restoring the GPU after a GPU 3373 * reset. 3374 * Returns 0 on success, negative error code on failure. 3375 */ 3376 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev) 3377 { 3378 int i, r; 3379 3380 for (i = 0; i < adev->num_ip_blocks; i++) { 3381 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw) 3382 continue; 3383 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || 3384 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || 3385 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH || 3386 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE || 3387 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) 3388 continue; 3389 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]); 3390 if (r) 3391 return r; 3392 } 3393 3394 return 0; 3395 } 3396 3397 /** 3398 * amdgpu_device_ip_resume_phase3 - run resume for hardware IPs 3399 * 3400 * @adev: amdgpu_device pointer 3401 * 3402 * Third resume function for hardware IPs. The list of all the hardware 3403 * IPs that make up the asic is walked and the resume callbacks are run for 3404 * all DCE. resume puts the hardware into a functional state after a suspend 3405 * and updates the software state as necessary. This function is also used 3406 * for restoring the GPU after a GPU reset. 3407 * 3408 * Returns 0 on success, negative error code on failure. 3409 */ 3410 static int amdgpu_device_ip_resume_phase3(struct amdgpu_device *adev) 3411 { 3412 int i, r; 3413 3414 for (i = 0; i < adev->num_ip_blocks; i++) { 3415 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw) 3416 continue; 3417 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) { 3418 r = amdgpu_ip_block_resume(&adev->ip_blocks[i]); 3419 if (r) 3420 return r; 3421 } 3422 } 3423 3424 return 0; 3425 } 3426 3427 /** 3428 * amdgpu_device_ip_resume - run resume for hardware IPs 3429 * 3430 * @adev: amdgpu_device pointer 3431 * 3432 * Main resume function for hardware IPs. The hardware IPs 3433 * are split into two resume functions because they are 3434 * also used in recovering from a GPU reset and some additional 3435 * steps need to be take between them. In this case (S3/S4) they are 3436 * run sequentially. 3437 * Returns 0 on success, negative error code on failure. 3438 */ 3439 static int amdgpu_device_ip_resume(struct amdgpu_device *adev) 3440 { 3441 int r; 3442 3443 r = amdgpu_device_ip_resume_phase1(adev); 3444 if (r) 3445 return r; 3446 3447 r = amdgpu_device_fw_loading(adev); 3448 if (r) 3449 return r; 3450 3451 r = amdgpu_device_ip_resume_phase2(adev); 3452 3453 amdgpu_ttm_enable_buffer_funcs(adev); 3454 3455 if (r) 3456 return r; 3457 3458 amdgpu_fence_driver_hw_init(adev); 3459 3460 r = amdgpu_device_ip_resume_phase3(adev); 3461 3462 return r; 3463 } 3464 3465 /** 3466 * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV 3467 * 3468 * @adev: amdgpu_device pointer 3469 * 3470 * Query the VBIOS data tables to determine if the board supports SR-IOV. 3471 */ 3472 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev) 3473 { 3474 if (amdgpu_sriov_vf(adev)) { 3475 if (adev->is_atom_fw) { 3476 if (amdgpu_atomfirmware_gpu_virtualization_supported(adev)) 3477 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; 3478 } else { 3479 if (amdgpu_atombios_has_gpu_virtualization_table(adev)) 3480 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; 3481 } 3482 3483 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS)) 3484 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0); 3485 } 3486 } 3487 3488 /** 3489 * amdgpu_device_asic_has_dc_support - determine if DC supports the asic 3490 * 3491 * @pdev : pci device context 3492 * @asic_type: AMD asic type 3493 * 3494 * Check if there is DC (new modesetting infrastructre) support for an asic. 3495 * returns true if DC has support, false if not. 3496 */ 3497 bool amdgpu_device_asic_has_dc_support(struct pci_dev *pdev, 3498 enum amd_asic_type asic_type) 3499 { 3500 switch (asic_type) { 3501 #ifdef CONFIG_DRM_AMDGPU_SI 3502 case CHIP_HAINAN: 3503 #endif 3504 case CHIP_TOPAZ: 3505 /* chips with no display hardware */ 3506 return false; 3507 #if defined(CONFIG_DRM_AMD_DC) 3508 case CHIP_TAHITI: 3509 case CHIP_PITCAIRN: 3510 case CHIP_VERDE: 3511 case CHIP_OLAND: 3512 return amdgpu_dc != 0 && IS_ENABLED(CONFIG_DRM_AMD_DC_SI); 3513 default: 3514 return amdgpu_dc != 0; 3515 #else 3516 default: 3517 if (amdgpu_dc > 0) 3518 dev_info_once( 3519 &pdev->dev, 3520 "Display Core has been requested via kernel parameter but isn't supported by ASIC, ignoring\n"); 3521 return false; 3522 #endif 3523 } 3524 } 3525 3526 /** 3527 * amdgpu_device_has_dc_support - check if dc is supported 3528 * 3529 * @adev: amdgpu_device pointer 3530 * 3531 * Returns true for supported, false for not supported 3532 */ 3533 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev) 3534 { 3535 if (adev->enable_virtual_display || 3536 (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK)) 3537 return false; 3538 3539 return amdgpu_device_asic_has_dc_support(adev->pdev, adev->asic_type); 3540 } 3541 3542 static void amdgpu_device_xgmi_reset_func(struct work_struct *__work) 3543 { 3544 struct amdgpu_device *adev = 3545 container_of(__work, struct amdgpu_device, xgmi_reset_work); 3546 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 3547 3548 /* It's a bug to not have a hive within this function */ 3549 if (WARN_ON(!hive)) 3550 return; 3551 3552 /* 3553 * Use task barrier to synchronize all xgmi reset works across the 3554 * hive. task_barrier_enter and task_barrier_exit will block 3555 * until all the threads running the xgmi reset works reach 3556 * those points. task_barrier_full will do both blocks. 3557 */ 3558 if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) { 3559 3560 task_barrier_enter(&hive->tb); 3561 adev->asic_reset_res = amdgpu_device_baco_enter(adev); 3562 3563 if (adev->asic_reset_res) 3564 goto fail; 3565 3566 task_barrier_exit(&hive->tb); 3567 adev->asic_reset_res = amdgpu_device_baco_exit(adev); 3568 3569 if (adev->asic_reset_res) 3570 goto fail; 3571 3572 amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__MMHUB); 3573 } else { 3574 3575 task_barrier_full(&hive->tb); 3576 adev->asic_reset_res = amdgpu_asic_reset(adev); 3577 } 3578 3579 fail: 3580 if (adev->asic_reset_res) 3581 dev_warn(adev->dev, 3582 "ASIC reset failed with error, %d for drm dev, %s", 3583 adev->asic_reset_res, adev_to_drm(adev)->unique); 3584 amdgpu_put_xgmi_hive(hive); 3585 } 3586 3587 static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev) 3588 { 3589 char buf[AMDGPU_MAX_TIMEOUT_PARAM_LENGTH]; 3590 char *input = buf; 3591 char *timeout_setting = NULL; 3592 int index = 0; 3593 long timeout; 3594 int ret = 0; 3595 3596 /* By default timeout for all queues is 2 sec */ 3597 adev->gfx_timeout = adev->compute_timeout = adev->sdma_timeout = 3598 adev->video_timeout = msecs_to_jiffies(2000); 3599 3600 if (!strnlen(amdgpu_lockup_timeout, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) 3601 return 0; 3602 3603 /* 3604 * strsep() destructively modifies its input by replacing delimiters 3605 * with '\0'. Use a stack copy so the global module parameter buffer 3606 * remains intact for multi-GPU systems where this function is called 3607 * once per device. 3608 */ 3609 strscpy(buf, amdgpu_lockup_timeout, sizeof(buf)); 3610 3611 while ((timeout_setting = strsep(&input, ",")) && 3612 strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) { 3613 ret = kstrtol(timeout_setting, 0, &timeout); 3614 if (ret) 3615 return ret; 3616 3617 if (timeout == 0) { 3618 index++; 3619 continue; 3620 } else if (timeout < 0) { 3621 timeout = MAX_SCHEDULE_TIMEOUT; 3622 dev_warn(adev->dev, "lockup timeout disabled"); 3623 add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK); 3624 } else { 3625 timeout = msecs_to_jiffies(timeout); 3626 } 3627 3628 switch (index++) { 3629 case 0: 3630 adev->gfx_timeout = timeout; 3631 break; 3632 case 1: 3633 adev->compute_timeout = timeout; 3634 break; 3635 case 2: 3636 adev->sdma_timeout = timeout; 3637 break; 3638 case 3: 3639 adev->video_timeout = timeout; 3640 break; 3641 default: 3642 break; 3643 } 3644 } 3645 3646 /* When only one value specified apply it to all queues. */ 3647 if (index == 1) 3648 adev->gfx_timeout = adev->compute_timeout = adev->sdma_timeout = 3649 adev->video_timeout = timeout; 3650 3651 return ret; 3652 } 3653 3654 /** 3655 * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU 3656 * 3657 * @adev: amdgpu_device pointer 3658 * 3659 * RAM direct mapped to GPU if IOMMU is not enabled or is pass through mode 3660 */ 3661 static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev) 3662 { 3663 struct iommu_domain *domain; 3664 3665 domain = iommu_get_domain_for_dev(adev->dev); 3666 if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY) 3667 adev->ram_is_direct_mapped = true; 3668 } 3669 3670 #if defined(CONFIG_HSA_AMD_P2P) 3671 /** 3672 * amdgpu_device_check_iommu_remap - Check if DMA remapping is enabled. 3673 * 3674 * @adev: amdgpu_device pointer 3675 * 3676 * return if IOMMU remapping bar address 3677 */ 3678 static bool amdgpu_device_check_iommu_remap(struct amdgpu_device *adev) 3679 { 3680 struct iommu_domain *domain; 3681 3682 domain = iommu_get_domain_for_dev(adev->dev); 3683 if (domain && (domain->type == IOMMU_DOMAIN_DMA || 3684 domain->type == IOMMU_DOMAIN_DMA_FQ)) 3685 return true; 3686 3687 return false; 3688 } 3689 #endif 3690 3691 static void amdgpu_device_set_mcbp(struct amdgpu_device *adev) 3692 { 3693 if (amdgpu_mcbp == 1) 3694 adev->gfx.mcbp = true; 3695 else if (amdgpu_mcbp == 0) 3696 adev->gfx.mcbp = false; 3697 3698 if (amdgpu_sriov_vf(adev)) 3699 adev->gfx.mcbp = true; 3700 3701 if (adev->gfx.mcbp) 3702 dev_info(adev->dev, "MCBP is enabled\n"); 3703 } 3704 3705 static int amdgpu_device_sys_interface_init(struct amdgpu_device *adev) 3706 { 3707 int r; 3708 3709 r = amdgpu_atombios_sysfs_init(adev); 3710 if (r) 3711 drm_err(&adev->ddev, 3712 "registering atombios sysfs failed (%d).\n", r); 3713 3714 r = amdgpu_pm_sysfs_init(adev); 3715 if (r) 3716 dev_err(adev->dev, "registering pm sysfs failed (%d).\n", r); 3717 3718 r = amdgpu_ucode_sysfs_init(adev); 3719 if (r) { 3720 adev->ucode_sysfs_en = false; 3721 dev_err(adev->dev, "Creating firmware sysfs failed (%d).\n", r); 3722 } else 3723 adev->ucode_sysfs_en = true; 3724 3725 r = amdgpu_device_attr_sysfs_init(adev); 3726 if (r) 3727 dev_err(adev->dev, "Could not create amdgpu device attr\n"); 3728 3729 r = devm_device_add_group(adev->dev, &amdgpu_board_attrs_group); 3730 if (r) 3731 dev_err(adev->dev, 3732 "Could not create amdgpu board attributes\n"); 3733 3734 amdgpu_fru_sysfs_init(adev); 3735 amdgpu_reg_state_sysfs_init(adev); 3736 amdgpu_xcp_sysfs_init(adev); 3737 amdgpu_uma_sysfs_init(adev); 3738 amdgpu_ptl_sysfs_init(adev); 3739 amdgpu_ualink_sysfs_init(adev); 3740 3741 return r; 3742 } 3743 3744 static void amdgpu_device_sys_interface_fini(struct amdgpu_device *adev) 3745 { 3746 if (adev->pm.sysfs_initialized) 3747 amdgpu_pm_sysfs_fini(adev); 3748 if (adev->ucode_sysfs_en) 3749 amdgpu_ucode_sysfs_fini(adev); 3750 3751 amdgpu_discovery_sysfs_fini(adev); 3752 amdgpu_preempt_mgr_sysfs_fini(adev); 3753 3754 amdgpu_device_attr_sysfs_fini(adev); 3755 amdgpu_fru_sysfs_fini(adev); 3756 3757 amdgpu_reg_state_sysfs_fini(adev); 3758 amdgpu_xcp_sysfs_fini(adev); 3759 amdgpu_uma_sysfs_fini(adev); 3760 amdgpu_ptl_sysfs_fini(adev); 3761 amdgpu_ualink_sysfs_fini(adev); 3762 } 3763 3764 static bool 3765 amdgpu_device_should_register_switcheroo(struct amdgpu_device *adev, bool px) 3766 { 3767 return !pci_is_thunderbolt_attached(adev->pdev) && 3768 (px || (!dev_is_removable(&adev->pdev->dev) && 3769 apple_gmux_detect(NULL, NULL))); 3770 } 3771 3772 /** 3773 * amdgpu_device_init - initialize the driver 3774 * 3775 * @adev: amdgpu_device pointer 3776 * @flags: driver flags 3777 * 3778 * Initializes the driver info and hw (all asics). 3779 * Returns 0 for success or an error on failure. 3780 * Called at driver startup. 3781 */ 3782 int amdgpu_device_init(struct amdgpu_device *adev, 3783 uint32_t flags) 3784 { 3785 struct pci_dev *pdev = adev->pdev; 3786 int r, i; 3787 bool px = false; 3788 u32 max_MBps; 3789 int tmp; 3790 3791 adev->shutdown = false; 3792 adev->flags = flags; 3793 amdgpu_device_init_pcie_links(adev); 3794 3795 if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST) 3796 adev->asic_type = amdgpu_force_asic_type; 3797 else 3798 adev->asic_type = flags & AMD_ASIC_MASK; 3799 3800 adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT; 3801 if (amdgpu_emu_mode == 1) 3802 adev->usec_timeout *= 10; 3803 adev->gmc.gart_size = 512 * 1024 * 1024; 3804 adev->accel_working = false; 3805 adev->num_rings = 0; 3806 RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub()); 3807 adev->mman.buffer_funcs = NULL; 3808 adev->mman.num_buffer_funcs_scheds = 0; 3809 adev->vm_manager.vm_pte_funcs = NULL; 3810 adev->vm_manager.vm_pte_num_scheds = 0; 3811 adev->gmc.gmc_funcs = NULL; 3812 adev->harvest_ip_mask = 0x0; 3813 adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS); 3814 bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 3815 3816 amdgpu_reg_access_init(adev); 3817 3818 dev_info( 3819 adev->dev, 3820 "initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n", 3821 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device, 3822 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision); 3823 3824 /* mutex initialization are all done here so we 3825 * can recall function without having locking issues 3826 */ 3827 mutex_init(&adev->firmware.mutex); 3828 mutex_init(&adev->pm.mutex); 3829 mutex_init(&adev->gfx.gpu_clock_mutex); 3830 mutex_init(&adev->srbm_mutex); 3831 mutex_init(&adev->gfx.pipe_reserve_mutex); 3832 mutex_init(&adev->gfx.gfx_off_mutex); 3833 mutex_init(&adev->gfx.partition_mutex); 3834 mutex_init(&adev->grbm_idx_mutex); 3835 mutex_init(&adev->mn_lock); 3836 mutex_init(&adev->virt.vf_errors.lock); 3837 hash_init(adev->mn_hash); 3838 mutex_init(&adev->psp.mutex); 3839 mutex_init(&adev->psp.ptl.mutex); 3840 mutex_init(&adev->notifier_lock); 3841 mutex_init(&adev->pm.stable_pstate_ctx_lock); 3842 mutex_init(&adev->benchmark_mutex); 3843 mutex_init(&adev->gfx.reset_sem_mutex); 3844 3845 /* Associate locks with lockdep classes for ordering validation */ 3846 amdgpu_lockdep_set_class(adev); 3847 /* Initialize the mutex for cleaner shader isolation between GFX and compute processes */ 3848 mutex_init(&adev->enforce_isolation_mutex); 3849 for (i = 0; i < MAX_XCP; ++i) { 3850 adev->isolation[i].spearhead = dma_fence_get_stub(); 3851 amdgpu_sync_create(&adev->isolation[i].active); 3852 amdgpu_sync_create(&adev->isolation[i].prev); 3853 } 3854 mutex_init(&adev->gfx.userq_sch_mutex); 3855 mutex_init(&adev->gfx.workload_profile_mutex); 3856 mutex_init(&adev->vcn.workload_profile_mutex); 3857 3858 spin_lock_init(&adev->irq.lock); 3859 3860 amdgpu_ip_map_init(adev); 3861 amdgpu_early_init_rlc_reg_funcs(adev); 3862 amdgpu_device_init_apu_flags(adev); 3863 3864 r = amdgpu_device_check_arguments(adev); 3865 if (r) 3866 return r; 3867 3868 spin_lock_init(&adev->mmio_idx_lock); 3869 spin_lock_init(&adev->mm_stats.lock); 3870 spin_lock_init(&adev->virt.rlcg_reg_lock); 3871 spin_lock_init(&adev->wb.lock); 3872 mutex_init(&adev->lsdma.lock); 3873 3874 INIT_LIST_HEAD(&adev->reset_list); 3875 3876 INIT_LIST_HEAD(&adev->ras_list); 3877 3878 INIT_LIST_HEAD(&adev->pm.od_kobj_list); 3879 3880 xa_init_flags(&adev->userq_doorbell_xa, XA_FLAGS_LOCK_IRQ); 3881 3882 INIT_DELAYED_WORK(&adev->delayed_init_work, 3883 amdgpu_device_delayed_init_work_handler); 3884 INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work, 3885 amdgpu_device_delay_enable_gfx_off); 3886 /* 3887 * Initialize the enforce_isolation work structures for each XCP 3888 * partition. This work handler is responsible for enforcing shader 3889 * isolation on AMD GPUs. It counts the number of emitted fences for 3890 * each GFX and compute ring. If there are any fences, it schedules 3891 * the `enforce_isolation_work` to be run after a delay. If there are 3892 * no fences, it signals the Kernel Fusion Driver (KFD) to resume the 3893 * runqueue. 3894 */ 3895 for (i = 0; i < MAX_XCP; i++) { 3896 INIT_DELAYED_WORK(&adev->gfx.enforce_isolation[i].work, 3897 amdgpu_gfx_enforce_isolation_handler); 3898 adev->gfx.enforce_isolation[i].adev = adev; 3899 adev->gfx.enforce_isolation[i].xcp_id = i; 3900 } 3901 3902 INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func); 3903 3904 amdgpu_coredump_init(adev); 3905 3906 adev->gfx.gfx_off_req_count = 1; 3907 adev->gfx.gfx_off_residency = 0; 3908 adev->gfx.gfx_off_entrycount = 0; 3909 adev->pm.ac_power = power_supply_is_system_supplied() > 0; 3910 3911 atomic_set(&adev->throttling_logging_enabled, 1); 3912 /* 3913 * If throttling continues, logging will be performed every minute 3914 * to avoid log flooding. "-1" is subtracted since the thermal 3915 * throttling interrupt comes every second. Thus, the total logging 3916 * interval is 59 seconds(retelimited printk interval) + 1(waiting 3917 * for throttling interrupt) = 60 seconds. 3918 */ 3919 ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1); 3920 3921 ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE); 3922 3923 /* Registers mapping */ 3924 /* TODO: block userspace mapping of io register */ 3925 if (adev->asic_type >= CHIP_BONAIRE) { 3926 adev->rmmio_base = pci_resource_start(adev->pdev, 5); 3927 adev->rmmio_size = pci_resource_len(adev->pdev, 5); 3928 } else { 3929 adev->rmmio_base = pci_resource_start(adev->pdev, 2); 3930 adev->rmmio_size = pci_resource_len(adev->pdev, 2); 3931 } 3932 3933 for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++) 3934 atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN); 3935 3936 adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size); 3937 if (!adev->rmmio) 3938 return -ENOMEM; 3939 3940 dev_info(adev->dev, "register mmio base: 0x%08X\n", 3941 (uint32_t)adev->rmmio_base); 3942 dev_info(adev->dev, "register mmio size: %u\n", 3943 (unsigned int)adev->rmmio_size); 3944 3945 /* 3946 * Reset domain needs to be present early, before XGMI hive discovered 3947 * (if any) and initialized to use reset sem and in_gpu reset flag 3948 * early on during init and before calling to RREG32. 3949 */ 3950 adev->reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "amdgpu-reset-dev"); 3951 if (!adev->reset_domain) 3952 return -ENOMEM; 3953 3954 /* detect hw virtualization here */ 3955 amdgpu_virt_init(adev); 3956 3957 amdgpu_device_get_pcie_info(adev); 3958 3959 r = amdgpu_device_get_job_timeout_settings(adev); 3960 if (r) { 3961 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n"); 3962 return r; 3963 } 3964 3965 amdgpu_device_set_mcbp(adev); 3966 3967 /* 3968 * By default, use default mode where all blocks are expected to be 3969 * initialized. At present a 'swinit' of blocks is required to be 3970 * completed before the need for a different level is detected. 3971 */ 3972 amdgpu_set_init_level(adev, AMDGPU_INIT_LEVEL_DEFAULT); 3973 3974 amdgpu_device_check_iommu_direct_map(adev); 3975 3976 /* early init functions */ 3977 r = amdgpu_device_ip_early_init(adev); 3978 if (r) 3979 return r; 3980 3981 /* 3982 * No need to remove conflicting FBs for non-display class devices. 3983 * This prevents the sysfb from being freed accidently. 3984 */ 3985 if ((pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA || 3986 (pdev->class >> 8) == PCI_CLASS_DISPLAY_OTHER) { 3987 /* Get rid of things like offb */ 3988 r = aperture_remove_conflicting_pci_devices(adev->pdev, amdgpu_kms_driver.name); 3989 if (r) 3990 return r; 3991 } 3992 3993 /* Enable TMZ based on IP_VERSION */ 3994 amdgpu_gmc_tmz_set(adev); 3995 3996 if (amdgpu_sriov_vf(adev) && 3997 amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 3, 0)) 3998 /* VF MMIO access (except mailbox range) from CPU 3999 * will be blocked during sriov runtime 4000 */ 4001 adev->virt.caps |= AMDGPU_VF_MMIO_ACCESS_PROTECT; 4002 4003 amdgpu_gmc_noretry_set(adev); 4004 /* Need to get xgmi info early to decide the reset behavior*/ 4005 if (adev->gmc.xgmi.supported) { 4006 if (adev->gfxhub.funcs && 4007 adev->gfxhub.funcs->get_xgmi_info) { 4008 r = adev->gfxhub.funcs->get_xgmi_info(adev); 4009 if (r) 4010 return r; 4011 } 4012 } 4013 4014 if (adev->gmc.xgmi.connected_to_cpu) { 4015 if (adev->mmhub.funcs && 4016 adev->mmhub.funcs->get_xgmi_info) { 4017 r = adev->mmhub.funcs->get_xgmi_info(adev); 4018 if (r) 4019 return r; 4020 } 4021 } 4022 4023 /* enable PCIE atomic ops */ 4024 if (!amdgpu_sriov_vf(adev)) { 4025 /* APUs w/ gfx9 onwards doesn't reply on PCIe atomics, rather it is a 4026 * internal path natively support atomics, set have_atomics_support to true. 4027 */ 4028 if ((adev->flags & AMD_IS_APU && 4029 amdgpu_ip_version(adev, GC_HWIP, 0) > IP_VERSION(9, 0, 0)) || 4030 (adev->gmc.xgmi.connected_to_cpu && 4031 amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(12, 1, 0))) { 4032 adev->have_atomics_support = true; 4033 } else { 4034 adev->have_atomics_support = 4035 !pci_enable_atomic_ops_to_root(adev->pdev, 4036 PCI_EXP_DEVCAP2_ATOMIC_COMP32 | 4037 PCI_EXP_DEVCAP2_ATOMIC_COMP64); 4038 } 4039 } 4040 4041 if (!adev->have_atomics_support) 4042 dev_info(adev->dev, "PCIE atomic ops is not supported\n"); 4043 4044 /* doorbell bar mapping and doorbell index init*/ 4045 amdgpu_doorbell_init(adev); 4046 4047 if (amdgpu_emu_mode == 1) { 4048 /* post the asic on emulation mode */ 4049 emu_soc_asic_init(adev); 4050 goto fence_driver_init; 4051 } 4052 4053 amdgpu_reset_init(adev); 4054 4055 /* detect if we are with an SRIOV vbios */ 4056 if (adev->bios) 4057 amdgpu_device_detect_sriov_bios(adev); 4058 4059 /* check if we need to reset the asic 4060 * E.g., driver was not cleanly unloaded previously, etc. 4061 */ 4062 if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) { 4063 if (adev->gmc.xgmi.num_physical_nodes) { 4064 dev_info(adev->dev, "Pending hive reset.\n"); 4065 amdgpu_set_init_level(adev, 4066 AMDGPU_INIT_LEVEL_MINIMAL_XGMI); 4067 } else { 4068 tmp = amdgpu_reset_method; 4069 /* It should do a default reset when loading or reloading the driver, 4070 * regardless of the module parameter reset_method. 4071 */ 4072 amdgpu_reset_method = AMD_RESET_METHOD_NONE; 4073 r = amdgpu_asic_reset(adev); 4074 amdgpu_reset_method = tmp; 4075 } 4076 4077 if (r) { 4078 dev_err(adev->dev, "asic reset on init failed\n"); 4079 goto failed; 4080 } 4081 } 4082 4083 /* Post card if necessary */ 4084 if (amdgpu_device_need_post(adev)) { 4085 if (!adev->bios) { 4086 dev_err(adev->dev, "no vBIOS found\n"); 4087 r = -EINVAL; 4088 goto failed; 4089 } 4090 dev_info(adev->dev, "GPU posting now...\n"); 4091 r = amdgpu_device_asic_init(adev); 4092 if (r) { 4093 dev_err(adev->dev, "gpu post error!\n"); 4094 goto failed; 4095 } 4096 } 4097 4098 if (adev->bios) { 4099 if (adev->is_atom_fw) { 4100 /* Initialize clocks */ 4101 r = amdgpu_atomfirmware_get_clock_info(adev); 4102 if (r) { 4103 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n"); 4104 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0); 4105 goto failed; 4106 } 4107 } else { 4108 /* Initialize clocks */ 4109 r = amdgpu_atombios_get_clock_info(adev); 4110 if (r) { 4111 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n"); 4112 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0); 4113 goto failed; 4114 } 4115 /* init i2c buses */ 4116 amdgpu_i2c_init(adev); 4117 } 4118 } 4119 4120 fence_driver_init: 4121 /* Fence driver */ 4122 r = amdgpu_fence_driver_sw_init(adev); 4123 if (r) { 4124 dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n"); 4125 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0); 4126 goto failed; 4127 } 4128 4129 /* init the mode config */ 4130 drm_mode_config_init(adev_to_drm(adev)); 4131 4132 r = amdgpu_device_ip_init(adev); 4133 if (r) { 4134 dev_err(adev->dev, "amdgpu_device_ip_init failed\n"); 4135 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0); 4136 goto release_ras_con; 4137 } 4138 4139 amdgpu_fence_driver_hw_init(adev); 4140 4141 dev_info(adev->dev, 4142 "SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n", 4143 adev->gfx.config.max_shader_engines, 4144 adev->gfx.config.max_sh_per_se, 4145 adev->gfx.config.max_cu_per_sh, 4146 adev->gfx.cu_info.number); 4147 4148 adev->accel_working = true; 4149 4150 amdgpu_vm_check_compute_bug(adev); 4151 4152 /* Initialize the buffer migration limit. */ 4153 if (amdgpu_moverate >= 0) 4154 max_MBps = amdgpu_moverate; 4155 else 4156 max_MBps = 8; /* Allow 8 MB/s. */ 4157 /* Get a log2 for easy divisions. */ 4158 adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps)); 4159 4160 /* 4161 * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost. 4162 * Otherwise the mgpu fan boost feature will be skipped due to the 4163 * gpu instance is counted less. 4164 */ 4165 amdgpu_register_gpu_instance(adev); 4166 4167 /* enable clockgating, etc. after ib tests, etc. since some blocks require 4168 * explicit gating rather than handling it automatically. 4169 */ 4170 if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) { 4171 r = amdgpu_device_ip_late_init(adev); 4172 if (r) { 4173 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n"); 4174 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r); 4175 goto release_ras_con; 4176 } 4177 /* must succeed. */ 4178 amdgpu_ras_resume(adev); 4179 queue_delayed_work(system_dfl_wq, &adev->delayed_init_work, 4180 msecs_to_jiffies(AMDGPU_RESUME_MS)); 4181 } 4182 4183 if (amdgpu_sriov_vf(adev)) { 4184 amdgpu_virt_release_full_gpu(adev, true); 4185 flush_delayed_work(&adev->delayed_init_work); 4186 } 4187 4188 /* Don't init kfd if whole hive need to be reset during init */ 4189 if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) { 4190 kgd2kfd_init_zone_device(adev); 4191 kfd_update_svm_support_properties(adev); 4192 } 4193 4194 if (adev->init_lvl->level == AMDGPU_INIT_LEVEL_MINIMAL_XGMI) 4195 amdgpu_xgmi_reset_on_init(adev); 4196 4197 /* 4198 * Place those sysfs registering after `late_init`. As some of those 4199 * operations performed in `late_init` might affect the sysfs 4200 * interfaces creating. 4201 */ 4202 r = amdgpu_device_sys_interface_init(adev); 4203 4204 if (IS_ENABLED(CONFIG_PERF_EVENTS)) 4205 r = amdgpu_pmu_init(adev); 4206 if (r) 4207 dev_err(adev->dev, "amdgpu_pmu_init failed\n"); 4208 4209 /* Have stored pci confspace at hand for restore in sudden PCI error */ 4210 if (amdgpu_device_cache_pci_state(adev->pdev)) 4211 pci_restore_state(pdev); 4212 4213 /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */ 4214 /* this will fail for cards that aren't VGA class devices, just 4215 * ignore it 4216 */ 4217 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) 4218 vga_client_register(adev->pdev, amdgpu_device_vga_set_decode, adev); 4219 4220 px = amdgpu_device_supports_px(adev); 4221 4222 if (amdgpu_device_should_register_switcheroo(adev, px)) 4223 vga_switcheroo_register_client(adev->pdev, 4224 &amdgpu_switcheroo_ops, px); 4225 4226 if (px) 4227 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain); 4228 4229 adev->pm_nb.notifier_call = amdgpu_device_pm_notifier; 4230 r = register_pm_notifier(&adev->pm_nb); 4231 if (r) 4232 goto failed; 4233 4234 return 0; 4235 4236 release_ras_con: 4237 if (amdgpu_sriov_vf(adev)) 4238 amdgpu_virt_release_full_gpu(adev, true); 4239 4240 /* failed in exclusive mode due to timeout */ 4241 if (amdgpu_sriov_vf(adev) && 4242 !amdgpu_sriov_runtime(adev) && 4243 amdgpu_virt_mmio_blocked(adev) && 4244 !amdgpu_virt_wait_reset(adev)) { 4245 dev_err(adev->dev, "VF exclusive mode timeout\n"); 4246 /* Don't send request since VF is inactive. */ 4247 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; 4248 adev->virt.ops = NULL; 4249 r = -EAGAIN; 4250 } 4251 amdgpu_release_ras_context(adev); 4252 4253 failed: 4254 amdgpu_vf_error_trans_all(adev); 4255 4256 return r; 4257 } 4258 4259 static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev) 4260 { 4261 4262 /* Clear all CPU mappings pointing to this device */ 4263 unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1); 4264 4265 /* Unmap all mapped bars - Doorbell, registers and VRAM */ 4266 amdgpu_doorbell_fini(adev); 4267 4268 iounmap(adev->rmmio); 4269 adev->rmmio = NULL; 4270 if (adev->mman.aper_base_kaddr) 4271 iounmap(adev->mman.aper_base_kaddr); 4272 adev->mman.aper_base_kaddr = NULL; 4273 4274 /* Memory manager related */ 4275 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { 4276 arch_phys_wc_del(adev->gmc.vram_mtrr); 4277 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); 4278 } 4279 } 4280 4281 /** 4282 * amdgpu_device_fini_hw - tear down the driver 4283 * 4284 * @adev: amdgpu_device pointer 4285 * 4286 * Tear down the driver info (all asics). 4287 * Called at driver shutdown. 4288 */ 4289 void amdgpu_device_fini_hw(struct amdgpu_device *adev) 4290 { 4291 dev_info(adev->dev, "finishing device.\n"); 4292 flush_delayed_work(&adev->delayed_init_work); 4293 4294 if (adev->mman.initialized) 4295 drain_workqueue(adev->mman.bdev.wq); 4296 4297 adev->shutdown = true; 4298 4299 unregister_pm_notifier(&adev->pm_nb); 4300 4301 /* make sure IB test finished before entering exclusive mode 4302 * to avoid preemption on IB test 4303 */ 4304 if (amdgpu_sriov_vf(adev)) { 4305 amdgpu_virt_request_full_gpu(adev, false); 4306 amdgpu_virt_fini_data_exchange(adev); 4307 } 4308 4309 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); 4310 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); 4311 4312 /* disable all interrupts */ 4313 amdgpu_irq_disable_all(adev); 4314 if (adev->mode_info.mode_config_initialized) { 4315 if (!drm_drv_uses_atomic_modeset(adev_to_drm(adev))) 4316 drm_helper_force_disable_all(adev_to_drm(adev)); 4317 else 4318 drm_atomic_helper_shutdown(adev_to_drm(adev)); 4319 } 4320 amdgpu_fence_driver_hw_fini(adev); 4321 4322 amdgpu_device_sys_interface_fini(adev); 4323 4324 /* disable ras feature must before hw fini */ 4325 amdgpu_ras_pre_fini(adev); 4326 4327 amdgpu_ttm_disable_buffer_funcs(adev); 4328 4329 /* 4330 * device went through surprise hotplug; we need to destroy topology 4331 * before ip_fini_early to prevent kfd locking refcount issues by calling 4332 * amdgpu_amdkfd_suspend() 4333 */ 4334 if (pci_dev_is_disconnected(adev->pdev)) { 4335 /* Drain KFD processes while kfd.dev is still valid */ 4336 amdgpu_amdkfd_teardown_processes(adev); 4337 amdgpu_amdkfd_device_fini_sw(adev); 4338 } 4339 4340 amdgpu_coredump_fini(adev); 4341 amdgpu_device_ip_fini_early(adev); 4342 4343 amdgpu_irq_fini_hw(adev); 4344 4345 if (adev->mman.initialized) 4346 ttm_device_clear_dma_mappings(&adev->mman.bdev); 4347 4348 amdgpu_gart_dummy_page_fini(adev); 4349 4350 if (pci_dev_is_disconnected(adev->pdev)) 4351 amdgpu_device_unmap_mmio(adev); 4352 4353 } 4354 4355 void amdgpu_device_fini_sw(struct amdgpu_device *adev) 4356 { 4357 int i; 4358 bool px; 4359 4360 amdgpu_device_ip_fini(adev); 4361 amdgpu_fence_driver_sw_fini(adev); 4362 amdgpu_ucode_release(&adev->firmware.gpu_info_fw); 4363 adev->accel_working = false; 4364 dma_fence_put(rcu_dereference_protected(adev->gang_submit, true)); 4365 for (i = 0; i < MAX_XCP; ++i) { 4366 dma_fence_put(adev->isolation[i].spearhead); 4367 amdgpu_sync_free(&adev->isolation[i].active); 4368 amdgpu_sync_free(&adev->isolation[i].prev); 4369 } 4370 4371 amdgpu_reset_fini(adev); 4372 4373 /* free i2c buses */ 4374 amdgpu_i2c_fini(adev); 4375 4376 if (adev->bios) { 4377 if (amdgpu_emu_mode != 1) 4378 amdgpu_atombios_fini(adev); 4379 amdgpu_bios_release(adev); 4380 } 4381 4382 kfree(adev->fru_info); 4383 adev->fru_info = NULL; 4384 4385 kfree(adev->xcp_mgr); 4386 adev->xcp_mgr = NULL; 4387 4388 px = amdgpu_device_supports_px(adev); 4389 4390 if (amdgpu_device_should_register_switcheroo(adev, px)) 4391 vga_switcheroo_unregister_client(adev->pdev); 4392 4393 if (px) 4394 vga_switcheroo_fini_domain_pm_ops(adev->dev); 4395 4396 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) 4397 vga_client_unregister(adev->pdev); 4398 4399 if (adev->rmmio) { 4400 iounmap(adev->rmmio); 4401 adev->rmmio = NULL; 4402 } 4403 4404 if (IS_ENABLED(CONFIG_PERF_EVENTS)) 4405 amdgpu_pmu_fini(adev); 4406 if (adev->discovery.bin) 4407 amdgpu_discovery_fini(adev); 4408 4409 amdgpu_reset_put_reset_domain(adev->reset_domain); 4410 adev->reset_domain = NULL; 4411 4412 kfree(adev->pci_state); 4413 kfree(adev->pcie_reset_ctx.swds_pcistate); 4414 kfree(adev->pcie_reset_ctx.swus_pcistate); 4415 } 4416 4417 /** 4418 * amdgpu_device_evict_resources - evict device resources 4419 * @adev: amdgpu device object 4420 * 4421 * Evicts all ttm device resources(vram BOs, gart table) from the lru list 4422 * of the vram memory type. Mainly used for evicting device resources 4423 * at suspend time. 4424 * 4425 */ 4426 static int amdgpu_device_evict_resources(struct amdgpu_device *adev) 4427 { 4428 int ret; 4429 4430 /* No need to evict vram on APUs unless going to S4 */ 4431 if (!adev->in_s4 && (adev->flags & AMD_IS_APU)) 4432 return 0; 4433 4434 /* No need to evict when going to S5 through S4 callbacks */ 4435 if (system_state == SYSTEM_POWER_OFF) 4436 return 0; 4437 4438 ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM); 4439 if (ret) { 4440 dev_warn(adev->dev, "evicting device resources failed\n"); 4441 return ret; 4442 } 4443 4444 if (adev->in_s4) { 4445 ret = ttm_device_prepare_hibernation(&adev->mman.bdev); 4446 if (ret) 4447 dev_err(adev->dev, "prepare hibernation failed, %d\n", ret); 4448 } 4449 return ret; 4450 } 4451 4452 /* 4453 * Suspend & resume. 4454 */ 4455 /** 4456 * amdgpu_device_pm_notifier - Notification block for Suspend/Hibernate events 4457 * @nb: notifier block 4458 * @mode: suspend mode 4459 * @data: data 4460 * 4461 * This function is called when the system is about to suspend or hibernate. 4462 * It is used to set the appropriate flags so that eviction can be optimized 4463 * in the pm prepare callback. 4464 */ 4465 static int amdgpu_device_pm_notifier(struct notifier_block *nb, unsigned long mode, 4466 void *data) 4467 { 4468 struct amdgpu_device *adev = container_of(nb, struct amdgpu_device, pm_nb); 4469 4470 switch (mode) { 4471 case PM_HIBERNATION_PREPARE: 4472 adev->in_s4 = true; 4473 break; 4474 case PM_POST_HIBERNATION: 4475 adev->in_s4 = false; 4476 break; 4477 } 4478 4479 return NOTIFY_DONE; 4480 } 4481 4482 /** 4483 * amdgpu_device_prepare - prepare for device suspend 4484 * 4485 * @dev: drm dev pointer 4486 * 4487 * Prepare to put the hw in the suspend state (all asics). 4488 * Returns 0 for success or an error on failure. 4489 * Called at driver suspend. 4490 */ 4491 int amdgpu_device_prepare(struct drm_device *dev) 4492 { 4493 struct amdgpu_device *adev = drm_to_adev(dev); 4494 int i, r; 4495 4496 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 4497 return 0; 4498 4499 /* Evict the majority of BOs before starting suspend sequence */ 4500 r = amdgpu_device_evict_resources(adev); 4501 if (r) 4502 return r; 4503 4504 flush_delayed_work(&adev->gfx.gfx_off_delay_work); 4505 4506 for (i = 0; i < adev->num_ip_blocks; i++) { 4507 if (!adev->ip_blocks[i].status.valid) 4508 continue; 4509 if (!adev->ip_blocks[i].version->funcs->prepare_suspend) 4510 continue; 4511 r = adev->ip_blocks[i].version->funcs->prepare_suspend(&adev->ip_blocks[i]); 4512 if (r) 4513 return r; 4514 } 4515 4516 return 0; 4517 } 4518 4519 /** 4520 * amdgpu_device_complete - complete power state transition 4521 * 4522 * @dev: drm dev pointer 4523 * 4524 * Undo the changes from amdgpu_device_prepare. This will be 4525 * called on all resume transitions, including those that failed. 4526 */ 4527 void amdgpu_device_complete(struct drm_device *dev) 4528 { 4529 struct amdgpu_device *adev = drm_to_adev(dev); 4530 int i; 4531 4532 for (i = 0; i < adev->num_ip_blocks; i++) { 4533 if (!adev->ip_blocks[i].status.valid) 4534 continue; 4535 if (!adev->ip_blocks[i].version->funcs->complete) 4536 continue; 4537 adev->ip_blocks[i].version->funcs->complete(&adev->ip_blocks[i]); 4538 } 4539 } 4540 4541 /** 4542 * amdgpu_device_suspend - initiate device suspend 4543 * 4544 * @dev: drm dev pointer 4545 * @notify_clients: notify in-kernel DRM clients 4546 * 4547 * Puts the hw in the suspend state (all asics). 4548 * Returns 0 for success or an error on failure. 4549 * Called at driver suspend. 4550 */ 4551 int amdgpu_device_suspend(struct drm_device *dev, bool notify_clients) 4552 { 4553 struct amdgpu_device *adev = drm_to_adev(dev); 4554 int r, rec; 4555 4556 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 4557 return 0; 4558 4559 adev->in_suspend = true; 4560 4561 if (amdgpu_sriov_vf(adev)) { 4562 if (!adev->in_runpm) 4563 amdgpu_amdkfd_suspend_process(adev); 4564 amdgpu_virt_fini_data_exchange(adev); 4565 r = amdgpu_virt_request_full_gpu(adev, false); 4566 if (r) 4567 return r; 4568 } 4569 4570 r = amdgpu_acpi_smart_shift_update(adev, AMDGPU_SS_DEV_D3); 4571 if (r) 4572 goto unwind_sriov; 4573 4574 if (notify_clients) 4575 drm_client_dev_suspend(adev_to_drm(adev)); 4576 4577 cancel_delayed_work_sync(&adev->delayed_init_work); 4578 4579 amdgpu_ras_suspend(adev); 4580 4581 r = amdgpu_device_ip_suspend_phase1(adev); 4582 if (r) 4583 goto unwind_smartshift; 4584 4585 amdgpu_amdkfd_suspend(adev, !amdgpu_sriov_vf(adev) && !adev->in_runpm); 4586 r = amdgpu_userq_suspend(adev); 4587 if (r) 4588 goto unwind_ip_phase1; 4589 4590 r = amdgpu_device_evict_resources(adev); 4591 if (r) 4592 goto unwind_userq; 4593 4594 amdgpu_ttm_disable_buffer_funcs(adev); 4595 4596 amdgpu_fence_driver_hw_fini(adev); 4597 4598 r = amdgpu_device_ip_suspend_phase2(adev); 4599 if (r) 4600 goto unwind_evict; 4601 4602 if (amdgpu_sriov_vf(adev)) 4603 amdgpu_virt_release_full_gpu(adev, false); 4604 4605 return 0; 4606 4607 unwind_evict: 4608 amdgpu_ttm_enable_buffer_funcs(adev); 4609 amdgpu_fence_driver_hw_init(adev); 4610 4611 unwind_userq: 4612 rec = amdgpu_userq_resume(adev); 4613 if (rec) { 4614 dev_warn(adev->dev, "failed to re-initialize user queues: %d\n", rec); 4615 return r; 4616 } 4617 rec = amdgpu_amdkfd_resume(adev, !amdgpu_sriov_vf(adev) && !adev->in_runpm); 4618 if (rec) { 4619 dev_warn(adev->dev, "failed to re-initialize kfd: %d\n", rec); 4620 return r; 4621 } 4622 4623 unwind_ip_phase1: 4624 /* suspend phase 1 = resume phase 3 */ 4625 rec = amdgpu_device_ip_resume_phase3(adev); 4626 if (rec) { 4627 dev_warn(adev->dev, "failed to re-initialize IPs phase1: %d\n", rec); 4628 return r; 4629 } 4630 4631 unwind_smartshift: 4632 rec = amdgpu_acpi_smart_shift_update(adev, AMDGPU_SS_DEV_D0); 4633 if (rec) { 4634 dev_warn(adev->dev, "failed to re-update smart shift: %d\n", rec); 4635 return r; 4636 } 4637 4638 if (notify_clients) 4639 drm_client_dev_resume(adev_to_drm(adev)); 4640 4641 amdgpu_ras_resume(adev); 4642 4643 unwind_sriov: 4644 if (amdgpu_sriov_vf(adev)) { 4645 rec = amdgpu_virt_request_full_gpu(adev, true); 4646 if (rec) { 4647 dev_warn(adev->dev, "failed to reinitialize sriov: %d\n", rec); 4648 return r; 4649 } 4650 } 4651 4652 adev->in_suspend = adev->in_s0ix = adev->in_s3 = false; 4653 4654 return r; 4655 } 4656 4657 static inline int amdgpu_virt_resume(struct amdgpu_device *adev) 4658 { 4659 int r; 4660 unsigned int prev_physical_node_id = adev->gmc.xgmi.physical_node_id; 4661 4662 /* During VM resume, QEMU programming of VF MSIX table (register GFXMSIX_VECT0_ADDR_LO) 4663 * may not work. The access could be blocked by nBIF protection as VF isn't in 4664 * exclusive access mode. Exclusive access is enabled now, disable/enable MSIX 4665 * so that QEMU reprograms MSIX table. 4666 */ 4667 amdgpu_restore_msix(adev); 4668 4669 r = adev->gfxhub.funcs->get_xgmi_info(adev); 4670 if (r) 4671 return r; 4672 4673 dev_info(adev->dev, "xgmi node, old id %d, new id %d\n", 4674 prev_physical_node_id, adev->gmc.xgmi.physical_node_id); 4675 4676 adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev); 4677 adev->vm_manager.vram_base_offset += 4678 adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 4679 4680 return 0; 4681 } 4682 4683 /** 4684 * amdgpu_device_resume - initiate device resume 4685 * 4686 * @dev: drm dev pointer 4687 * @notify_clients: notify in-kernel DRM clients 4688 * 4689 * Bring the hw back to operating state (all asics). 4690 * Returns 0 for success or an error on failure. 4691 * Called at driver resume. 4692 */ 4693 int amdgpu_device_resume(struct drm_device *dev, bool notify_clients) 4694 { 4695 struct amdgpu_device *adev = drm_to_adev(dev); 4696 int r = 0; 4697 4698 if (amdgpu_sriov_vf(adev)) { 4699 r = amdgpu_virt_request_full_gpu(adev, true); 4700 if (r) 4701 return r; 4702 } 4703 4704 if (amdgpu_virt_xgmi_migrate_enabled(adev)) { 4705 r = amdgpu_virt_resume(adev); 4706 if (r) 4707 goto exit; 4708 } 4709 4710 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 4711 return 0; 4712 4713 if (adev->in_s0ix) 4714 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry); 4715 4716 /* post card */ 4717 if (amdgpu_device_need_post(adev)) { 4718 r = amdgpu_device_asic_init(adev); 4719 if (r) 4720 dev_err(adev->dev, "amdgpu asic init failed\n"); 4721 } 4722 4723 r = amdgpu_device_ip_resume(adev); 4724 4725 if (r) { 4726 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r); 4727 goto exit; 4728 } 4729 4730 r = amdgpu_amdkfd_resume(adev, !amdgpu_sriov_vf(adev) && !adev->in_runpm); 4731 if (r) 4732 goto exit; 4733 4734 r = amdgpu_userq_resume(adev); 4735 if (r) 4736 goto exit; 4737 4738 r = amdgpu_device_ip_late_init(adev); 4739 if (r) 4740 goto exit; 4741 4742 queue_delayed_work(system_dfl_wq, &adev->delayed_init_work, 4743 msecs_to_jiffies(AMDGPU_RESUME_MS)); 4744 exit: 4745 if (amdgpu_sriov_vf(adev)) { 4746 amdgpu_virt_init_data_exchange(adev); 4747 amdgpu_virt_release_full_gpu(adev, true); 4748 4749 if (!r && !adev->in_runpm) 4750 r = amdgpu_amdkfd_resume_process(adev); 4751 } 4752 4753 if (r) 4754 return r; 4755 4756 /* Make sure IB tests flushed */ 4757 flush_delayed_work(&adev->delayed_init_work); 4758 4759 if (notify_clients) 4760 drm_client_dev_resume(adev_to_drm(adev)); 4761 4762 amdgpu_ras_resume(adev); 4763 4764 if (adev->mode_info.num_crtc) { 4765 /* 4766 * Most of the connector probing functions try to acquire runtime pm 4767 * refs to ensure that the GPU is powered on when connector polling is 4768 * performed. Since we're calling this from a runtime PM callback, 4769 * trying to acquire rpm refs will cause us to deadlock. 4770 * 4771 * Since we're guaranteed to be holding the rpm lock, it's safe to 4772 * temporarily disable the rpm helpers so this doesn't deadlock us. 4773 */ 4774 #ifdef CONFIG_PM 4775 dev->dev->power.disable_depth++; 4776 #endif 4777 if (!adev->dc_enabled) 4778 drm_helper_hpd_irq_event(dev); 4779 else 4780 drm_kms_helper_hotplug_event(dev); 4781 #ifdef CONFIG_PM 4782 dev->dev->power.disable_depth--; 4783 #endif 4784 } 4785 4786 amdgpu_vram_mgr_clear_reset_blocks(adev); 4787 adev->in_suspend = false; 4788 4789 if (amdgpu_acpi_smart_shift_update(adev, AMDGPU_SS_DEV_D0)) 4790 dev_warn(adev->dev, "smart shift update failed\n"); 4791 4792 return 0; 4793 } 4794 4795 /** 4796 * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf 4797 * 4798 * @adev: amdgpu_device pointer 4799 * @reset_context: amdgpu reset context pointer 4800 * 4801 * do VF FLR and reinitialize Asic 4802 * return 0 means succeeded otherwise failed 4803 */ 4804 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev, 4805 struct amdgpu_reset_context *reset_context) 4806 { 4807 int r; 4808 struct amdgpu_hive_info *hive = NULL; 4809 4810 if (test_bit(AMDGPU_HOST_FLR, &reset_context->flags)) { 4811 if (!amdgpu_ras_get_fed_status(adev)) 4812 amdgpu_virt_ready_to_reset(adev); 4813 amdgpu_virt_wait_reset(adev); 4814 clear_bit(AMDGPU_HOST_FLR, &reset_context->flags); 4815 r = amdgpu_virt_request_full_gpu(adev, true); 4816 } else { 4817 r = amdgpu_virt_reset_gpu(adev); 4818 } 4819 if (r) 4820 return r; 4821 4822 amdgpu_ras_clear_err_state(adev); 4823 amdgpu_irq_gpu_reset_resume_helper(adev); 4824 4825 /* some sw clean up VF needs to do before recover */ 4826 amdgpu_virt_post_reset(adev); 4827 4828 /* Resume IP prior to SMC */ 4829 r = amdgpu_device_ip_reinit_early_sriov(adev); 4830 if (r) 4831 return r; 4832 4833 amdgpu_virt_init_data_exchange(adev); 4834 4835 r = amdgpu_device_fw_loading(adev); 4836 if (r) 4837 return r; 4838 4839 /* now we are okay to resume SMC/CP/SDMA */ 4840 r = amdgpu_device_ip_reinit_late_sriov(adev); 4841 if (r) 4842 return r; 4843 4844 hive = amdgpu_get_xgmi_hive(adev); 4845 /* Update PSP FW topology after reset */ 4846 if (hive && adev->gmc.xgmi.num_physical_nodes > 1) 4847 r = amdgpu_xgmi_update_topology(hive, adev); 4848 if (hive) 4849 amdgpu_put_xgmi_hive(hive); 4850 if (r) 4851 return r; 4852 4853 r = amdgpu_ib_ring_tests(adev); 4854 if (r) 4855 return r; 4856 4857 if (adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) 4858 amdgpu_inc_vram_lost(adev); 4859 4860 /* need to be called during full access so we can't do it later like 4861 * bare-metal does. 4862 */ 4863 amdgpu_amdkfd_post_reset(adev); 4864 amdgpu_virt_release_full_gpu(adev, true); 4865 4866 /* Aldebaran and gfx_11_0_3 support ras in SRIOV, so need resume ras during reset */ 4867 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) || 4868 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) || 4869 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4) || 4870 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0) || 4871 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3)) 4872 amdgpu_ras_resume(adev); 4873 4874 amdgpu_virt_ras_telemetry_post_reset(adev); 4875 4876 return 0; 4877 } 4878 4879 /** 4880 * amdgpu_device_has_job_running - check if there is any unfinished job 4881 * 4882 * @adev: amdgpu_device pointer 4883 * 4884 * check if there is any job running on the device when guest driver receives 4885 * FLR notification from host driver. If there are still jobs running, then 4886 * the guest driver will not respond the FLR reset. Instead, let the job hit 4887 * the timeout and guest driver then issue the reset request. 4888 */ 4889 bool amdgpu_device_has_job_running(struct amdgpu_device *adev) 4890 { 4891 int i; 4892 4893 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 4894 struct amdgpu_ring *ring = adev->rings[i]; 4895 4896 if (!amdgpu_ring_sched_ready(ring)) 4897 continue; 4898 4899 if (amdgpu_fence_count_emitted(ring)) 4900 return true; 4901 } 4902 return false; 4903 } 4904 4905 /** 4906 * amdgpu_device_should_recover_gpu - check if we should try GPU recovery 4907 * 4908 * @adev: amdgpu_device pointer 4909 * 4910 * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover 4911 * a hung GPU. 4912 */ 4913 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev) 4914 { 4915 4916 if (amdgpu_gpu_recovery == 0) 4917 goto disabled; 4918 4919 /* Skip soft reset check in fatal error mode */ 4920 if (!amdgpu_ras_is_poison_mode_supported(adev)) 4921 return true; 4922 4923 if (amdgpu_sriov_vf(adev)) 4924 return true; 4925 4926 if (amdgpu_gpu_recovery == -1) { 4927 switch (adev->asic_type) { 4928 #ifdef CONFIG_DRM_AMDGPU_SI 4929 case CHIP_VERDE: 4930 case CHIP_TAHITI: 4931 case CHIP_PITCAIRN: 4932 case CHIP_OLAND: 4933 case CHIP_HAINAN: 4934 #endif 4935 #ifdef CONFIG_DRM_AMDGPU_CIK 4936 case CHIP_KAVERI: 4937 case CHIP_KABINI: 4938 case CHIP_MULLINS: 4939 #endif 4940 case CHIP_CARRIZO: 4941 case CHIP_STONEY: 4942 case CHIP_CYAN_SKILLFISH: 4943 goto disabled; 4944 default: 4945 break; 4946 } 4947 } 4948 4949 return true; 4950 4951 disabled: 4952 dev_info(adev->dev, "GPU recovery disabled.\n"); 4953 return false; 4954 } 4955 4956 int amdgpu_device_mode1_reset(struct amdgpu_device *adev) 4957 { 4958 u32 i; 4959 int ret = 0; 4960 4961 if (adev->bios) 4962 amdgpu_atombios_scratch_regs_engine_hung(adev, true); 4963 4964 dev_info(adev->dev, "GPU mode1 reset\n"); 4965 4966 /* Cache the state before bus master disable. The saved config space 4967 * values are used in other cases like restore after mode-2 reset. 4968 */ 4969 amdgpu_device_cache_pci_state(adev->pdev); 4970 4971 /* disable BM */ 4972 pci_clear_master(adev->pdev); 4973 4974 if (amdgpu_dpm_is_mode1_reset_supported(adev)) { 4975 dev_info(adev->dev, "GPU smu mode1 reset\n"); 4976 ret = amdgpu_dpm_mode1_reset(adev); 4977 } else { 4978 dev_info(adev->dev, "GPU psp mode1 reset\n"); 4979 ret = psp_gpu_reset(adev); 4980 } 4981 4982 if (ret) 4983 goto mode1_reset_failed; 4984 4985 /* enable mmio access after mode 1 reset completed */ 4986 adev->no_hw_access = false; 4987 4988 /* ensure no_hw_access is updated before we access hw */ 4989 smp_mb(); 4990 4991 amdgpu_device_load_pci_state(adev->pdev); 4992 ret = amdgpu_psp_wait_for_bootloader(adev); 4993 if (ret) 4994 goto mode1_reset_failed; 4995 4996 /* wait for asic to come out of reset */ 4997 for (i = 0; i < adev->usec_timeout; i++) { 4998 u32 memsize = adev->nbio.funcs->get_memsize(adev); 4999 5000 if (memsize != 0xffffffff) 5001 break; 5002 udelay(1); 5003 } 5004 5005 if (i >= adev->usec_timeout) { 5006 ret = -ETIMEDOUT; 5007 goto mode1_reset_failed; 5008 } 5009 5010 if (adev->bios) 5011 amdgpu_atombios_scratch_regs_engine_hung(adev, false); 5012 5013 return 0; 5014 5015 mode1_reset_failed: 5016 dev_err(adev->dev, "GPU mode1 reset failed\n"); 5017 return ret; 5018 } 5019 5020 int amdgpu_device_link_reset(struct amdgpu_device *adev) 5021 { 5022 int ret = 0; 5023 5024 dev_info(adev->dev, "GPU link reset\n"); 5025 5026 if (!amdgpu_reset_in_dpc(adev)) 5027 ret = amdgpu_dpm_link_reset(adev); 5028 5029 if (ret) 5030 goto link_reset_failed; 5031 5032 ret = amdgpu_psp_wait_for_bootloader(adev); 5033 if (ret) 5034 goto link_reset_failed; 5035 5036 return 0; 5037 5038 link_reset_failed: 5039 dev_err(adev->dev, "GPU link reset failed\n"); 5040 return ret; 5041 } 5042 5043 int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, 5044 struct amdgpu_reset_context *reset_context) 5045 { 5046 struct amdgpu_job *job = NULL; 5047 struct dma_fence *fence = NULL; 5048 struct amdgpu_device *tmp_adev = reset_context->reset_req_dev; 5049 bool need_full_reset = 5050 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 5051 int i, r; 5052 5053 if (reset_context->reset_req_dev == adev) 5054 job = reset_context->job; 5055 5056 if (amdgpu_sriov_vf(adev)) 5057 amdgpu_virt_pre_reset(adev); 5058 5059 amdgpu_fence_driver_isr_toggle(adev, true); 5060 5061 if (job) 5062 fence = &job->hw_fence->base; 5063 5064 /* block all schedulers and reset given job's ring */ 5065 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 5066 struct amdgpu_ring *ring = adev->rings[i]; 5067 5068 if (!amdgpu_ring_sched_ready(ring)) 5069 continue; 5070 5071 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */ 5072 amdgpu_fence_driver_force_completion(ring, fence); 5073 } 5074 5075 /* 5076 * MES scheduler rings have no drm scheduler, so they are missed by the 5077 * loop above. Realign their polling fence too (one per XCC), otherwise the 5078 * first post-reset submission polls forever on a stale seq. sched.ready is 5079 * only set while the driver owns the ring. 5080 */ 5081 for (i = 0; i < AMDGPU_MAX_MES_INST_PIPES; i++) { 5082 struct amdgpu_ring *mes_ring = &adev->mes.ring[i]; 5083 5084 if (mes_ring->fence_drv.initialized && mes_ring->sched.ready) 5085 amdgpu_fence_driver_force_completion(mes_ring, fence); 5086 } 5087 5088 /* 5089 * KIQ rings are polling-fence/no_scheduler like MES, so realign their 5090 * fence too (one ring per XCC), otherwise the first post-reset KIQ 5091 * submission polls forever on a stale seq. 5092 */ 5093 for (i = 0; i < AMDGPU_MAX_GC_INSTANCES; i++) { 5094 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[i].ring; 5095 5096 if (kiq_ring->fence_drv.initialized && kiq_ring->sched.ready) 5097 amdgpu_fence_driver_force_completion(kiq_ring, fence); 5098 } 5099 5100 amdgpu_fence_driver_isr_toggle(adev, false); 5101 5102 r = amdgpu_reset_prepare_hwcontext(adev, reset_context); 5103 /* If reset handler not implemented, continue; otherwise return */ 5104 if (r == -EOPNOTSUPP) 5105 r = 0; 5106 else 5107 return r; 5108 5109 /* Don't suspend on bare metal if we are not going to HW reset the ASIC */ 5110 if (!amdgpu_sriov_vf(adev)) { 5111 need_full_reset = true; 5112 5113 if (!test_bit(AMDGPU_SKIP_COREDUMP, &reset_context->flags)) { 5114 dev_info(tmp_adev->dev, "Dumping IP State\n"); 5115 /* Trigger ip dump before we reset the asic */ 5116 for (i = 0; i < tmp_adev->num_ip_blocks; i++) 5117 if (tmp_adev->ip_blocks[i].version->funcs->dump_ip_state) 5118 tmp_adev->ip_blocks[i].version->funcs 5119 ->dump_ip_state((void *)&tmp_adev->ip_blocks[i]); 5120 dev_info(tmp_adev->dev, "Dumping IP State Completed\n"); 5121 } 5122 5123 if (need_full_reset) 5124 r = amdgpu_device_ip_suspend(adev); 5125 if (need_full_reset) 5126 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 5127 else 5128 clear_bit(AMDGPU_NEED_FULL_RESET, 5129 &reset_context->flags); 5130 } 5131 5132 return r; 5133 } 5134 5135 int amdgpu_device_reinit_after_reset(struct amdgpu_reset_context *reset_context) 5136 { 5137 struct list_head *device_list_handle; 5138 bool full_reset, vram_lost = false; 5139 struct amdgpu_device *tmp_adev; 5140 int r, init_level; 5141 5142 device_list_handle = reset_context->reset_device_list; 5143 5144 if (!device_list_handle) 5145 return -EINVAL; 5146 5147 full_reset = test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 5148 5149 /** 5150 * If it's reset on init, it's default init level, otherwise keep level 5151 * as recovery level. 5152 */ 5153 if (reset_context->method == AMD_RESET_METHOD_ON_INIT) 5154 init_level = AMDGPU_INIT_LEVEL_DEFAULT; 5155 else 5156 init_level = AMDGPU_INIT_LEVEL_RESET_RECOVERY; 5157 5158 r = 0; 5159 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 5160 amdgpu_set_init_level(tmp_adev, init_level); 5161 if (full_reset) { 5162 /* post card */ 5163 amdgpu_reset_set_dpc_status(tmp_adev, false); 5164 amdgpu_ras_clear_err_state(tmp_adev); 5165 r = amdgpu_device_asic_init(tmp_adev); 5166 if (r) { 5167 dev_warn(tmp_adev->dev, "asic atom init failed!"); 5168 } else { 5169 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n"); 5170 5171 r = amdgpu_device_ip_resume_phase1(tmp_adev); 5172 if (r) 5173 goto out; 5174 5175 vram_lost = amdgpu_device_check_vram_lost(tmp_adev); 5176 5177 if (!test_bit(AMDGPU_SKIP_COREDUMP, &reset_context->flags)) 5178 amdgpu_coredump(tmp_adev, false, vram_lost, reset_context->job); 5179 5180 if (vram_lost) { 5181 dev_info( 5182 tmp_adev->dev, 5183 "VRAM is lost due to GPU reset!\n"); 5184 amdgpu_inc_vram_lost(tmp_adev); 5185 } 5186 5187 r = amdgpu_device_fw_loading(tmp_adev); 5188 if (r) 5189 return r; 5190 5191 r = amdgpu_xcp_restore_partition_mode( 5192 tmp_adev->xcp_mgr); 5193 if (r) 5194 goto out; 5195 5196 r = amdgpu_device_ip_resume_phase2(tmp_adev); 5197 if (r) 5198 goto out; 5199 5200 r = amdgpu_device_ip_resume_phase3(tmp_adev); 5201 if (r) 5202 goto out; 5203 5204 if (vram_lost) 5205 amdgpu_device_fill_reset_magic(tmp_adev); 5206 5207 /* 5208 * Add this ASIC as tracked as reset was already 5209 * complete successfully. 5210 */ 5211 amdgpu_register_gpu_instance(tmp_adev); 5212 5213 if (!reset_context->hive && 5214 tmp_adev->gmc.xgmi.num_physical_nodes > 1) 5215 amdgpu_xgmi_add_device(tmp_adev); 5216 5217 r = amdgpu_device_ip_late_init(tmp_adev); 5218 if (r) 5219 goto out; 5220 5221 r = amdgpu_userq_post_reset(tmp_adev, vram_lost); 5222 if (r) 5223 goto out; 5224 5225 drm_client_dev_resume(adev_to_drm(tmp_adev)); 5226 5227 /* 5228 * The GPU enters bad state once faulty pages 5229 * by ECC has reached the threshold, and ras 5230 * recovery is scheduled next. So add one check 5231 * here to break recovery if it indeed exceeds 5232 * bad page threshold, and remind user to 5233 * retire this GPU or setting one bigger 5234 * bad_page_threshold value to fix this once 5235 * probing driver again. 5236 */ 5237 if (!amdgpu_ras_is_rma(tmp_adev)) { 5238 /* must succeed. */ 5239 amdgpu_ras_resume(tmp_adev); 5240 } else { 5241 r = -EINVAL; 5242 goto out; 5243 } 5244 5245 /* Update PSP FW topology after reset */ 5246 if (reset_context->hive && 5247 tmp_adev->gmc.xgmi.num_physical_nodes > 1) 5248 r = amdgpu_xgmi_update_topology( 5249 reset_context->hive, tmp_adev); 5250 } 5251 } 5252 5253 out: 5254 if (!r) { 5255 /* IP init is complete now, set level as default */ 5256 amdgpu_set_init_level(tmp_adev, 5257 AMDGPU_INIT_LEVEL_DEFAULT); 5258 amdgpu_irq_gpu_reset_resume_helper(tmp_adev); 5259 r = amdgpu_ib_ring_tests(tmp_adev); 5260 if (r) { 5261 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r); 5262 r = -EAGAIN; 5263 goto end; 5264 } 5265 } 5266 5267 if (r) 5268 tmp_adev->asic_reset_res = r; 5269 } 5270 5271 end: 5272 return r; 5273 } 5274 5275 int amdgpu_do_asic_reset(struct list_head *device_list_handle, 5276 struct amdgpu_reset_context *reset_context) 5277 { 5278 struct amdgpu_device *tmp_adev = NULL; 5279 bool need_full_reset, skip_hw_reset; 5280 int r = 0; 5281 5282 /* Try reset handler method first */ 5283 tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device, 5284 reset_list); 5285 5286 reset_context->reset_device_list = device_list_handle; 5287 r = amdgpu_reset_perform_reset(tmp_adev, reset_context); 5288 /* If reset handler not implemented, continue; otherwise return */ 5289 if (r == -EOPNOTSUPP) 5290 r = 0; 5291 else 5292 return r; 5293 5294 /* Reset handler not implemented, use the default method */ 5295 need_full_reset = 5296 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 5297 skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags); 5298 5299 /* 5300 * ASIC reset has to be done on all XGMI hive nodes ASAP 5301 * to allow proper links negotiation in FW (within 1 sec) 5302 */ 5303 if (!skip_hw_reset && need_full_reset) { 5304 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 5305 /* For XGMI run all resets in parallel to speed up the process */ 5306 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) { 5307 if (!queue_work(system_dfl_wq, 5308 &tmp_adev->xgmi_reset_work)) 5309 r = -EALREADY; 5310 } else 5311 r = amdgpu_asic_reset(tmp_adev); 5312 5313 if (r) { 5314 dev_err(tmp_adev->dev, 5315 "ASIC reset failed with error, %d for drm dev, %s", 5316 r, adev_to_drm(tmp_adev)->unique); 5317 goto out; 5318 } 5319 } 5320 5321 /* For XGMI wait for all resets to complete before proceed */ 5322 if (!r) { 5323 list_for_each_entry(tmp_adev, device_list_handle, 5324 reset_list) { 5325 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) { 5326 flush_work(&tmp_adev->xgmi_reset_work); 5327 r = tmp_adev->asic_reset_res; 5328 if (r) 5329 break; 5330 } 5331 } 5332 } 5333 } 5334 5335 if (!r && amdgpu_ras_intr_triggered()) { 5336 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 5337 amdgpu_ras_reset_error_count(tmp_adev, 5338 AMDGPU_RAS_BLOCK__MMHUB); 5339 } 5340 5341 amdgpu_ras_intr_cleared(); 5342 } 5343 5344 r = amdgpu_device_reinit_after_reset(reset_context); 5345 if (r == -EAGAIN) 5346 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 5347 else 5348 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 5349 5350 out: 5351 return r; 5352 } 5353 5354 static void amdgpu_device_set_mp1_state(struct amdgpu_device *adev) 5355 { 5356 5357 switch (amdgpu_asic_reset_method(adev)) { 5358 case AMD_RESET_METHOD_MODE1: 5359 case AMD_RESET_METHOD_LINK: 5360 adev->mp1_state = PP_MP1_STATE_SHUTDOWN; 5361 break; 5362 case AMD_RESET_METHOD_MODE2: 5363 adev->mp1_state = PP_MP1_STATE_RESET; 5364 break; 5365 default: 5366 adev->mp1_state = PP_MP1_STATE_NONE; 5367 break; 5368 } 5369 } 5370 5371 static void amdgpu_device_unset_mp1_state(struct amdgpu_device *adev) 5372 { 5373 amdgpu_vf_error_trans_all(adev); 5374 adev->mp1_state = PP_MP1_STATE_NONE; 5375 } 5376 5377 static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev) 5378 { 5379 struct pci_dev *p = NULL; 5380 5381 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus), 5382 adev->pdev->bus->number, 1); 5383 if (p) { 5384 pm_runtime_enable(&(p->dev)); 5385 pm_runtime_resume(&(p->dev)); 5386 } 5387 5388 pci_dev_put(p); 5389 } 5390 5391 static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev) 5392 { 5393 enum amd_reset_method reset_method; 5394 struct pci_dev *p = NULL; 5395 u64 expires; 5396 5397 /* 5398 * For now, only BACO and mode1 reset are confirmed 5399 * to suffer the audio issue without proper suspended. 5400 */ 5401 reset_method = amdgpu_asic_reset_method(adev); 5402 if ((reset_method != AMD_RESET_METHOD_BACO) && 5403 (reset_method != AMD_RESET_METHOD_MODE1)) 5404 return -EINVAL; 5405 5406 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus), 5407 adev->pdev->bus->number, 1); 5408 if (!p) 5409 return -ENODEV; 5410 5411 expires = pm_runtime_autosuspend_expiration(&(p->dev)); 5412 if (!expires) 5413 /* 5414 * If we cannot get the audio device autosuspend delay, 5415 * a fixed 4S interval will be used. Considering 3S is 5416 * the audio controller default autosuspend delay setting. 5417 * 4S used here is guaranteed to cover that. 5418 */ 5419 expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL; 5420 5421 while (!pm_runtime_status_suspended(&(p->dev))) { 5422 if (!pm_runtime_suspend(&(p->dev))) 5423 break; 5424 5425 if (expires < ktime_get_mono_fast_ns()) { 5426 dev_warn(adev->dev, "failed to suspend display audio\n"); 5427 pci_dev_put(p); 5428 /* TODO: abort the succeeding gpu reset? */ 5429 return -ETIMEDOUT; 5430 } 5431 } 5432 5433 pm_runtime_disable(&(p->dev)); 5434 5435 pci_dev_put(p); 5436 return 0; 5437 } 5438 5439 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev) 5440 { 5441 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5442 5443 #if defined(CONFIG_DEBUG_FS) 5444 if (!amdgpu_sriov_vf(adev)) 5445 cancel_work(&adev->reset_work); 5446 #endif 5447 amdgpu_userq_mgr_cancel_reset_work(adev); 5448 5449 if (adev->kfd.dev) 5450 cancel_work(&adev->kfd.reset_work); 5451 5452 if (amdgpu_sriov_vf(adev)) 5453 cancel_work(&adev->virt.flr_work); 5454 5455 if (con && adev->ras_enabled) 5456 cancel_work(&con->recovery_work); 5457 5458 } 5459 5460 static int amdgpu_device_health_check(struct list_head *device_list_handle) 5461 { 5462 struct amdgpu_device *tmp_adev; 5463 int ret = 0; 5464 5465 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 5466 ret |= amdgpu_device_bus_status_check(tmp_adev); 5467 } 5468 5469 return ret; 5470 } 5471 5472 static void amdgpu_device_recovery_prepare(struct amdgpu_device *adev, 5473 struct list_head *device_list, 5474 struct amdgpu_hive_info *hive) 5475 { 5476 struct amdgpu_device *tmp_adev = NULL; 5477 5478 /* 5479 * Build list of devices to reset. 5480 * In case we are in XGMI hive mode, resort the device list 5481 * to put adev in the 1st position. 5482 */ 5483 if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1) && hive) { 5484 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { 5485 list_add_tail(&tmp_adev->reset_list, device_list); 5486 if (adev->shutdown) 5487 tmp_adev->shutdown = true; 5488 } 5489 if (!list_is_first(&adev->reset_list, device_list)) 5490 list_rotate_to_front(&adev->reset_list, device_list); 5491 } else { 5492 list_add_tail(&adev->reset_list, device_list); 5493 } 5494 } 5495 5496 static void amdgpu_device_recovery_get_reset_lock(struct amdgpu_device *adev, 5497 struct list_head *device_list) 5498 { 5499 struct amdgpu_device *tmp_adev = NULL; 5500 5501 if (list_empty(device_list)) 5502 return; 5503 tmp_adev = 5504 list_first_entry(device_list, struct amdgpu_device, reset_list); 5505 amdgpu_device_lock_reset_domain(tmp_adev->reset_domain); 5506 } 5507 5508 static void amdgpu_device_recovery_put_reset_lock(struct amdgpu_device *adev, 5509 struct list_head *device_list) 5510 { 5511 struct amdgpu_device *tmp_adev = NULL; 5512 5513 if (list_empty(device_list)) 5514 return; 5515 tmp_adev = 5516 list_first_entry(device_list, struct amdgpu_device, reset_list); 5517 amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain); 5518 } 5519 5520 static void amdgpu_device_halt_activities(struct amdgpu_device *adev, 5521 struct amdgpu_job *job, 5522 struct amdgpu_reset_context *reset_context, 5523 struct list_head *device_list, 5524 struct amdgpu_hive_info *hive, 5525 bool need_emergency_restart) 5526 { 5527 struct amdgpu_device *tmp_adev = NULL; 5528 int i; 5529 5530 /* Stop job submission first, the preparation below can take seconds */ 5531 list_for_each_entry(tmp_adev, device_list, reset_list) { 5532 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 5533 struct amdgpu_ring *ring = tmp_adev->rings[i]; 5534 5535 if (!amdgpu_ring_sched_ready(ring)) 5536 continue; 5537 5538 drm_sched_wqueue_stop(&ring->sched); 5539 5540 if (need_emergency_restart) 5541 amdgpu_job_stop_all_jobs_on_sched(&ring->sched); 5542 } 5543 } 5544 5545 list_for_each_entry(tmp_adev, device_list, reset_list) { 5546 amdgpu_device_set_mp1_state(tmp_adev); 5547 5548 /* 5549 * Try to put the audio codec into suspend state 5550 * before gpu reset started. 5551 * 5552 * Due to the power domain of the graphics device 5553 * is shared with AZ power domain. Without this, 5554 * we may change the audio hardware from behind 5555 * the audio driver's back. That will trigger 5556 * some audio codec errors. 5557 */ 5558 if (!amdgpu_device_suspend_display_audio(tmp_adev)) 5559 tmp_adev->pcie_reset_ctx.audio_suspended = true; 5560 5561 amdgpu_ras_set_error_query_ready(tmp_adev, false); 5562 5563 cancel_delayed_work_sync(&tmp_adev->delayed_init_work); 5564 5565 amdgpu_amdkfd_pre_reset(tmp_adev, reset_context); 5566 5567 /* 5568 * Mark these ASICs to be reset as untracked first 5569 * And add them back after reset completed 5570 */ 5571 amdgpu_unregister_gpu_instance(tmp_adev); 5572 5573 drm_client_dev_suspend(adev_to_drm(tmp_adev)); 5574 5575 /* disable ras on ALL IPs */ 5576 if (!need_emergency_restart && !amdgpu_reset_in_dpc(adev)) 5577 amdgpu_ras_suspend(tmp_adev); 5578 5579 amdgpu_userq_pre_reset(tmp_adev); 5580 atomic_inc(&tmp_adev->gpu_reset_counter); 5581 } 5582 } 5583 5584 static int amdgpu_device_asic_reset(struct amdgpu_device *adev, 5585 struct list_head *device_list, 5586 struct amdgpu_reset_context *reset_context) 5587 { 5588 struct amdgpu_device *tmp_adev = NULL; 5589 int retry_limit = AMDGPU_MAX_RETRY_LIMIT; 5590 int r = 0; 5591 5592 retry: /* Rest of adevs pre asic reset from XGMI hive. */ 5593 list_for_each_entry(tmp_adev, device_list, reset_list) { 5594 r = amdgpu_device_pre_asic_reset(tmp_adev, reset_context); 5595 /*TODO Should we stop ?*/ 5596 if (r) { 5597 dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ", 5598 r, adev_to_drm(tmp_adev)->unique); 5599 tmp_adev->asic_reset_res = r; 5600 } 5601 } 5602 5603 /* Actual ASIC resets if needed.*/ 5604 /* Host driver will handle XGMI hive reset for SRIOV */ 5605 if (amdgpu_sriov_vf(adev)) { 5606 5607 /* Bail out of reset early */ 5608 if (amdgpu_ras_is_rma(adev)) 5609 return -ENODEV; 5610 5611 if (amdgpu_ras_get_fed_status(adev) || amdgpu_virt_rcvd_ras_interrupt(adev)) { 5612 dev_dbg(adev->dev, "Detected RAS error, wait for FLR completion\n"); 5613 amdgpu_ras_set_fed(adev, true); 5614 set_bit(AMDGPU_HOST_FLR, &reset_context->flags); 5615 } 5616 5617 r = amdgpu_device_reset_sriov(adev, reset_context); 5618 if (AMDGPU_RETRY_SRIOV_RESET(r) && (retry_limit--) > 0) { 5619 amdgpu_virt_release_full_gpu(adev, true); 5620 goto retry; 5621 } 5622 if (r) 5623 adev->asic_reset_res = r; 5624 } else { 5625 r = amdgpu_do_asic_reset(device_list, reset_context); 5626 if (r && r == -EAGAIN) 5627 goto retry; 5628 } 5629 5630 list_for_each_entry(tmp_adev, device_list, reset_list) { 5631 /* 5632 * Drop any pending non scheduler resets queued before reset is done. 5633 * Any reset scheduled after this point would be valid. Scheduler resets 5634 * were already dropped during drm_sched_stop and no new ones can come 5635 * in before drm_sched_start. 5636 */ 5637 amdgpu_device_stop_pending_resets(tmp_adev); 5638 } 5639 5640 return r; 5641 } 5642 5643 static int amdgpu_device_sched_resume(struct list_head *device_list, 5644 struct amdgpu_reset_context *reset_context, 5645 bool job_signaled) 5646 { 5647 struct amdgpu_device *tmp_adev = NULL; 5648 int i, r = 0; 5649 5650 /* Post ASIC reset for all devs .*/ 5651 list_for_each_entry(tmp_adev, device_list, reset_list) { 5652 5653 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 5654 struct amdgpu_ring *ring = tmp_adev->rings[i]; 5655 5656 if (!amdgpu_ring_sched_ready(ring)) 5657 continue; 5658 5659 drm_sched_wqueue_start(&ring->sched); 5660 } 5661 5662 if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled) 5663 drm_helper_resume_force_mode(adev_to_drm(tmp_adev)); 5664 5665 if (tmp_adev->asic_reset_res) { 5666 /* bad news, how to tell it to userspace ? 5667 * for ras error, we should report GPU bad status instead of 5668 * reset failure 5669 */ 5670 if (reset_context->src != AMDGPU_RESET_SRC_RAS || 5671 !amdgpu_ras_eeprom_check_err_threshold(tmp_adev)) 5672 dev_info( 5673 tmp_adev->dev, 5674 "GPU reset(%d) failed with error %d\n", 5675 atomic_read( 5676 &tmp_adev->gpu_reset_counter), 5677 tmp_adev->asic_reset_res); 5678 amdgpu_vf_error_put(tmp_adev, 5679 AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, 5680 tmp_adev->asic_reset_res); 5681 if (!r) 5682 r = tmp_adev->asic_reset_res; 5683 tmp_adev->asic_reset_res = 0; 5684 } else { 5685 dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", 5686 atomic_read(&tmp_adev->gpu_reset_counter)); 5687 if (amdgpu_acpi_smart_shift_update(tmp_adev, 5688 AMDGPU_SS_DEV_D0)) 5689 dev_warn(tmp_adev->dev, 5690 "smart shift update failed\n"); 5691 } 5692 } 5693 5694 return r; 5695 } 5696 5697 static void amdgpu_device_gpu_resume(struct amdgpu_device *adev, 5698 struct list_head *device_list, 5699 bool need_emergency_restart) 5700 { 5701 struct amdgpu_device *tmp_adev = NULL; 5702 5703 list_for_each_entry(tmp_adev, device_list, reset_list) { 5704 /* unlock kfd: SRIOV would do it separately */ 5705 if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev)) 5706 amdgpu_amdkfd_post_reset(tmp_adev); 5707 5708 /* kfd_post_reset will do nothing if kfd device is not initialized, 5709 * need to bring up kfd here if it's not be initialized before 5710 */ 5711 if (!adev->kfd.init_complete) 5712 amdgpu_amdkfd_device_init(adev); 5713 5714 if (tmp_adev->pcie_reset_ctx.audio_suspended) 5715 amdgpu_device_resume_display_audio(tmp_adev); 5716 5717 amdgpu_device_unset_mp1_state(tmp_adev); 5718 5719 amdgpu_ras_set_error_query_ready(tmp_adev, true); 5720 5721 } 5722 } 5723 5724 5725 /** 5726 * amdgpu_device_gpu_recover - reset the asic and recover scheduler 5727 * 5728 * @adev: amdgpu_device pointer 5729 * @job: which job trigger hang 5730 * @reset_context: amdgpu reset context pointer 5731 * 5732 * Attempt to reset the GPU if it has hung (all asics). 5733 * Attempt to do soft-reset or full-reset and reinitialize Asic 5734 * Returns 0 for success or an error on failure. 5735 */ 5736 5737 int amdgpu_device_gpu_recover(struct amdgpu_device *adev, 5738 struct amdgpu_job *job, 5739 struct amdgpu_reset_context *reset_context) 5740 { 5741 struct list_head device_list; 5742 bool job_signaled = false; 5743 struct amdgpu_hive_info *hive = NULL; 5744 int r = 0; 5745 bool need_emergency_restart = false; 5746 /* save the pasid here as the job may be freed before the end of the reset */ 5747 int pasid = job ? job->pasid : -EINVAL; 5748 5749 /* 5750 * If it reaches here because of hang/timeout and a RAS error is 5751 * detected at the same time, let RAS recovery take care of it. 5752 */ 5753 if (amdgpu_ras_is_err_state(adev, AMDGPU_RAS_BLOCK__ANY) && 5754 !amdgpu_sriov_vf(adev) && 5755 reset_context->src != AMDGPU_RESET_SRC_RAS) { 5756 dev_dbg(adev->dev, 5757 "Gpu recovery from source: %d yielding to RAS error recovery handling", 5758 reset_context->src); 5759 return 0; 5760 } 5761 5762 /* 5763 * Special case: RAS triggered and full reset isn't supported 5764 */ 5765 need_emergency_restart = amdgpu_ras_need_emergency_restart(adev); 5766 5767 /* 5768 * Flush RAM to disk so that after reboot 5769 * the user can read log and see why the system rebooted. 5770 */ 5771 if (need_emergency_restart && amdgpu_ras_get_context(adev) && 5772 amdgpu_ras_get_context(adev)->reboot) { 5773 dev_warn(adev->dev, "Emergency reboot."); 5774 5775 ksys_sync_helper(); 5776 emergency_restart(); 5777 } 5778 5779 dev_info(adev->dev, "GPU %s begin!. Source: %d\n", 5780 need_emergency_restart ? "jobs stop" : "reset", 5781 reset_context->src); 5782 5783 if (!amdgpu_sriov_vf(adev)) 5784 hive = amdgpu_get_xgmi_hive(adev); 5785 if (hive) 5786 mutex_lock(&hive->hive_lock); 5787 5788 reset_context->job = job; 5789 reset_context->hive = hive; 5790 INIT_LIST_HEAD(&device_list); 5791 5792 amdgpu_device_recovery_prepare(adev, &device_list, hive); 5793 5794 if (!amdgpu_sriov_vf(adev)) { 5795 r = amdgpu_device_health_check(&device_list); 5796 if (r) 5797 goto end_reset; 5798 } 5799 5800 /* Cannot be called after locking reset domain */ 5801 amdgpu_ras_pre_reset(adev, &device_list); 5802 5803 /* We need to lock reset domain only once both for XGMI and single device */ 5804 amdgpu_device_recovery_get_reset_lock(adev, &device_list); 5805 5806 /* unmap all the mappings of doorbell and framebuffer to prevent user space from 5807 * accessing them 5808 */ 5809 unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1); 5810 amdgpu_amdkfd_clear_kfd_mapping(adev); 5811 5812 amdgpu_device_halt_activities(adev, job, reset_context, &device_list, 5813 hive, need_emergency_restart); 5814 if (need_emergency_restart) { 5815 amdgpu_device_gpu_resume(adev, &device_list, need_emergency_restart); 5816 goto reset_unlock; 5817 } 5818 /* 5819 * Must check guilty signal here since after this point all old 5820 * HW fences are force signaled. 5821 * 5822 * job->base holds a reference to parent fence 5823 */ 5824 if (job && (dma_fence_get_status(&job->hw_fence->base) > 0)) { 5825 job_signaled = true; 5826 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset"); 5827 goto post_resume; 5828 } 5829 5830 r = amdgpu_device_asic_reset(adev, &device_list, reset_context); 5831 if (r) 5832 goto reset_unlock; 5833 post_resume: 5834 amdgpu_device_gpu_resume(adev, &device_list, need_emergency_restart); 5835 r = amdgpu_device_sched_resume(&device_list, reset_context, job_signaled); 5836 if (r) 5837 goto reset_unlock; 5838 reset_unlock: 5839 amdgpu_device_recovery_put_reset_lock(adev, &device_list); 5840 amdgpu_ras_post_reset(adev, &device_list); 5841 end_reset: 5842 if (hive) { 5843 mutex_unlock(&hive->hive_lock); 5844 amdgpu_put_xgmi_hive(hive); 5845 } 5846 5847 if (r) 5848 dev_info(adev->dev, "GPU reset end with ret = %d\n", r); 5849 5850 atomic_set(&adev->reset_domain->reset_res, r); 5851 5852 if (!r) { 5853 struct amdgpu_task_info *ti = NULL; 5854 5855 /* 5856 * The job may already be freed at this point via the sched tdr workqueue so 5857 * use the cached pasid. 5858 */ 5859 if (pasid >= 0) 5860 ti = amdgpu_vm_get_task_info_pasid(adev, pasid); 5861 5862 drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE, 5863 ti ? &ti->task : NULL); 5864 5865 amdgpu_vm_put_task_info(ti); 5866 } 5867 5868 return r; 5869 } 5870 5871 /** 5872 * amdgpu_device_partner_bandwidth - find the bandwidth of appropriate partner 5873 * 5874 * @adev: amdgpu_device pointer 5875 * @speed: pointer to the speed of the link 5876 * @width: pointer to the width of the link 5877 * 5878 * Evaluate the hierarchy to find the speed and bandwidth capabilities of the 5879 * first physical partner to an AMD dGPU. 5880 * This will exclude any virtual switches and links. 5881 */ 5882 static void amdgpu_device_partner_bandwidth(struct amdgpu_device *adev, 5883 enum pci_bus_speed *speed, 5884 enum pcie_link_width *width) 5885 { 5886 if (!speed || !width) 5887 return; 5888 5889 *speed = PCI_SPEED_UNKNOWN; 5890 *width = PCIE_LNK_WIDTH_UNKNOWN; 5891 5892 if (amdgpu_device_pcie_dynamic_switching_supported(adev)) { 5893 if (adev->link_partner) { 5894 *speed = pcie_get_speed_cap(adev->link_partner); 5895 *width = pcie_get_width_cap(adev->link_partner); 5896 } 5897 } else { 5898 /* use the current speeds rather than max if switching is not supported */ 5899 pcie_bandwidth_available(adev->pdev, NULL, speed, width); 5900 } 5901 } 5902 5903 /** 5904 * amdgpu_device_gpu_bandwidth - find the bandwidth of the GPU 5905 * 5906 * @adev: amdgpu_device pointer 5907 * @speed: pointer to the speed of the link 5908 * @width: pointer to the width of the link 5909 * 5910 * Evaluate the hierarchy to find the speed and bandwidth capabilities of the 5911 * AMD dGPU which may be a virtual upstream bridge. 5912 */ 5913 static void amdgpu_device_gpu_bandwidth(struct amdgpu_device *adev, 5914 enum pci_bus_speed *speed, 5915 enum pcie_link_width *width) 5916 { 5917 if (!speed || !width) 5918 return; 5919 5920 *speed = pcie_get_speed_cap(adev->link_dev); 5921 *width = pcie_get_width_cap(adev->link_dev); 5922 } 5923 5924 /** 5925 * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot 5926 * 5927 * @adev: amdgpu_device pointer 5928 * 5929 * Fetches and stores in the driver the PCIE capabilities (gen speed 5930 * and lanes) of the slot the device is in. Handles APUs and 5931 * virtualized environments where PCIE config space may not be available. 5932 */ 5933 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev) 5934 { 5935 enum pci_bus_speed speed_cap, platform_speed_cap; 5936 enum pcie_link_width platform_link_width, link_width; 5937 5938 if (amdgpu_pcie_gen_cap) 5939 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap; 5940 5941 if (amdgpu_pcie_lane_cap) 5942 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap; 5943 5944 /* covers APUs as well */ 5945 if (pci_is_root_bus(adev->pdev->bus) && !amdgpu_passthrough(adev)) { 5946 if (adev->pm.pcie_gen_mask == 0) 5947 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK; 5948 if (adev->pm.pcie_mlw_mask == 0) 5949 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK; 5950 return; 5951 } 5952 5953 if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask) 5954 return; 5955 5956 amdgpu_device_partner_bandwidth(adev, &platform_speed_cap, 5957 &platform_link_width); 5958 amdgpu_device_gpu_bandwidth(adev, &speed_cap, &link_width); 5959 5960 if (adev->pm.pcie_gen_mask == 0) { 5961 /* asic caps */ 5962 if (speed_cap == PCI_SPEED_UNKNOWN) { 5963 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 5964 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 5965 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3); 5966 } else { 5967 if (speed_cap == PCIE_SPEED_32_0GT) 5968 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 5969 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 5970 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 | 5971 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 | 5972 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5); 5973 else if (speed_cap == PCIE_SPEED_16_0GT) 5974 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 5975 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 5976 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 | 5977 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4); 5978 else if (speed_cap == PCIE_SPEED_8_0GT) 5979 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 5980 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 5981 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3); 5982 else if (speed_cap == PCIE_SPEED_5_0GT) 5983 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 5984 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2); 5985 else 5986 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1; 5987 } 5988 /* platform caps */ 5989 if (platform_speed_cap == PCI_SPEED_UNKNOWN) { 5990 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 5991 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2); 5992 } else { 5993 if (platform_speed_cap == PCIE_SPEED_32_0GT) 5994 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 5995 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | 5996 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 | 5997 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 | 5998 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5); 5999 else if (platform_speed_cap == PCIE_SPEED_16_0GT) 6000 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6001 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | 6002 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 | 6003 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4); 6004 else if (platform_speed_cap == PCIE_SPEED_8_0GT) 6005 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6006 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | 6007 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3); 6008 else if (platform_speed_cap == PCIE_SPEED_5_0GT) 6009 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 6010 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2); 6011 else 6012 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1; 6013 6014 } 6015 } 6016 if (adev->pm.pcie_mlw_mask == 0) { 6017 /* asic caps */ 6018 if (link_width == PCIE_LNK_WIDTH_UNKNOWN) { 6019 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_ASIC_PCIE_MLW_MASK; 6020 } else { 6021 switch (link_width) { 6022 case PCIE_LNK_X32: 6023 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X32 | 6024 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X16 | 6025 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X12 | 6026 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 | 6027 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 | 6028 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 | 6029 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1); 6030 break; 6031 case PCIE_LNK_X16: 6032 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X16 | 6033 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X12 | 6034 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 | 6035 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 | 6036 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 | 6037 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1); 6038 break; 6039 case PCIE_LNK_X12: 6040 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X12 | 6041 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 | 6042 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 | 6043 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 | 6044 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1); 6045 break; 6046 case PCIE_LNK_X8: 6047 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X8 | 6048 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 | 6049 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 | 6050 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1); 6051 break; 6052 case PCIE_LNK_X4: 6053 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X4 | 6054 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 | 6055 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1); 6056 break; 6057 case PCIE_LNK_X2: 6058 adev->pm.pcie_mlw_mask |= (CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X2 | 6059 CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1); 6060 break; 6061 case PCIE_LNK_X1: 6062 adev->pm.pcie_mlw_mask |= CAIL_ASIC_PCIE_LINK_WIDTH_SUPPORT_X1; 6063 break; 6064 default: 6065 break; 6066 } 6067 } 6068 /* platform caps */ 6069 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) { 6070 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK; 6071 } else { 6072 switch (platform_link_width) { 6073 case PCIE_LNK_X32: 6074 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 | 6075 CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 | 6076 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | 6077 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 6078 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 6079 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 6080 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 6081 break; 6082 case PCIE_LNK_X16: 6083 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 | 6084 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | 6085 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 6086 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 6087 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 6088 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 6089 break; 6090 case PCIE_LNK_X12: 6091 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | 6092 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 6093 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 6094 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 6095 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 6096 break; 6097 case PCIE_LNK_X8: 6098 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 6099 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 6100 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 6101 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 6102 break; 6103 case PCIE_LNK_X4: 6104 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 6105 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 6106 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 6107 break; 6108 case PCIE_LNK_X2: 6109 adev->pm.pcie_mlw_mask |= (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 6110 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 6111 break; 6112 case PCIE_LNK_X1: 6113 adev->pm.pcie_mlw_mask |= CAIL_PCIE_LINK_WIDTH_SUPPORT_X1; 6114 break; 6115 default: 6116 break; 6117 } 6118 } 6119 } 6120 } 6121 6122 /** 6123 * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR 6124 * 6125 * @adev: amdgpu_device pointer 6126 * @peer_adev: amdgpu_device pointer for peer device trying to access @adev 6127 * 6128 * Return true if @peer_adev can access (DMA) @adev through the PCIe 6129 * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of 6130 * @peer_adev. 6131 */ 6132 bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev, 6133 struct amdgpu_device *peer_adev) 6134 { 6135 #ifdef CONFIG_HSA_AMD_P2P 6136 bool p2p_access = 6137 !adev->gmc.xgmi.connected_to_cpu && 6138 !(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0); 6139 if (!p2p_access) 6140 dev_info(adev->dev, "PCIe P2P access from peer device %s is not supported by the chipset\n", 6141 pci_name(peer_adev->pdev)); 6142 6143 bool is_large_bar = adev->gmc.visible_vram_size && 6144 adev->gmc.real_vram_size == adev->gmc.visible_vram_size; 6145 bool p2p_addressable = amdgpu_device_check_iommu_remap(peer_adev); 6146 6147 if (!p2p_addressable) { 6148 uint64_t address_mask = peer_adev->dev->dma_mask ? 6149 ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1); 6150 resource_size_t aper_limit = 6151 adev->gmc.aper_base + adev->gmc.aper_size - 1; 6152 6153 p2p_addressable = !(adev->gmc.aper_base & address_mask || 6154 aper_limit & address_mask); 6155 } 6156 return pcie_p2p && is_large_bar && p2p_access && p2p_addressable; 6157 #else 6158 return false; 6159 #endif 6160 } 6161 6162 int amdgpu_device_baco_enter(struct amdgpu_device *adev) 6163 { 6164 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 6165 6166 if (!amdgpu_device_supports_baco(adev)) 6167 return -ENOTSUPP; 6168 6169 if (ras && adev->ras_enabled && 6170 adev->nbio.funcs->enable_doorbell_interrupt) 6171 adev->nbio.funcs->enable_doorbell_interrupt(adev, false); 6172 6173 return amdgpu_dpm_baco_enter(adev); 6174 } 6175 6176 int amdgpu_device_baco_exit(struct amdgpu_device *adev) 6177 { 6178 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 6179 int ret = 0; 6180 6181 if (!amdgpu_device_supports_baco(adev)) 6182 return -ENOTSUPP; 6183 6184 ret = amdgpu_dpm_baco_exit(adev); 6185 if (ret) 6186 return ret; 6187 6188 if (ras && adev->ras_enabled && 6189 adev->nbio.funcs->enable_doorbell_interrupt) 6190 adev->nbio.funcs->enable_doorbell_interrupt(adev, true); 6191 6192 if (amdgpu_passthrough(adev) && adev->nbio.funcs && 6193 adev->nbio.funcs->clear_doorbell_interrupt) 6194 adev->nbio.funcs->clear_doorbell_interrupt(adev); 6195 6196 return 0; 6197 } 6198 6199 /** 6200 * amdgpu_pci_error_detected - Called when a PCI error is detected. 6201 * @pdev: PCI device struct 6202 * @state: PCI channel state 6203 * 6204 * Description: Called when a PCI error is detected. 6205 * 6206 * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT. 6207 */ 6208 pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state) 6209 { 6210 struct drm_device *dev = pci_get_drvdata(pdev); 6211 struct amdgpu_device *adev = drm_to_adev(dev); 6212 struct amdgpu_hive_info *hive __free(xgmi_put_hive) = 6213 amdgpu_get_xgmi_hive(adev); 6214 struct amdgpu_reset_context reset_context; 6215 struct list_head device_list; 6216 6217 dev_info(adev->dev, "PCI error: detected callback!!\n"); 6218 6219 adev->pci_channel_state = state; 6220 6221 switch (state) { 6222 case pci_channel_io_normal: 6223 dev_info(adev->dev, "pci_channel_io_normal: state(%d)!!\n", state); 6224 return PCI_ERS_RESULT_CAN_RECOVER; 6225 case pci_channel_io_frozen: 6226 /* Fatal error, prepare for slot reset */ 6227 dev_info(adev->dev, "pci_channel_io_frozen: state(%d)!!\n", state); 6228 if (hive) { 6229 /* Hive devices should be able to support FW based 6230 * link reset on other devices, if not return. 6231 */ 6232 if (!amdgpu_dpm_is_link_reset_supported(adev)) { 6233 dev_warn(adev->dev, 6234 "No support for XGMI hive yet...\n"); 6235 return PCI_ERS_RESULT_DISCONNECT; 6236 } 6237 /* Set dpc status only if device is part of hive 6238 * Non-hive devices should be able to recover after 6239 * link reset. 6240 */ 6241 amdgpu_reset_set_dpc_status(adev, true); 6242 6243 mutex_lock(&hive->hive_lock); 6244 } else { 6245 if (amdgpu_device_bus_status_check(adev)) 6246 amdgpu_reset_set_dpc_status(adev, true); 6247 } 6248 memset(&reset_context, 0, sizeof(reset_context)); 6249 INIT_LIST_HEAD(&device_list); 6250 6251 amdgpu_device_recovery_prepare(adev, &device_list, hive); 6252 amdgpu_device_recovery_get_reset_lock(adev, &device_list); 6253 amdgpu_device_halt_activities(adev, NULL, &reset_context, &device_list, 6254 hive, false); 6255 if (hive) 6256 mutex_unlock(&hive->hive_lock); 6257 return PCI_ERS_RESULT_NEED_RESET; 6258 case pci_channel_io_perm_failure: 6259 /* Permanent error, prepare for device removal */ 6260 dev_info(adev->dev, "pci_channel_io_perm_failure: state(%d)!!\n", state); 6261 return PCI_ERS_RESULT_DISCONNECT; 6262 } 6263 6264 return PCI_ERS_RESULT_NEED_RESET; 6265 } 6266 6267 /** 6268 * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers 6269 * @pdev: pointer to PCI device 6270 */ 6271 pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev) 6272 { 6273 struct drm_device *dev = pci_get_drvdata(pdev); 6274 struct amdgpu_device *adev = drm_to_adev(dev); 6275 6276 dev_info(adev->dev, "PCI error: mmio enabled callback!!\n"); 6277 6278 /* TODO - dump whatever for debugging purposes */ 6279 6280 /* This called only if amdgpu_pci_error_detected returns 6281 * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still 6282 * works, no need to reset slot. 6283 */ 6284 6285 return PCI_ERS_RESULT_RECOVERED; 6286 } 6287 6288 /** 6289 * amdgpu_pci_slot_reset - Called when PCI slot has been reset. 6290 * @pdev: PCI device struct 6291 * 6292 * Description: This routine is called by the pci error recovery 6293 * code after the PCI slot has been reset, just before we 6294 * should resume normal operations. 6295 */ 6296 pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev) 6297 { 6298 struct drm_device *dev = pci_get_drvdata(pdev); 6299 struct amdgpu_device *adev = drm_to_adev(dev); 6300 struct amdgpu_reset_context reset_context; 6301 struct amdgpu_device *tmp_adev; 6302 struct amdgpu_hive_info *hive; 6303 struct list_head device_list; 6304 struct pci_dev *link_dev; 6305 int r = 0, i, timeout; 6306 u32 memsize; 6307 u16 status; 6308 6309 dev_info(adev->dev, "PCI error: slot reset callback!!\n"); 6310 6311 memset(&reset_context, 0, sizeof(reset_context)); 6312 INIT_LIST_HEAD(&device_list); 6313 hive = amdgpu_get_xgmi_hive(adev); 6314 if (hive) { 6315 mutex_lock(&hive->hive_lock); 6316 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) 6317 list_add_tail(&tmp_adev->reset_list, &device_list); 6318 } else { 6319 list_add_tail(&adev->reset_list, &device_list); 6320 } 6321 6322 if (adev->pcie_reset_ctx.swus) 6323 link_dev = adev->pcie_reset_ctx.swus; 6324 else 6325 link_dev = adev->pdev; 6326 /* wait for asic to come out of reset, timeout = 10s */ 6327 timeout = 10000; 6328 do { 6329 usleep_range(10000, 10500); 6330 r = pci_read_config_word(link_dev, PCI_VENDOR_ID, &status); 6331 timeout -= 10; 6332 } while (timeout > 0 && (status != PCI_VENDOR_ID_ATI) && 6333 (status != PCI_VENDOR_ID_AMD)); 6334 6335 if ((status != PCI_VENDOR_ID_ATI) && (status != PCI_VENDOR_ID_AMD)) { 6336 r = -ETIME; 6337 goto out; 6338 } 6339 6340 amdgpu_device_load_switch_state(adev); 6341 /* Restore PCI confspace */ 6342 amdgpu_device_load_pci_state(pdev); 6343 6344 /* confirm ASIC came out of reset */ 6345 for (i = 0; i < adev->usec_timeout; i++) { 6346 memsize = amdgpu_asic_get_config_memsize(adev); 6347 6348 if (memsize != 0xffffffff) 6349 break; 6350 udelay(1); 6351 } 6352 if (memsize == 0xffffffff) { 6353 r = -ETIME; 6354 goto out; 6355 } 6356 6357 reset_context.method = AMD_RESET_METHOD_NONE; 6358 reset_context.reset_req_dev = adev; 6359 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 6360 set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags); 6361 6362 if (hive) { 6363 reset_context.hive = hive; 6364 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) 6365 tmp_adev->pcie_reset_ctx.in_link_reset = true; 6366 } else { 6367 adev->pcie_reset_ctx.in_link_reset = true; 6368 set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags); 6369 } 6370 6371 r = amdgpu_device_asic_reset(adev, &device_list, &reset_context); 6372 out: 6373 if (!r) { 6374 if (amdgpu_device_cache_pci_state(adev->pdev)) 6375 pci_restore_state(adev->pdev); 6376 dev_info(adev->dev, "PCIe error recovery succeeded\n"); 6377 } else { 6378 dev_err(adev->dev, "PCIe error recovery failed, err:%d\n", r); 6379 if (hive) { 6380 list_for_each_entry(tmp_adev, &device_list, reset_list) 6381 amdgpu_device_unset_mp1_state(tmp_adev); 6382 } 6383 amdgpu_device_recovery_put_reset_lock(adev, &device_list); 6384 } 6385 6386 if (hive) { 6387 mutex_unlock(&hive->hive_lock); 6388 amdgpu_put_xgmi_hive(hive); 6389 } 6390 6391 return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED; 6392 } 6393 6394 /** 6395 * amdgpu_pci_resume() - resume normal ops after PCI reset 6396 * @pdev: pointer to PCI device 6397 * 6398 * Called when the error recovery driver tells us that its 6399 * OK to resume normal operation. 6400 */ 6401 void amdgpu_pci_resume(struct pci_dev *pdev) 6402 { 6403 struct drm_device *dev = pci_get_drvdata(pdev); 6404 struct amdgpu_device *adev = drm_to_adev(dev); 6405 struct list_head device_list; 6406 struct amdgpu_hive_info *hive = NULL; 6407 struct amdgpu_device *tmp_adev = NULL; 6408 6409 dev_info(adev->dev, "PCI error: resume callback!!\n"); 6410 6411 /* Only continue execution for the case of pci_channel_io_frozen */ 6412 if (adev->pci_channel_state != pci_channel_io_frozen) 6413 return; 6414 6415 INIT_LIST_HEAD(&device_list); 6416 6417 hive = amdgpu_get_xgmi_hive(adev); 6418 if (hive) { 6419 mutex_lock(&hive->hive_lock); 6420 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { 6421 tmp_adev->pcie_reset_ctx.in_link_reset = false; 6422 list_add_tail(&tmp_adev->reset_list, &device_list); 6423 } 6424 } else { 6425 adev->pcie_reset_ctx.in_link_reset = false; 6426 list_add_tail(&adev->reset_list, &device_list); 6427 } 6428 amdgpu_device_sched_resume(&device_list, NULL, NULL); 6429 amdgpu_device_gpu_resume(adev, &device_list, false); 6430 amdgpu_device_recovery_put_reset_lock(adev, &device_list); 6431 6432 if (hive) { 6433 mutex_unlock(&hive->hive_lock); 6434 amdgpu_put_xgmi_hive(hive); 6435 } 6436 } 6437 6438 static void amdgpu_device_cache_switch_state(struct amdgpu_device *adev) 6439 { 6440 struct pci_dev *swus, *swds; 6441 int r; 6442 6443 swus = adev->link_dev != adev->pdev ? adev->link_dev : NULL; 6444 if (!swus) 6445 return; 6446 swds = pci_upstream_bridge(adev->pdev); 6447 6448 /* If already saved, return */ 6449 if (adev->pcie_reset_ctx.swus) 6450 return; 6451 /* Upstream bridge is ATI, assume it's SWUS/DS architecture */ 6452 r = pci_save_state(swds); 6453 if (r) 6454 return; 6455 adev->pcie_reset_ctx.swds_pcistate = pci_store_saved_state(swds); 6456 6457 r = pci_save_state(swus); 6458 if (r) 6459 return; 6460 adev->pcie_reset_ctx.swus_pcistate = pci_store_saved_state(swus); 6461 6462 adev->pcie_reset_ctx.swus = swus; 6463 } 6464 6465 static void amdgpu_device_load_switch_state(struct amdgpu_device *adev) 6466 { 6467 struct pci_dev *pdev; 6468 int r; 6469 6470 if (!adev->pcie_reset_ctx.swds_pcistate || 6471 !adev->pcie_reset_ctx.swus_pcistate) 6472 return; 6473 6474 pdev = adev->pcie_reset_ctx.swus; 6475 r = pci_load_saved_state(pdev, adev->pcie_reset_ctx.swus_pcistate); 6476 if (!r) { 6477 pci_restore_state(pdev); 6478 } else { 6479 dev_warn(adev->dev, "Failed to load SWUS state, err:%d\n", r); 6480 return; 6481 } 6482 6483 pdev = pci_upstream_bridge(adev->pdev); 6484 r = pci_load_saved_state(pdev, adev->pcie_reset_ctx.swds_pcistate); 6485 if (!r) 6486 pci_restore_state(pdev); 6487 else 6488 dev_warn(adev->dev, "Failed to load SWDS state, err:%d\n", r); 6489 } 6490 6491 bool amdgpu_device_cache_pci_state(struct pci_dev *pdev) 6492 { 6493 struct drm_device *dev = pci_get_drvdata(pdev); 6494 struct amdgpu_device *adev = drm_to_adev(dev); 6495 int r; 6496 6497 if (amdgpu_sriov_vf(adev)) 6498 return false; 6499 6500 r = pci_save_state(pdev); 6501 if (!r) { 6502 kfree(adev->pci_state); 6503 6504 adev->pci_state = pci_store_saved_state(pdev); 6505 6506 if (!adev->pci_state) { 6507 dev_err(adev->dev, "Failed to store PCI saved state"); 6508 return false; 6509 } 6510 } else { 6511 dev_warn(adev->dev, "Failed to save PCI state, err:%d\n", r); 6512 return false; 6513 } 6514 6515 amdgpu_device_cache_switch_state(adev); 6516 6517 return true; 6518 } 6519 6520 bool amdgpu_device_load_pci_state(struct pci_dev *pdev) 6521 { 6522 struct drm_device *dev = pci_get_drvdata(pdev); 6523 struct amdgpu_device *adev = drm_to_adev(dev); 6524 int r; 6525 6526 if (!adev->pci_state) 6527 return false; 6528 6529 r = pci_load_saved_state(pdev, adev->pci_state); 6530 6531 if (!r) { 6532 pci_restore_state(pdev); 6533 } else { 6534 dev_warn(adev->dev, "Failed to load PCI state, err:%d\n", r); 6535 return false; 6536 } 6537 6538 return true; 6539 } 6540 6541 void amdgpu_device_flush_hdp(struct amdgpu_device *adev, 6542 struct amdgpu_ring *ring) 6543 { 6544 #ifdef CONFIG_X86_64 6545 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) 6546 return; 6547 #endif 6548 if (adev->gmc.xgmi.connected_to_cpu) 6549 return; 6550 6551 if (ring && ring->funcs->emit_hdp_flush) { 6552 amdgpu_ring_emit_hdp_flush(ring); 6553 return; 6554 } 6555 6556 if (!ring && amdgpu_sriov_runtime(adev)) { 6557 if (!amdgpu_kiq_hdp_flush(adev)) 6558 return; 6559 } 6560 6561 amdgpu_hdp_flush(adev, ring); 6562 } 6563 6564 void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev, 6565 struct amdgpu_ring *ring) 6566 { 6567 #ifdef CONFIG_X86_64 6568 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) 6569 return; 6570 #endif 6571 if (adev->gmc.xgmi.connected_to_cpu) 6572 return; 6573 6574 amdgpu_hdp_invalidate(adev, ring); 6575 } 6576 6577 int amdgpu_in_reset(struct amdgpu_device *adev) 6578 { 6579 return atomic_read(&adev->reset_domain->in_gpu_reset); 6580 } 6581 6582 /** 6583 * amdgpu_device_halt() - bring hardware to some kind of halt state 6584 * 6585 * @adev: amdgpu_device pointer 6586 * 6587 * Bring hardware to some kind of halt state so that no one can touch it 6588 * any more. It will help to maintain error context when error occurred. 6589 * Compare to a simple hang, the system will keep stable at least for SSH 6590 * access. Then it should be trivial to inspect the hardware state and 6591 * see what's going on. Implemented as following: 6592 * 6593 * 1. drm_dev_unplug() makes device inaccessible to user space(IOCTLs, etc), 6594 * clears all CPU mappings to device, disallows remappings through page faults 6595 * 2. amdgpu_irq_disable_all() disables all interrupts 6596 * 3. amdgpu_fence_driver_hw_fini() signals all HW fences 6597 * 4. set adev->no_hw_access to avoid potential crashes after setp 5 6598 * 5. amdgpu_device_unmap_mmio() clears all MMIO mappings 6599 * 6. pci_disable_device() and pci_wait_for_pending_transaction() 6600 * flush any in flight DMA operations 6601 */ 6602 void amdgpu_device_halt(struct amdgpu_device *adev) 6603 { 6604 struct pci_dev *pdev = adev->pdev; 6605 struct drm_device *ddev = adev_to_drm(adev); 6606 6607 amdgpu_xcp_dev_unplug(adev); 6608 drm_dev_unplug(ddev); 6609 6610 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); 6611 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); 6612 6613 amdgpu_irq_disable_all(adev); 6614 6615 amdgpu_fence_driver_hw_fini(adev); 6616 6617 adev->no_hw_access = true; 6618 6619 amdgpu_device_unmap_mmio(adev); 6620 6621 pci_disable_device(pdev); 6622 pci_wait_for_pending_transaction(pdev); 6623 } 6624 6625 /** 6626 * amdgpu_device_get_gang - return a reference to the current gang 6627 * @adev: amdgpu_device pointer 6628 * 6629 * Returns: A new reference to the current gang leader. 6630 */ 6631 struct dma_fence *amdgpu_device_get_gang(struct amdgpu_device *adev) 6632 { 6633 struct dma_fence *fence; 6634 6635 rcu_read_lock(); 6636 fence = dma_fence_get_rcu_safe(&adev->gang_submit); 6637 rcu_read_unlock(); 6638 return fence; 6639 } 6640 6641 /** 6642 * amdgpu_device_switch_gang - switch to a new gang 6643 * @adev: amdgpu_device pointer 6644 * @gang: the gang to switch to 6645 * 6646 * Try to switch to a new gang. 6647 * Returns: NULL if we switched to the new gang or a reference to the current 6648 * gang leader. 6649 */ 6650 struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev, 6651 struct dma_fence *gang) 6652 { 6653 struct dma_fence *old = NULL; 6654 6655 dma_fence_get(gang); 6656 do { 6657 dma_fence_put(old); 6658 old = amdgpu_device_get_gang(adev); 6659 if (old == gang) 6660 break; 6661 6662 if (!dma_fence_is_signaled(old)) { 6663 dma_fence_put(gang); 6664 return old; 6665 } 6666 6667 } while (cmpxchg((struct dma_fence __force **)&adev->gang_submit, 6668 old, gang) != old); 6669 6670 /* 6671 * Drop it once for the exchanged reference in adev and once for the 6672 * thread local reference acquired in amdgpu_device_get_gang(). 6673 */ 6674 dma_fence_put(old); 6675 dma_fence_put(old); 6676 return NULL; 6677 } 6678 6679 /** 6680 * amdgpu_device_enforce_isolation - enforce HW isolation 6681 * @adev: the amdgpu device pointer 6682 * @ring: the HW ring the job is supposed to run on 6683 * @job: the job which is about to be pushed to the HW ring 6684 * 6685 * Makes sure that only one client at a time can use the GFX block. 6686 * Returns: The dependency to wait on before the job can be pushed to the HW. 6687 * The function is called multiple times until NULL is returned. 6688 */ 6689 struct dma_fence *amdgpu_device_enforce_isolation(struct amdgpu_device *adev, 6690 struct amdgpu_ring *ring, 6691 struct amdgpu_job *job) 6692 { 6693 struct drm_sched_fence *f = job->base.s_fence; 6694 struct amdgpu_isolation *isolation; 6695 struct dma_fence *dep; 6696 void *owner; 6697 int r; 6698 6699 /* 6700 * For now enforce isolation only for the GFX block since we only need 6701 * the cleaner shader on those rings. 6702 */ 6703 if (ring->funcs->type != AMDGPU_RING_TYPE_GFX && 6704 ring->funcs->type != AMDGPU_RING_TYPE_COMPUTE) 6705 return NULL; 6706 6707 isolation = &adev->isolation[ring->xcp_id == AMDGPU_XCP_NO_PARTITION ? 6708 0 : ring->xcp_id]; 6709 6710 /* 6711 * All submissions where enforce isolation is false are handled as if 6712 * they come from a single client. Use ~0l as the owner to distinct it 6713 * from kernel submissions where the owner is NULL. 6714 */ 6715 owner = job->enforce_isolation ? f->owner : (void *)~0l; 6716 6717 mutex_lock(&adev->enforce_isolation_mutex); 6718 6719 /* 6720 * The "spearhead" submission is the first one which changes the 6721 * ownership to its client. We always need to wait for it to be 6722 * pushed to the HW before proceeding with anything. 6723 */ 6724 if (&f->scheduled != isolation->spearhead && 6725 !dma_fence_is_signaled(isolation->spearhead)) { 6726 dep = isolation->spearhead; 6727 goto out_grab_ref; 6728 } 6729 6730 if (isolation->owner != owner) { 6731 6732 /* 6733 * Wait for any gang to be assembled before switching to a 6734 * different owner or otherwise we could deadlock the 6735 * submissions. 6736 */ 6737 if (!job->gang_submit) { 6738 dep = amdgpu_device_get_gang(adev); 6739 if (!dma_fence_is_signaled(dep)) 6740 goto out_return_dep; 6741 dma_fence_put(dep); 6742 } 6743 6744 dma_fence_put(isolation->spearhead); 6745 isolation->spearhead = dma_fence_get(&f->scheduled); 6746 amdgpu_sync_move(&isolation->active, &isolation->prev); 6747 trace_amdgpu_isolation(isolation->owner, owner); 6748 isolation->owner = owner; 6749 } 6750 6751 /* 6752 * Specifying the ring here helps to pipeline submissions even when 6753 * isolation is enabled. If that is not desired for testing NULL can be 6754 * used instead of the ring to enforce a CPU round trip while switching 6755 * between clients. 6756 */ 6757 dep = amdgpu_sync_peek_fence(&isolation->prev, ring); 6758 r = amdgpu_sync_fence(&isolation->active, &f->finished, GFP_NOWAIT); 6759 if (r) 6760 dev_warn(adev->dev, "OOM tracking isolation\n"); 6761 6762 out_grab_ref: 6763 dma_fence_get(dep); 6764 out_return_dep: 6765 mutex_unlock(&adev->enforce_isolation_mutex); 6766 return dep; 6767 } 6768 6769 bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev) 6770 { 6771 switch (adev->asic_type) { 6772 #ifdef CONFIG_DRM_AMDGPU_SI 6773 case CHIP_HAINAN: 6774 #endif 6775 case CHIP_TOPAZ: 6776 /* chips with no display hardware */ 6777 return false; 6778 #ifdef CONFIG_DRM_AMDGPU_SI 6779 case CHIP_TAHITI: 6780 case CHIP_PITCAIRN: 6781 case CHIP_VERDE: 6782 case CHIP_OLAND: 6783 #endif 6784 #ifdef CONFIG_DRM_AMDGPU_CIK 6785 case CHIP_BONAIRE: 6786 case CHIP_HAWAII: 6787 case CHIP_KAVERI: 6788 case CHIP_KABINI: 6789 case CHIP_MULLINS: 6790 #endif 6791 case CHIP_TONGA: 6792 case CHIP_FIJI: 6793 case CHIP_POLARIS10: 6794 case CHIP_POLARIS11: 6795 case CHIP_POLARIS12: 6796 case CHIP_VEGAM: 6797 case CHIP_CARRIZO: 6798 case CHIP_STONEY: 6799 /* chips with display hardware */ 6800 return true; 6801 default: 6802 /* IP discovery */ 6803 if (!amdgpu_ip_version(adev, DCE_HWIP, 0) || 6804 (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK)) 6805 return false; 6806 return true; 6807 } 6808 } 6809 6810 ssize_t amdgpu_get_soft_full_reset_mask(struct amdgpu_ring *ring) 6811 { 6812 ssize_t size = 0; 6813 6814 if (!ring || !ring->adev) 6815 return size; 6816 6817 if (amdgpu_device_should_recover_gpu(ring->adev)) 6818 size |= AMDGPU_RESET_TYPE_FULL; 6819 6820 if (unlikely(!ring->adev->debug_disable_soft_recovery) && 6821 !amdgpu_sriov_vf(ring->adev) && ring->funcs->soft_recovery) 6822 size |= AMDGPU_RESET_TYPE_SOFT_RECOVERY; 6823 6824 return size; 6825 } 6826 6827 ssize_t amdgpu_show_reset_mask(char *buf, uint32_t supported_reset) 6828 { 6829 ssize_t size = 0; 6830 6831 if (supported_reset == 0) { 6832 size += sysfs_emit_at(buf, size, "unsupported"); 6833 size += sysfs_emit_at(buf, size, "\n"); 6834 return size; 6835 6836 } 6837 6838 if (supported_reset & AMDGPU_RESET_TYPE_SOFT_RECOVERY) 6839 size += sysfs_emit_at(buf, size, "soft_recovery "); 6840 6841 if (supported_reset & AMDGPU_RESET_TYPE_PER_QUEUE) 6842 size += sysfs_emit_at(buf, size, "queue "); 6843 6844 if (supported_reset & AMDGPU_RESET_TYPE_PER_PIPE) 6845 size += sysfs_emit_at(buf, size, "pipe "); 6846 6847 if (supported_reset & AMDGPU_RESET_TYPE_FULL) 6848 size += sysfs_emit_at(buf, size, "full "); 6849 6850 size += sysfs_emit_at(buf, size, "\n"); 6851 return size; 6852 } 6853 6854 void amdgpu_device_set_uid(struct amdgpu_uid *uid_info, 6855 enum amdgpu_uid_type type, uint8_t inst, 6856 uint64_t uid) 6857 { 6858 if (!uid_info) 6859 return; 6860 6861 if (type >= AMDGPU_UID_TYPE_MAX) { 6862 dev_err_once(uid_info->adev->dev, "Invalid UID type %d\n", 6863 type); 6864 return; 6865 } 6866 6867 if (inst >= AMDGPU_UID_INST_MAX) { 6868 dev_err_once(uid_info->adev->dev, "Invalid UID instance %d\n", 6869 inst); 6870 return; 6871 } 6872 6873 if (uid_info->uid[type][inst] != 0) { 6874 dev_warn_once( 6875 uid_info->adev->dev, 6876 "Overwriting existing UID %llu for type %d instance %d\n", 6877 uid_info->uid[type][inst], type, inst); 6878 } 6879 6880 uid_info->uid[type][inst] = uid; 6881 } 6882 6883 u64 amdgpu_device_get_uid(struct amdgpu_uid *uid_info, 6884 enum amdgpu_uid_type type, uint8_t inst) 6885 { 6886 if (!uid_info) 6887 return 0; 6888 6889 if (type >= AMDGPU_UID_TYPE_MAX) { 6890 dev_err_once(uid_info->adev->dev, "Invalid UID type %d\n", 6891 type); 6892 return 0; 6893 } 6894 6895 if (inst >= AMDGPU_UID_INST_MAX) { 6896 dev_err_once(uid_info->adev->dev, "Invalid UID instance %d\n", 6897 inst); 6898 return 0; 6899 } 6900 6901 return uid_info->uid[type][inst]; 6902 } 6903